What changed, and why
The base Sentry wiring (#353) captures unhandled errors and auto-traces routes. This follow-up makes the signal everwhen already emits — the [ai] cost line, the x-run-id correlation, the player behind a request — queryable, correlated, and safe, without touching a single bracketed console line or any fail-open/fail-closed behavior.
Three additive pieces, each below: every priced model call becomes a span (metrics by task/model/lane); each request is tagged with its run and a PII-free player id; and a scrub drops player free-text and PII before anything leaves the process. No new dependency, no Docker or CI change — instrumentation code and its tests.
The AI seam: one sink, many calls
The gateway already enriches every call's telemetry in one place — emit(). It fanned out to a single swappable observer (the eval harness / unit tests). The change adds a second, persistent channel — addAiCallObserver — so a long-lived subscriber can coexist with that swappable slot instead of clobbering it:
Two channels: the swappable observer, and a Set of persistent sinks.
server/utils/ai-gateway.ts · 553 lines
server/utils/ai-gateway.ts
⋯ 94 lines hidden (lines 1–94)
1/**
2 * The AI gateway — one transport for every structured model call, dispatched
3 * per task by the routing table (ai-routing.ts). Callers keep their own prompts,
4 * schemas, parsing, guards, and never-throw envelopes; the gateway only decides
5 * WHICH provider runs the request and HOW (model + parameters), and emits the
6 * cost telemetry the go-to-market plan's dispatch 1 requires.
7 *
8 * Telemetry: every call logs one structured line (task, lane, model, tokens
9 * in/out, latency, ok) — intentional production output, suppressed under vitest.
10 * The eval harness subscribes via setAiCallObserver to price bake-off lanes.
11 */
12import OpenAI from "openai";
13import Anthropic from "@anthropic-ai/sdk";
14import { routeFor, type AiTask, type AiLane } from "./ai-routing";
15import { currentRunId } from "./run-context";
16import { costUsd } from "./cost";
17import { spendStore } from "./spend-store";
18import { devFixtureFor } from "./dev-fixtures";
20let openaiClient: OpenAI | null = null;
21let anthropicClient: Anthropic | null = null;
23/** Reads a runtime-config key, falling back to raw env outside a Nuxt context
24 * (the eval harness exercises these utils directly in node). */
25function configuredKey(
26 name: "openaiApiKey" | "anthropicApiKey",
27 envName: string,
28): string | undefined {
29 let key: string | undefined;
30 try {
31 key = useRuntimeConfig()[name] as string | undefined;
32 } catch {
33 key = undefined;
34 }
35 return key || process.env[envName];
36}
38export function getOpenAIClient(): OpenAI {
39 if (!openaiClient) {
40 const apiKey = configuredKey("openaiApiKey", "OPENAI_API_KEY");
41 if (!apiKey) throw new Error("OpenAI API key is not configured");
42 openaiClient = new OpenAI({
43 apiKey,
44 // Fail fast rather than hang on the SDK's 10-minute default if a
45 // call stalls (e.g. a bad key or network hiccup).
46 timeout: 30_000,
47 maxRetries: 1,
48 });
49 }
50 return openaiClient;
51}
53export function getAnthropicClient(): Anthropic {
54 if (!anthropicClient) {
55 const apiKey = configuredKey("anthropicApiKey", "ANTHROPIC_API_KEY");
56 if (!apiKey) throw new Error("Anthropic API key is not configured");
57 anthropicClient = new Anthropic({
58 apiKey,
59 // The epilogue's adaptive thinking can run long; everything else is
60 // seconds. 60s bounds the worst case without hanging a turn forever.
61 timeout: 60_000,
62 maxRetries: 1,
63 });
64 }
65 return anthropicClient;
66}
68export interface AiUsage {
69 inputTokens: number;
70 outputTokens: number;
71 cacheReadTokens: number;
72 /** Tokens WRITTEN to the prompt cache this call (Anthropic cache-creation),
73 * billed at the cache-write premium. Always 0 on the OpenAI lane (auto-cache,
74 * no write line) and on any call with no cached prefix. */
75 cacheWriteTokens: number;
76}
78export interface AiCallTelemetry {
79 task: AiTask;
80 lane: AiLane;
81 model: string;
82 ms: number;
83 ok: boolean;
84 usage?: AiUsage;
85 /** The game run this call belongs to (from the request's x-run-id); absent
86 * outside a run-scoped request. The key that groups calls into a run. */
87 runId?: string;
88 /** The call's dollar cost from the rate card; absent when usage is missing
89 * (a failed call) or the model has no listed rate. */
90 usd?: number;
91}
93type AiCallObserver = (t: AiCallTelemetry) => void;
94let observer: AiCallObserver | null = null;
96/** The eval harness (and unit tests) subscribe here; null to clear. Single-slot on
97 * purpose — a later set replaces the earlier one, the swap-per-run semantics those
98 * callers rely on. For a long-lived additional sink, use addAiCallObserver below. */
99export function setAiCallObserver(fn: AiCallObserver | null): void {
100 observer = fn;
101}
103/** Persistent additional telemetry sinks, kept apart from the swappable observer above
104 * so a long-lived subscriber (the Sentry span emitter, registered once at server
105 * startup) coexists with the eval/test observer instead of clobbering its slot. Returns
106 * an unsubscribe. emit() isolates each sink, so a faulty one can't break the [ai] line,
107 * the spend feed, or the turn. */
108const sinks = new Set<AiCallObserver>();
109export function addAiCallObserver(fn: AiCallObserver): () => void {
110 sinks.add(fn);
111 return () => {
112 sinks.delete(fn);
113 };
114}
⋯ 439 lines hidden (lines 115–553)
116function emit(t: AiCallTelemetry): void {
117 // Enrich once, here, so both the success and failure call sites get the run
118 // id, and so the observer (the eval harness) sees the same priced line the
119 // log does. usd needs usage, so it rides only on calls that reported tokens.
120 const enriched: AiCallTelemetry = {
121 ...t,
122 runId: t.runId ?? currentRunId(),
123 usd: t.usd ?? (t.usage ? costUsd(t.model, t.usage, t.lane) : undefined),
124 };
125 observer?.(enriched);
126 // Fan out to persistent sinks (the Sentry span emitter). Each is isolated: a sink is
127 // secondary telemetry and must never break the cost line, the spend feed, or the turn.
128 for (const sink of sinks) {
129 try {
130 sink(enriched);
131 } catch {
132 // Swallow — a faulty sink degrades observability, never the game.
133 }
134 }
135 // One greppable line per call — the GTM's "structured log line is enough to
136 // start". Intentional production telemetry; quiet under the test runner.
137 if (!process.env.VITEST) {
138 console.info(`[ai] ${JSON.stringify(enriched)}`);
139 }
140 // Feed the spend cap: record this call's cost (best-effort, non-blocking).
141 // Outside the VITEST guard — unlike the log line, recording must happen under
142 // test (so the wiring is exercised) and for eval traffic (which spends real
143 // money), not just in production.
144 if (enriched.usd) void spendStore().record(enriched.usd);
145}
147export interface ChatTurn {
148 role: "user" | "assistant";
149 content: string;
150}
152/**
153 * Thrown when Claude's own safety classifiers decline a request mid-flight
154 * (Claude 4 returns stop_reason "refusal"). It is NOT an infra error and NOT an
155 * empty answer — it's a model-side block, so callers surface it as a moderation
156 * block, not a "try again" refund. (See Anthropic's streaming-refusals guide.)
157 */
158export class ModelRefusalError extends Error {
159 constructor() {
160 super("The model declined this request (safety refusal).");
161 this.name = "ModelRefusalError";
162 }
163}
165/**
166 * A prompt split for prompt caching: a STATIC, reusable prefix (the
167 * figure/turn-agnostic rubric) and the FRESH per-call remainder (figure, message,
168 * ledger, …). The gateway puts `cached` behind a cache_control breakpoint so it
169 * bills at the cache-read rate on repeat calls; `fresh` is re-read every time.
170 * Builders return this to keep the static rubric first — the cacheable prefix.
171 * A plain string is treated as all-fresh (no cache), so unrefactored callers (and
172 * the rarer one-shot tasks) are unchanged.
173 */
174export interface SplitPrompt {
175 cached: string;
176 fresh: string;
177}
179/** Normalize a builder's output: a plain string is all-fresh (uncached). */
180function splitOf(system: string | SplitPrompt): SplitPrompt {
181 return typeof system === "string" ? { cached: "", fresh: system } : system;
182}
184/** The cache breakpoint. Default 5-minute ephemeral TTL — long enough to span the
185 * turns of one run, short enough to age out quietly between players. (For denser
186 * traffic, `{ type: 'ephemeral', ttl: '1h' }` widens cross-run reuse at 2x write.) */
187const CACHE = { type: "ephemeral" as const };
189export interface StructuredRequest {
190 task: AiTask;
191 /** The built prompt — either a plain string (all-fresh, uncached) or a
192 * {cached, fresh} split whose static prefix is cached. Single-shot tasks send
193 * `fresh` as the user message; with `turns` it is the conversation's context. */
194 system: string | SplitPrompt;
195 turns?: ChatTurn[];
196 /** OpenAI requires a schema name; Anthropic ignores it. */
197 schemaName: string;
198 schema: Record<string, unknown>;
199}
201/**
202 * Runs one structured completion on the task's routed lane and returns the raw
203 * JSON text (trimmed), or null on an empty answer. Throws on transport errors —
204 * callers own their never-throw envelopes, exactly as before the seam.
205 */
206export async function completeStructured(
207 req: StructuredRequest,
208): Promise<string | null> {
209 // Dev-only deterministic fixtures (issue #105): when the dev panel has scripted
210 // this task, return its canned JSON instead of calling a model — every mechanic
211 // below the seam still runs for real, but no key is needed and nothing is spent.
212 // Inert in production: the registry can only be populated behind the dev gate, so
213 // here it is always empty and the real lanes below run untouched. Telemetry still
214 // logs one line (lane 'fixture', no usage → no cost recorded).
215 const fixture = devFixtureFor(req.task);
216 if (fixture !== undefined) {
217 emit({
218 task: req.task,
219 lane: "fixture",
220 model: "fixture",
221 ms: 0,
222 ok: true,
223 });
224 return JSON.stringify(fixture);
225 }
227 const { lane, route } = routeFor(req.task);
228 const model =
229 lane === "anthropic" ? route.anthropic.model : route.openai.model;
230 const started = Date.now();
231 try {
232 const result =
233 lane === "anthropic" ? await viaAnthropic(req) : await viaOpenAI(req);
234 emit({
235 task: req.task,
236 lane,
237 model,
238 ms: Date.now() - started,
239 ok: true,
240 usage: result.usage,
241 });
242 return result.content;
243 } catch (error) {
244 emit({ task: req.task, lane, model, ms: Date.now() - started, ok: false });
245 throw error;
246 }
247}
249interface LaneResult {
250 content: string | null;
251 usage?: AiUsage;
252}
254async function viaOpenAI(req: StructuredRequest): Promise<LaneResult> {
255 const { route } = routeFor(req.task);
256 const params = route.openai;
257 const client = getOpenAIClient();
258 // Static-first: the cached rubric leads (OpenAI auto-caches a stable prefix —
259 // no cache_control needed), then the fresh data. A plain-string prompt
260 // (cached '') keeps the prior shape: the whole prompt as the system message.
261 const { cached, fresh } = splitOf(req.system);
262 const turns = req.turns ?? [];
263 const messages = turns.length
264 ? [
265 {
266 role: "system" as const,
267 content: cached ? `${cached}\n\n${fresh}` : fresh,
268 },
269 ...turns,
270 ]
271 : cached
272 ? [
273 { role: "system" as const, content: cached },
274 { role: "user" as const, content: fresh },
275 ]
276 : [{ role: "system" as const, content: fresh }];
277 const completion = await client.chat.completions.create({
278 model: params.model,
279 messages,
280 // gpt-5.5 is a reasoning model: omit temperature, keep effort low, leave
281 // headroom for hidden reasoning tokens inside the output cap.
282 ...(params.reasoningEffort
283 ? { reasoning_effort: params.reasoningEffort }
284 : {}),
285 max_completion_tokens: params.maxTokens,
286 response_format: {
287 type: "json_schema",
288 json_schema: { name: req.schemaName, strict: true, schema: req.schema },
289 },
290 });
291 const usage = completion.usage
292 ? {
293 inputTokens: completion.usage.prompt_tokens ?? 0,
294 outputTokens: completion.usage.completion_tokens ?? 0,
295 cacheReadTokens:
296 completion.usage.prompt_tokens_details?.cached_tokens ?? 0,
297 cacheWriteTokens: 0,
298 }
299 : undefined;
300 return {
301 content: completion.choices?.[0]?.message?.content?.trim() || null,
302 usage,
303 };
304}
306async function viaAnthropic(req: StructuredRequest): Promise<LaneResult> {
307 const { route } = routeFor(req.task);
308 const params = route.anthropic;
309 const client = getAnthropicClient();
311 // Anthropic requires the first message to be a user turn. The static rubric
312 // rides in the system param behind a cache_control breakpoint (so it bills at
313 // the cache-read rate on repeat calls); a single-shot task's fresh data is the
314 // user message, a conversational task's fresh context is a second (uncached)
315 // system block before its turns. A plain-string prompt (cached '') keeps the
316 // prior shape: no system on single-shot, the whole prompt as the user message.
317 const { cached, fresh } = splitOf(req.system);
318 const turns = req.turns ?? [];
319 const systemBlocks: Anthropic.TextBlockParam[] = [];
320 if (cached)
321 systemBlocks.push({ type: "text", text: cached, cache_control: CACHE });
322 if (turns.length && fresh) systemBlocks.push({ type: "text", text: fresh });
323 const messages: Anthropic.MessageParam[] = turns.length
324 ? turns[0].role === "user"
325 ? turns
326 : [
327 { role: "user" as const, content: "(The conversation resumes.)" },
328 ...turns,
329 ]
330 : [{ role: "user" as const, content: fresh }];
332 const response = await client.messages.create({
333 model: params.model,
334 max_tokens: params.maxTokens,
335 ...(systemBlocks.length ? { system: systemBlocks } : {}),
336 messages,
337 // Thinking rejects non-default sampling params — temperature only rides
338 // on thinking-off calls (the family rule the routing table can't express).
339 ...(params.temperature !== undefined && params.thinking !== "adaptive"
340 ? { temperature: params.temperature }
341 : {}),
342 ...(params.thinking === "adaptive"
343 ? { thinking: { type: "adaptive" as const } }
344 : {}),
345 output_config: {
346 ...(params.effort ? { effort: params.effort } : {}),
347 format: { type: "json_schema" as const, schema: req.schema },
348 },
349 });
351 // Claude's streaming classifiers can decline mid-flight (stop_reason
352 // "refusal"); that's a model-side block, surfaced distinctly so callers
353 // don't mistake it for an empty/infra response. Cast: the union may predate
354 // this SDK's type for the value.
355 if ((response.stop_reason as string) === "refusal") {
356 throw new ModelRefusalError();
357 }
359 // Thinking blocks (epilogue) precede the text block; take the text.
360 const text = response.content
361 .find((b): b is Anthropic.TextBlock => b.type === "text")
362 ?.text?.trim();
363 const usage: AiUsage = {
364 inputTokens: response.usage?.input_tokens ?? 0,
365 outputTokens: response.usage?.output_tokens ?? 0,
366 cacheReadTokens: response.usage?.cache_read_input_tokens ?? 0,
367 cacheWriteTokens: response.usage?.cache_creation_input_tokens ?? 0,
368 };
369 return { content: text || null, usage };
370}
372// ─────────────────────────────────────────────────────────────────────────────
373// Streaming prose — the reusable text-delta path. The structured path above
374// returns JSON in one shot; this returns PLAIN TEXT as it generates, so a
375// long-form prose task (the Chronicler's telling) can be watched being written
376// in realtime instead of staring at a loader. Phase 1 ships it on the
377// off-critical-path chronicle endpoint; a later pass reuses it for the reply.
378// ─────────────────────────────────────────────────────────────────────────────
380export interface ProseRequest {
381 task: AiTask;
382 /** The built prompt — a plain string (all-fresh, uncached) or a {cached, fresh}
383 * split. Single-shot: `cached` rides the system param as a cache breakpoint,
384 * `fresh` is the user message (a plain string becomes the whole user message). */
385 system: string | SplitPrompt;
386}
388export interface ProseStreamResult {
389 text: string;
390 usage?: AiUsage;
391 /** Claude's classifiers declined mid-stream (stop_reason "refusal"). The
392 * caller discards the partial and keeps its prior content. */
393 refused: boolean;
394}
396/** Renders a dev fixture (a scripted chronicle object, or anything) to prose so
397 * the streaming path stays inert-but-shaped under the dev gate. */
398function fixtureToProse(fixture: unknown): string {
399 const f = fixture as { title?: unknown; paragraphs?: unknown };
400 if (typeof f?.title === "string" && Array.isArray(f.paragraphs)) {
401 return [
402 f.title,
403 ...(f.paragraphs as unknown[]).filter(
404 (p): p is string => typeof p === "string",
405 ),
406 ].join("\n\n");
407 }
408 return JSON.stringify(fixture);
409}
411/**
412 * Streams a prose completion on the task's routed lane, invoking `onDelta` with
413 * each text chunk as it arrives and resolving with the full text + usage at the
414 * end. Honors the routing table's model/params for the task. Throws on transport
415 * errors (callers own their never-throw envelopes), exactly like completeStructured.
416 */
417export async function streamProse(
418 req: ProseRequest,
419 onDelta?: (delta: string) => void,
420): Promise<ProseStreamResult> {
421 const fixture = devFixtureFor(req.task);
422 if (fixture !== undefined) {
423 const text = fixtureToProse(fixture);
424 onDelta?.(text);
425 emit({
426 task: req.task,
427 lane: "fixture",
428 model: "fixture",
429 ms: 0,
430 ok: true,
431 });
432 return { text, refused: false };
433 }
435 const { lane, route } = routeFor(req.task);
436 const model =
437 lane === "anthropic" ? route.anthropic.model : route.openai.model;
438 const started = Date.now();
439 try {
440 const result =
441 lane === "anthropic"
442 ? await streamViaAnthropic(req, onDelta)
443 : await streamViaOpenAI(req, onDelta);
444 emit({
445 task: req.task,
446 lane,
447 model,
448 ms: Date.now() - started,
449 ok: true,
450 usage: result.usage,
451 });
452 return result;
453 } catch (error) {
454 emit({ task: req.task, lane, model, ms: Date.now() - started, ok: false });
455 throw error;
456 }
457}
459async function streamViaAnthropic(
460 req: ProseRequest,
461 onDelta?: (delta: string) => void,
462): Promise<ProseStreamResult> {
463 const { route } = routeFor(req.task);
464 const params = route.anthropic;
465 const client = getAnthropicClient();
467 // Static-first: the cached prefix rides the system param behind a cache_control
468 // breakpoint; the fresh remainder is the user message (a plain string is the
469 // whole user message, cached '' → no system, exactly the prior shape).
470 const { cached, fresh } = splitOf(req.system);
471 const stream = client.messages.stream({
472 model: params.model,
473 max_tokens: params.maxTokens,
474 ...(cached
475 ? {
476 system: [
477 { type: "text" as const, text: cached, cache_control: CACHE },
478 ],
479 }
480 : {}),
481 messages: [{ role: "user" as const, content: fresh }],
482 ...(params.temperature !== undefined && params.thinking !== "adaptive"
483 ? { temperature: params.temperature }
484 : {}),
485 ...(params.thinking === "adaptive"
486 ? { thinking: { type: "adaptive" as const } }
487 : {}),
488 // No `format`: plain prose, so the stream is clean readable text — never
489 // JSON syntax on screen. Effort still rides for Sonnet's pinned depth.
490 ...(params.effort ? { output_config: { effort: params.effort } } : {}),
491 });
492 if (onDelta) stream.on("text", (delta: string) => onDelta(delta));
494 const final = await stream.finalMessage();
495 const refused = (final.stop_reason as string) === "refusal";
496 const text = final.content
497 .filter((b): b is Anthropic.TextBlock => b.type === "text")
498 .map((b) => b.text)
499 .join("");
500 const usage: AiUsage = {
501 inputTokens: final.usage?.input_tokens ?? 0,
502 outputTokens: final.usage?.output_tokens ?? 0,
503 cacheReadTokens: final.usage?.cache_read_input_tokens ?? 0,
504 cacheWriteTokens: final.usage?.cache_creation_input_tokens ?? 0,
505 };
506 return { text, usage, refused };
507}
509async function streamViaOpenAI(
510 req: ProseRequest,
511 onDelta?: (delta: string) => void,
512): Promise<ProseStreamResult> {
513 const { route } = routeFor(req.task);
514 const params = route.openai;
515 const client = getOpenAIClient();
517 // Static-first: cached rubric as the system message (OpenAI auto-caches the
518 // stable prefix), fresh as the user message; a plain string stays the system.
519 const { cached, fresh } = splitOf(req.system);
520 const stream = await client.chat.completions.create({
521 model: params.model,
522 messages: cached
523 ? [
524 { role: "system" as const, content: cached },
525 { role: "user" as const, content: fresh },
526 ]
527 : [{ role: "system" as const, content: fresh }],
528 ...(params.reasoningEffort
529 ? { reasoning_effort: params.reasoningEffort }
530 : {}),
531 max_completion_tokens: params.maxTokens,
532 stream: true,
533 stream_options: { include_usage: true },
534 });
535 let text = "";
536 let usage: AiUsage | undefined;
537 for await (const chunk of stream) {
538 const delta = chunk.choices?.[0]?.delta?.content;
539 if (delta) {
540 text += delta;
541 onDelta?.(delta);
542 }
543 if (chunk.usage) {
544 usage = {
545 inputTokens: chunk.usage.prompt_tokens ?? 0,
546 outputTokens: chunk.usage.completion_tokens ?? 0,
547 cacheReadTokens: chunk.usage.prompt_tokens_details?.cached_tokens ?? 0,
548 cacheWriteTokens: 0,
549 };
550 }
551 }
552 return { text, usage, refused: false };
553}
emit() then fans out to the sinks with each one isolated — the load-bearing bit:
A faulty sink is swallowed — it can't break the cost line or the turn.
server/utils/ai-gateway.ts · 553 lines
server/utils/ai-gateway.ts
⋯ 123 lines hidden (lines 1–123)
1/**
2 * The AI gateway — one transport for every structured model call, dispatched
3 * per task by the routing table (ai-routing.ts). Callers keep their own prompts,
4 * schemas, parsing, guards, and never-throw envelopes; the gateway only decides
5 * WHICH provider runs the request and HOW (model + parameters), and emits the
6 * cost telemetry the go-to-market plan's dispatch 1 requires.
7 *
8 * Telemetry: every call logs one structured line (task, lane, model, tokens
9 * in/out, latency, ok) — intentional production output, suppressed under vitest.
10 * The eval harness subscribes via setAiCallObserver to price bake-off lanes.
11 */
12import OpenAI from "openai";
13import Anthropic from "@anthropic-ai/sdk";
14import { routeFor, type AiTask, type AiLane } from "./ai-routing";
15import { currentRunId } from "./run-context";
16import { costUsd } from "./cost";
17import { spendStore } from "./spend-store";
18import { devFixtureFor } from "./dev-fixtures";
20let openaiClient: OpenAI | null = null;
21let anthropicClient: Anthropic | null = null;
23/** Reads a runtime-config key, falling back to raw env outside a Nuxt context
24 * (the eval harness exercises these utils directly in node). */
25function configuredKey(
26 name: "openaiApiKey" | "anthropicApiKey",
27 envName: string,
28): string | undefined {
29 let key: string | undefined;
30 try {
31 key = useRuntimeConfig()[name] as string | undefined;
32 } catch {
33 key = undefined;
34 }
35 return key || process.env[envName];
36}
38export function getOpenAIClient(): OpenAI {
39 if (!openaiClient) {
40 const apiKey = configuredKey("openaiApiKey", "OPENAI_API_KEY");
41 if (!apiKey) throw new Error("OpenAI API key is not configured");
42 openaiClient = new OpenAI({
43 apiKey,
44 // Fail fast rather than hang on the SDK's 10-minute default if a
45 // call stalls (e.g. a bad key or network hiccup).
46 timeout: 30_000,
47 maxRetries: 1,
48 });
49 }
50 return openaiClient;
51}
53export function getAnthropicClient(): Anthropic {
54 if (!anthropicClient) {
55 const apiKey = configuredKey("anthropicApiKey", "ANTHROPIC_API_KEY");
56 if (!apiKey) throw new Error("Anthropic API key is not configured");
57 anthropicClient = new Anthropic({
58 apiKey,
59 // The epilogue's adaptive thinking can run long; everything else is
60 // seconds. 60s bounds the worst case without hanging a turn forever.
61 timeout: 60_000,
62 maxRetries: 1,
63 });
64 }
65 return anthropicClient;
66}
68export interface AiUsage {
69 inputTokens: number;
70 outputTokens: number;
71 cacheReadTokens: number;
72 /** Tokens WRITTEN to the prompt cache this call (Anthropic cache-creation),
73 * billed at the cache-write premium. Always 0 on the OpenAI lane (auto-cache,
74 * no write line) and on any call with no cached prefix. */
75 cacheWriteTokens: number;
76}
78export interface AiCallTelemetry {
79 task: AiTask;
80 lane: AiLane;
81 model: string;
82 ms: number;
83 ok: boolean;
84 usage?: AiUsage;
85 /** The game run this call belongs to (from the request's x-run-id); absent
86 * outside a run-scoped request. The key that groups calls into a run. */
87 runId?: string;
88 /** The call's dollar cost from the rate card; absent when usage is missing
89 * (a failed call) or the model has no listed rate. */
90 usd?: number;
91}
93type AiCallObserver = (t: AiCallTelemetry) => void;
94let observer: AiCallObserver | null = null;
96/** The eval harness (and unit tests) subscribe here; null to clear. Single-slot on
97 * purpose — a later set replaces the earlier one, the swap-per-run semantics those
98 * callers rely on. For a long-lived additional sink, use addAiCallObserver below. */
99export function setAiCallObserver(fn: AiCallObserver | null): void {
100 observer = fn;
101}
103/** Persistent additional telemetry sinks, kept apart from the swappable observer above
104 * so a long-lived subscriber (the Sentry span emitter, registered once at server
105 * startup) coexists with the eval/test observer instead of clobbering its slot. Returns
106 * an unsubscribe. emit() isolates each sink, so a faulty one can't break the [ai] line,
107 * the spend feed, or the turn. */
108const sinks = new Set<AiCallObserver>();
109export function addAiCallObserver(fn: AiCallObserver): () => void {
110 sinks.add(fn);
111 return () => {
112 sinks.delete(fn);
113 };
114}
116function emit(t: AiCallTelemetry): void {
117 // Enrich once, here, so both the success and failure call sites get the run
118 // id, and so the observer (the eval harness) sees the same priced line the
119 // log does. usd needs usage, so it rides only on calls that reported tokens.
120 const enriched: AiCallTelemetry = {
121 ...t,
122 runId: t.runId ?? currentRunId(),
123 usd: t.usd ?? (t.usage ? costUsd(t.model, t.usage, t.lane) : undefined),
124 };
125 observer?.(enriched);
126 // Fan out to persistent sinks (the Sentry span emitter). Each is isolated: a sink is
127 // secondary telemetry and must never break the cost line, the spend feed, or the turn.
128 for (const sink of sinks) {
129 try {
130 sink(enriched);
131 } catch {
132 // Swallow — a faulty sink degrades observability, never the game.
133 }
134 }
⋯ 419 lines hidden (lines 135–553)
135 // One greppable line per call — the GTM's "structured log line is enough to
136 // start". Intentional production telemetry; quiet under the test runner.
137 if (!process.env.VITEST) {
138 console.info(`[ai] ${JSON.stringify(enriched)}`);
139 }
140 // Feed the spend cap: record this call's cost (best-effort, non-blocking).
141 // Outside the VITEST guard — unlike the log line, recording must happen under
142 // test (so the wiring is exercised) and for eval traffic (which spends real
143 // money), not just in production.
144 if (enriched.usd) void spendStore().record(enriched.usd);
145}
147export interface ChatTurn {
148 role: "user" | "assistant";
149 content: string;
150}
152/**
153 * Thrown when Claude's own safety classifiers decline a request mid-flight
154 * (Claude 4 returns stop_reason "refusal"). It is NOT an infra error and NOT an
155 * empty answer — it's a model-side block, so callers surface it as a moderation
156 * block, not a "try again" refund. (See Anthropic's streaming-refusals guide.)
157 */
158export class ModelRefusalError extends Error {
159 constructor() {
160 super("The model declined this request (safety refusal).");
161 this.name = "ModelRefusalError";
162 }
163}
165/**
166 * A prompt split for prompt caching: a STATIC, reusable prefix (the
167 * figure/turn-agnostic rubric) and the FRESH per-call remainder (figure, message,
168 * ledger, …). The gateway puts `cached` behind a cache_control breakpoint so it
169 * bills at the cache-read rate on repeat calls; `fresh` is re-read every time.
170 * Builders return this to keep the static rubric first — the cacheable prefix.
171 * A plain string is treated as all-fresh (no cache), so unrefactored callers (and
172 * the rarer one-shot tasks) are unchanged.
173 */
174export interface SplitPrompt {
175 cached: string;
176 fresh: string;
177}
179/** Normalize a builder's output: a plain string is all-fresh (uncached). */
180function splitOf(system: string | SplitPrompt): SplitPrompt {
181 return typeof system === "string" ? { cached: "", fresh: system } : system;
182}
184/** The cache breakpoint. Default 5-minute ephemeral TTL — long enough to span the
185 * turns of one run, short enough to age out quietly between players. (For denser
186 * traffic, `{ type: 'ephemeral', ttl: '1h' }` widens cross-run reuse at 2x write.) */
187const CACHE = { type: "ephemeral" as const };
189export interface StructuredRequest {
190 task: AiTask;
191 /** The built prompt — either a plain string (all-fresh, uncached) or a
192 * {cached, fresh} split whose static prefix is cached. Single-shot tasks send
193 * `fresh` as the user message; with `turns` it is the conversation's context. */
194 system: string | SplitPrompt;
195 turns?: ChatTurn[];
196 /** OpenAI requires a schema name; Anthropic ignores it. */
197 schemaName: string;
198 schema: Record<string, unknown>;
199}
201/**
202 * Runs one structured completion on the task's routed lane and returns the raw
203 * JSON text (trimmed), or null on an empty answer. Throws on transport errors —
204 * callers own their never-throw envelopes, exactly as before the seam.
205 */
206export async function completeStructured(
207 req: StructuredRequest,
208): Promise<string | null> {
209 // Dev-only deterministic fixtures (issue #105): when the dev panel has scripted
210 // this task, return its canned JSON instead of calling a model — every mechanic
211 // below the seam still runs for real, but no key is needed and nothing is spent.
212 // Inert in production: the registry can only be populated behind the dev gate, so
213 // here it is always empty and the real lanes below run untouched. Telemetry still
214 // logs one line (lane 'fixture', no usage → no cost recorded).
215 const fixture = devFixtureFor(req.task);
216 if (fixture !== undefined) {
217 emit({
218 task: req.task,
219 lane: "fixture",
220 model: "fixture",
221 ms: 0,
222 ok: true,
223 });
224 return JSON.stringify(fixture);
225 }
227 const { lane, route } = routeFor(req.task);
228 const model =
229 lane === "anthropic" ? route.anthropic.model : route.openai.model;
230 const started = Date.now();
231 try {
232 const result =
233 lane === "anthropic" ? await viaAnthropic(req) : await viaOpenAI(req);
234 emit({
235 task: req.task,
236 lane,
237 model,
238 ms: Date.now() - started,
239 ok: true,
240 usage: result.usage,
241 });
242 return result.content;
243 } catch (error) {
244 emit({ task: req.task, lane, model, ms: Date.now() - started, ok: false });
245 throw error;
246 }
247}
249interface LaneResult {
250 content: string | null;
251 usage?: AiUsage;
252}
254async function viaOpenAI(req: StructuredRequest): Promise<LaneResult> {
255 const { route } = routeFor(req.task);
256 const params = route.openai;
257 const client = getOpenAIClient();
258 // Static-first: the cached rubric leads (OpenAI auto-caches a stable prefix —
259 // no cache_control needed), then the fresh data. A plain-string prompt
260 // (cached '') keeps the prior shape: the whole prompt as the system message.
261 const { cached, fresh } = splitOf(req.system);
262 const turns = req.turns ?? [];
263 const messages = turns.length
264 ? [
265 {
266 role: "system" as const,
267 content: cached ? `${cached}\n\n${fresh}` : fresh,
268 },
269 ...turns,
270 ]
271 : cached
272 ? [
273 { role: "system" as const, content: cached },
274 { role: "user" as const, content: fresh },
275 ]
276 : [{ role: "system" as const, content: fresh }];
277 const completion = await client.chat.completions.create({
278 model: params.model,
279 messages,
280 // gpt-5.5 is a reasoning model: omit temperature, keep effort low, leave
281 // headroom for hidden reasoning tokens inside the output cap.
282 ...(params.reasoningEffort
283 ? { reasoning_effort: params.reasoningEffort }
284 : {}),
285 max_completion_tokens: params.maxTokens,
286 response_format: {
287 type: "json_schema",
288 json_schema: { name: req.schemaName, strict: true, schema: req.schema },
289 },
290 });
291 const usage = completion.usage
292 ? {
293 inputTokens: completion.usage.prompt_tokens ?? 0,
294 outputTokens: completion.usage.completion_tokens ?? 0,
295 cacheReadTokens:
296 completion.usage.prompt_tokens_details?.cached_tokens ?? 0,
297 cacheWriteTokens: 0,
298 }
299 : undefined;
300 return {
301 content: completion.choices?.[0]?.message?.content?.trim() || null,
302 usage,
303 };
304}
306async function viaAnthropic(req: StructuredRequest): Promise<LaneResult> {
307 const { route } = routeFor(req.task);
308 const params = route.anthropic;
309 const client = getAnthropicClient();
311 // Anthropic requires the first message to be a user turn. The static rubric
312 // rides in the system param behind a cache_control breakpoint (so it bills at
313 // the cache-read rate on repeat calls); a single-shot task's fresh data is the
314 // user message, a conversational task's fresh context is a second (uncached)
315 // system block before its turns. A plain-string prompt (cached '') keeps the
316 // prior shape: no system on single-shot, the whole prompt as the user message.
317 const { cached, fresh } = splitOf(req.system);
318 const turns = req.turns ?? [];
319 const systemBlocks: Anthropic.TextBlockParam[] = [];
320 if (cached)
321 systemBlocks.push({ type: "text", text: cached, cache_control: CACHE });
322 if (turns.length && fresh) systemBlocks.push({ type: "text", text: fresh });
323 const messages: Anthropic.MessageParam[] = turns.length
324 ? turns[0].role === "user"
325 ? turns
326 : [
327 { role: "user" as const, content: "(The conversation resumes.)" },
328 ...turns,
329 ]
330 : [{ role: "user" as const, content: fresh }];
332 const response = await client.messages.create({
333 model: params.model,
334 max_tokens: params.maxTokens,
335 ...(systemBlocks.length ? { system: systemBlocks } : {}),
336 messages,
337 // Thinking rejects non-default sampling params — temperature only rides
338 // on thinking-off calls (the family rule the routing table can't express).
339 ...(params.temperature !== undefined && params.thinking !== "adaptive"
340 ? { temperature: params.temperature }
341 : {}),
342 ...(params.thinking === "adaptive"
343 ? { thinking: { type: "adaptive" as const } }
344 : {}),
345 output_config: {
346 ...(params.effort ? { effort: params.effort } : {}),
347 format: { type: "json_schema" as const, schema: req.schema },
348 },
349 });
351 // Claude's streaming classifiers can decline mid-flight (stop_reason
352 // "refusal"); that's a model-side block, surfaced distinctly so callers
353 // don't mistake it for an empty/infra response. Cast: the union may predate
354 // this SDK's type for the value.
355 if ((response.stop_reason as string) === "refusal") {
356 throw new ModelRefusalError();
357 }
359 // Thinking blocks (epilogue) precede the text block; take the text.
360 const text = response.content
361 .find((b): b is Anthropic.TextBlock => b.type === "text")
362 ?.text?.trim();
363 const usage: AiUsage = {
364 inputTokens: response.usage?.input_tokens ?? 0,
365 outputTokens: response.usage?.output_tokens ?? 0,
366 cacheReadTokens: response.usage?.cache_read_input_tokens ?? 0,
367 cacheWriteTokens: response.usage?.cache_creation_input_tokens ?? 0,
368 };
369 return { content: text || null, usage };
370}
372// ─────────────────────────────────────────────────────────────────────────────
373// Streaming prose — the reusable text-delta path. The structured path above
374// returns JSON in one shot; this returns PLAIN TEXT as it generates, so a
375// long-form prose task (the Chronicler's telling) can be watched being written
376// in realtime instead of staring at a loader. Phase 1 ships it on the
377// off-critical-path chronicle endpoint; a later pass reuses it for the reply.
378// ─────────────────────────────────────────────────────────────────────────────
380export interface ProseRequest {
381 task: AiTask;
382 /** The built prompt — a plain string (all-fresh, uncached) or a {cached, fresh}
383 * split. Single-shot: `cached` rides the system param as a cache breakpoint,
384 * `fresh` is the user message (a plain string becomes the whole user message). */
385 system: string | SplitPrompt;
386}
388export interface ProseStreamResult {
389 text: string;
390 usage?: AiUsage;
391 /** Claude's classifiers declined mid-stream (stop_reason "refusal"). The
392 * caller discards the partial and keeps its prior content. */
393 refused: boolean;
394}
396/** Renders a dev fixture (a scripted chronicle object, or anything) to prose so
397 * the streaming path stays inert-but-shaped under the dev gate. */
398function fixtureToProse(fixture: unknown): string {
399 const f = fixture as { title?: unknown; paragraphs?: unknown };
400 if (typeof f?.title === "string" && Array.isArray(f.paragraphs)) {
401 return [
402 f.title,
403 ...(f.paragraphs as unknown[]).filter(
404 (p): p is string => typeof p === "string",
405 ),
406 ].join("\n\n");
407 }
408 return JSON.stringify(fixture);
409}
411/**
412 * Streams a prose completion on the task's routed lane, invoking `onDelta` with
413 * each text chunk as it arrives and resolving with the full text + usage at the
414 * end. Honors the routing table's model/params for the task. Throws on transport
415 * errors (callers own their never-throw envelopes), exactly like completeStructured.
416 */
417export async function streamProse(
418 req: ProseRequest,
419 onDelta?: (delta: string) => void,
420): Promise<ProseStreamResult> {
421 const fixture = devFixtureFor(req.task);
422 if (fixture !== undefined) {
423 const text = fixtureToProse(fixture);
424 onDelta?.(text);
425 emit({
426 task: req.task,
427 lane: "fixture",
428 model: "fixture",
429 ms: 0,
430 ok: true,
431 });
432 return { text, refused: false };
433 }
435 const { lane, route } = routeFor(req.task);
436 const model =
437 lane === "anthropic" ? route.anthropic.model : route.openai.model;
438 const started = Date.now();
439 try {
440 const result =
441 lane === "anthropic"
442 ? await streamViaAnthropic(req, onDelta)
443 : await streamViaOpenAI(req, onDelta);
444 emit({
445 task: req.task,
446 lane,
447 model,
448 ms: Date.now() - started,
449 ok: true,
450 usage: result.usage,
451 });
452 return result;
453 } catch (error) {
454 emit({ task: req.task, lane, model, ms: Date.now() - started, ok: false });
455 throw error;
456 }
457}
459async function streamViaAnthropic(
460 req: ProseRequest,
461 onDelta?: (delta: string) => void,
462): Promise<ProseStreamResult> {
463 const { route } = routeFor(req.task);
464 const params = route.anthropic;
465 const client = getAnthropicClient();
467 // Static-first: the cached prefix rides the system param behind a cache_control
468 // breakpoint; the fresh remainder is the user message (a plain string is the
469 // whole user message, cached '' → no system, exactly the prior shape).
470 const { cached, fresh } = splitOf(req.system);
471 const stream = client.messages.stream({
472 model: params.model,
473 max_tokens: params.maxTokens,
474 ...(cached
475 ? {
476 system: [
477 { type: "text" as const, text: cached, cache_control: CACHE },
478 ],
479 }
480 : {}),
481 messages: [{ role: "user" as const, content: fresh }],
482 ...(params.temperature !== undefined && params.thinking !== "adaptive"
483 ? { temperature: params.temperature }
484 : {}),
485 ...(params.thinking === "adaptive"
486 ? { thinking: { type: "adaptive" as const } }
487 : {}),
488 // No `format`: plain prose, so the stream is clean readable text — never
489 // JSON syntax on screen. Effort still rides for Sonnet's pinned depth.
490 ...(params.effort ? { output_config: { effort: params.effort } } : {}),
491 });
492 if (onDelta) stream.on("text", (delta: string) => onDelta(delta));
494 const final = await stream.finalMessage();
495 const refused = (final.stop_reason as string) === "refusal";
496 const text = final.content
497 .filter((b): b is Anthropic.TextBlock => b.type === "text")
498 .map((b) => b.text)
499 .join("");
500 const usage: AiUsage = {
501 inputTokens: final.usage?.input_tokens ?? 0,
502 outputTokens: final.usage?.output_tokens ?? 0,
503 cacheReadTokens: final.usage?.cache_read_input_tokens ?? 0,
504 cacheWriteTokens: final.usage?.cache_creation_input_tokens ?? 0,
505 };
506 return { text, usage, refused };
507}
509async function streamViaOpenAI(
510 req: ProseRequest,
511 onDelta?: (delta: string) => void,
512): Promise<ProseStreamResult> {
513 const { route } = routeFor(req.task);
514 const params = route.openai;
515 const client = getOpenAIClient();
517 // Static-first: cached rubric as the system message (OpenAI auto-caches the
518 // stable prefix), fresh as the user message; a plain string stays the system.
519 const { cached, fresh } = splitOf(req.system);
520 const stream = await client.chat.completions.create({
521 model: params.model,
522 messages: cached
523 ? [
524 { role: "system" as const, content: cached },
525 { role: "user" as const, content: fresh },
526 ]
527 : [{ role: "system" as const, content: fresh }],
528 ...(params.reasoningEffort
529 ? { reasoning_effort: params.reasoningEffort }
530 : {}),
531 max_completion_tokens: params.maxTokens,
532 stream: true,
533 stream_options: { include_usage: true },
534 });
535 let text = "";
536 let usage: AiUsage | undefined;
537 for await (const chunk of stream) {
538 const delta = chunk.choices?.[0]?.delta?.content;
539 if (delta) {
540 text += delta;
541 onDelta?.(delta);
542 }
543 if (chunk.usage) {
544 usage = {
545 inputTokens: chunk.usage.prompt_tokens ?? 0,
546 outputTokens: chunk.usage.completion_tokens ?? 0,
547 cacheReadTokens: chunk.usage.prompt_tokens_details?.cached_tokens ?? 0,
548 cacheWriteTokens: 0,
549 };
550 }
551 }
552 return { text, usage, refused: false };
553}
Turning each call into a span
The plugin registers one sink at server startup. It maps the gateway's telemetry onto a Sentry span — task, lane, model, tokens, cost, latency — so the [ai] line becomes queryable trace data grouped into the turn's trace:
server/plugins/sentry-ai-spans.ts · 46 lines
server/plugins/sentry-ai-spans.ts
1/**
2 * Turns every priced model call into a Sentry span, so the `[ai]` cost telemetry the
3 * gateway already emits also becomes queryable trace data — per-layer latency, tokens,
4 * and cost, grouped into the turn's trace and aggregable in Sentry's Trace Explorer.
5 *
6 * Registered as an additional gateway sink (never the swappable eval/test observer), so
7 * @sentry/nuxt is imported ONLY here — never in the unit-tested ai-gateway. Loaded once
8 * at Nitro startup. A no-op when Sentry is disabled (dev / blanked DSN): the span calls
9 * just don't send. The gateway isolates sink failures, so nothing here can touch a turn.
10 */
11import * as Sentry from "@sentry/nuxt";
12import {
13 addAiCallObserver,
14 type AiCallTelemetry,
15} from "~/server/utils/ai-gateway";
17export default defineNitroPlugin(() => {
18 addAiCallObserver(recordAiSpan);
19});
21function recordAiSpan(t: AiCallTelemetry): void {
22 // emit() runs after the call returns, so backdate the start by the measured duration
23 // and end now — a correctly-timed child of the active turn transaction.
24 const span = Sentry.startInactiveSpan({
25 name: `ai ${t.task}`,
26 op: "gen_ai",
27 startTime: new Date(Date.now() - t.ms),
28 attributes: {
29 "ai.task": t.task,
30 "ai.lane": t.lane,
31 "ai.model": t.model,
32 "ai.ok": t.ok,
33 ...(t.runId ? { "ai.run_id": t.runId } : {}),
34 ...(t.usd !== undefined ? { "ai.cost_usd": t.usd } : {}),
35 ...(t.usage
36 ? {
37 "ai.tokens.input": t.usage.inputTokens,
38 "ai.tokens.output": t.usage.outputTokens,
39 "ai.tokens.cache_read": t.usage.cacheReadTokens,
40 "ai.tokens.cache_write": t.usage.cacheWriteTokens,
41 }
42 : {}),
43 },
44 });
45 span.end();
46}
Identity + run correlation, PII-free
The run-context middleware already reads the sanitized x-run-id per request. It now also stamps the request's Sentry isolation scope, so every error, trace, and log ties back to a run and a player:
run_id tag + an opaque device-id user on the per-request scope.
server/middleware/run-context.ts · 34 lines
server/middleware/run-context.ts
⋯ 23 lines hidden (lines 1–23)
1/**
2 * Stamps each request with its game run id (the `x-run-id` header the client
3 * sends on every AI call) so the gateway's cost telemetry groups per run.
4 *
5 * Header-based on purpose: it works uniformly across the GET and POST AI routes
6 * without consuming the body, and is harmless on requests that carry no id
7 * (static assets, the dashboard) — they simply leave the context unset. The id
8 * is untrusted client input that lands in a log line, so it is bounded and
9 * stripped to id-safe characters before use.
10 */
11import * as Sentry from "@sentry/nuxt";
12import { setRunContext, sanitizeRunId } from "~/server/utils/run-context";
13import { devModeEnabled } from "~/server/utils/dev-fixtures";
14import { deviceId, existingDeviceId } from "~/server/utils/device";
16export default defineEventHandler((event) => {
17 const runId = sanitizeRunId(getHeader(event, "x-run-id"));
18 // In dev mode also stamp the device, so the deterministic AI fixtures key per operator
19 // instead of a process-global slot — a shared dev deploy must never let one visitor's
20 // fixture POST script another visitor's turn (#247). Never in prod: the flag is off and
21 // the fixtures route 404s, so nothing is ever stamped or scripted there.
22 const device = devModeEnabled() ? deviceId(event) : undefined;
23 if (runId || device) setRunContext({ runId: runId || undefined, device });
25 // Tag this request's Sentry isolation scope so every error, trace, and log from it
26 // ties back to the run and the player. PII-free by construction: the sanitized run id
27 // and the opaque device id only (read-only — never minted here), never email.
28 // getIsolationScope() is the per-request scope @sentry/nuxt forks, so nothing leaks
29 // across requests; inert when Sentry is disabled (dev / non-prod build).
30 const scope = Sentry.getIsolationScope();
31 if (runId) scope.setTag("run_id", runId);
32 const player = existingDeviceId(event);
33 if (player) scope.setUser({ id: player });
⋯ 1 line hidden (lines 34–34)
34});
The scrub: free-text never leaves
The last gate before an event is sent. everwhen's moderation seam exists because the dispatch, figure name, and Archive query are sensitive free text — and they all travel in request bodies. So the scrub drops the body, query string, cookies, and any email/ip, keeping only what a triager needs:
sentry.scrub.ts · 34 lines
sentry.scrub.ts
1import type { Event } from "@sentry/nuxt";
3/**
4 * Strip player free-text and PII from a Sentry event before it leaves the process —
5 * wired as `beforeSend` + `beforeSendTransaction` on both the client and the server.
6 *
7 * everwhen's entire moderation seam exists because the 160-char dispatch, the figure
8 * name, and the Archive query are sensitive free text; none of it may ride along in an
9 * error report, and it all travels in request bodies. So we drop the request body, its
10 * query string, and cookies wholesale — never needed to triage, and keeping them would
11 * leak exactly what moderation fences. Identity is reduced to an opaque id (matching the
12 * PII-free `setUser` the run-context middleware sets) — never email, username, or IP.
13 *
14 * Generic over the event kind so the one function types cleanly as both hooks: T is
15 * inferred as ErrorEvent for beforeSend and TransactionEvent for beforeSendTransaction.
16 */
17export function scrubEvent<T extends Event>(event: T): T {
18 const req = event.request;
19 if (req) {
20 delete req.data;
21 delete req.query_string;
22 delete req.cookies;
23 if (req.headers) {
24 delete req.headers.cookie;
25 delete req.headers.authorization;
26 }
27 }
28 if (event.user) {
29 // Keep only an opaque id; discard email/username/ip/geo if anything attached them.
30 const { id } = event.user;
31 event.user = id === undefined ? undefined : { id };
32 }
33 return event;
34}
It's wired as both hooks on each side (client shown; server identical):
beforeSend + beforeSendTransaction — errors and transactions both.
sentry.client.config.ts · 25 lines
sentry.client.config.ts
⋯ 20 lines hidden (lines 1–20)
1import * as Sentry from "@sentry/nuxt";
2import { SENTRY_TRACES_SAMPLE_RATE } from "./sentry.shared";
3import { scrubEvent } from "./sentry.scrub";
5// Runs before the Nuxt app mounts. useRuntimeConfig() is available here and carries the
6// public Sentry settings resolved in nuxt.config, so a deploy can override the DSN or the
7// environment tag at runtime (NUXT_PUBLIC_SENTRY_*) without a rebuild.
8const { public: pub } = useRuntimeConfig();
9const dsn = pub.sentry.dsn;
11Sentry.init({
12 // Report only from a production build that has a DSN wired. Local `npm run dev` (and the
13 // mocked e2e preview, which blanks the DSN) stay inert, so development noise never
14 // reaches the project.
15 enabled: Boolean(dsn) && process.env.NODE_ENV === "production",
16 dsn,
17 environment: pub.sentry.environment,
18 // Browser performance tracing (page loads, navigations, web vitals). Errors are captured
19 // regardless of this sample rate.
20 tracesSampleRate: SENTRY_TRACES_SAMPLE_RATE,
21 // Drop request bodies/cookies and reduce identity to an opaque id before send, so
22 // player free-text and PII never leave the browser.
23 beforeSend: scrubEvent,
24 beforeSendTransaction: scrubEvent,
25});
What the tests pin
Two discriminating specs, both failing on the pre-change code. The sink spec proves the two channels coexist, that unsubscribe works, and — most important — that a throwing sink is isolated from the other subscribers and the call:
A sink that throws: the call still resolves, every other subscriber still fires.
tests/unit/server/utils/ai-call-observer.spec.ts · 86 lines
tests/unit/server/utils/ai-call-observer.spec.ts
⋯ 68 lines hidden (lines 1–68)
1import { describe, it, expect, beforeEach, afterEach } from "vitest";
2import {
3 completeStructured,
4 setAiCallObserver,
5 addAiCallObserver,
6 type AiCallTelemetry,
7} from "../../../../server/utils/ai-gateway";
8import { setDevFixtures } from "../../../../server/utils/dev-fixtures";
9import { setRunContext } from "../../../../server/utils/run-context";
11/**
12 * The gateway fans each priced call out to (a) the single swappable observer the eval
13 * harness and unit tests use and (b) any number of persistent sinks added via
14 * addAiCallObserver — the Sentry span emitter is one. These pin that the two coexist,
15 * that unsubscribe works, and — load-bearing — that a THROWING sink is isolated: it must
16 * never break the other subscribers, the [ai] line, or the call, so faulty observability
17 * can never touch a turn. Driven through the dev fixture lane (no key, no spend).
18 */
19const DEV = "observer-device";
20const req = {
21 task: "judge" as const,
22 system: "unused under the fixture lane",
23 schemaName: "x",
24 schema: { type: "object" as const },
25};
27describe("ai gateway — telemetry sinks (addAiCallObserver)", () => {
28 const offs: Array<() => void> = [];
29 const track = (off: () => void) => (offs.push(off), off);
31 beforeEach(() => {
32 setRunContext({ device: DEV });
33 setDevFixtures(DEV, {
34 scripts: {
35 judge: { craft: "sound", continuity: "neutral", reason: "ok" },
36 },
37 });
38 });
39 afterEach(() => {
40 offs.splice(0).forEach((off) => off());
41 setDevFixtures(DEV, null);
42 setAiCallObserver(null);
43 });
45 it("delivers each call to a persistent sink alongside the primary observer", async () => {
46 const primary: AiCallTelemetry[] = [];
47 const sink: AiCallTelemetry[] = [];
48 setAiCallObserver((t) => primary.push(t));
49 track(addAiCallObserver((t) => sink.push(t)));
51 await completeStructured(req);
53 expect(primary).toHaveLength(1);
54 expect(sink).toHaveLength(1);
55 expect(sink[0].task).toBe("judge");
56 expect(sink[0].lane).toBe("fixture");
57 });
59 it("unsubscribe stops delivery to that sink", async () => {
60 const sink: AiCallTelemetry[] = [];
61 const off = addAiCallObserver((t) => sink.push(t));
62 off();
64 await completeStructured(req);
66 expect(sink).toHaveLength(0);
67 });
69 it("isolates a throwing sink from the other subscribers and the call", async () => {
70 const primary: AiCallTelemetry[] = [];
71 const good: AiCallTelemetry[] = [];
72 setAiCallObserver((t) => primary.push(t));
73 track(
74 addAiCallObserver(() => {
75 throw new Error("sink boom");
76 }),
77 );
78 track(addAiCallObserver((t) => good.push(t)));
80 // The call resolves normally despite the faulty sink...
81 await expect(completeStructured(req)).resolves.not.toBeNull();
82 // ...and every other subscriber still sees the telemetry.
83 expect(primary).toHaveLength(1);
84 expect(good).toHaveLength(1);
85 });
⋯ 1 line hidden (lines 86–86)
86});
The scrub spec pins the exact contract — the dispatch-bearing body is dropped, identity is reduced to an opaque id:
Body / query / cookies / sensitive headers gone; non-sensitive fields kept.
tests/unit/sentry-scrub.spec.ts · 58 lines
tests/unit/sentry-scrub.spec.ts
⋯ 11 lines hidden (lines 1–11)
1import { describe, it, expect } from "vitest";
2import type { Event } from "@sentry/nuxt";
3import { scrubEvent } from "../../sentry.scrub";
5/**
6 * scrubEvent is the last gate before an event leaves the process. It must strip every
7 * carrier of player free-text (the dispatch + figure name ride in the POST body) and PII
8 * (email/ip), while leaving the non-sensitive fields a triager needs. These pin exactly
9 * that contract — a regression here would leak the content moderation exists to fence.
10 */
11describe("scrubEvent — Sentry PII / free-text scrub", () => {
12 it("drops the request body, query string, cookies, and sensitive headers", () => {
13 const event = {
14 request: {
15 url: "https://playeverwhen.com/api/send-message",
16 method: "POST",
17 data: { message: "the 160-char dispatch", figureName: "Cleopatra" },
18 query_string: "q=secret",
19 cookies: "sb-access-token=xyz",
20 headers: {
21 cookie: "sb=1",
22 authorization: "Bearer x",
23 "user-agent": "UA",
24 },
25 },
26 } as unknown as Event;
28 const out = scrubEvent(event);
30 expect(out.request?.data).toBeUndefined();
31 expect(out.request?.query_string).toBeUndefined();
32 expect(out.request?.cookies).toBeUndefined();
33 expect(out.request?.headers?.cookie).toBeUndefined();
34 expect(out.request?.headers?.authorization).toBeUndefined();
35 // Non-sensitive fields survive for triage.
36 expect(out.request?.url).toContain("/api/send-message");
37 expect(out.request?.headers?.["user-agent"]).toBe("UA");
38 });
⋯ 20 lines hidden (lines 39–58)
40 it("reduces user to an opaque id, dropping email / ip / username", () => {
41 const event = {
42 user: {
43 id: "device-uuid",
44 email: "a@b.com",
45 username: "x",
46 ip_address: "1.2.3.4",
47 },
48 } as unknown as Event;
50 expect(scrubEvent(event).user).toEqual({ id: "device-uuid" });
51 });
53 it("clears a user with no id, and tolerates an event with neither field", () => {
54 const noId = { user: { email: "a@b.com" } } as unknown as Event;
55 expect(scrubEvent(noId).user).toBeUndefined();
56 expect(() => scrubEvent({} as Event)).not.toThrow();
57 });
58});