diff --git a/docs/config-catalog.md b/docs/config-catalog.md index d437649d9b..b28634ccae 100644 --- a/docs/config-catalog.md +++ b/docs/config-catalog.md @@ -688,7 +688,7 @@ export interface ReplayModelConfig { } ``` -Source: [`packages/support/llm-replay/src/index.ts:459`](../packages/support/llm-replay/src/index.ts) +Source: [`packages/support/llm-replay/src/index.ts:496`](../packages/support/llm-replay/src/index.ts) ## `@deepseek-ai/dsh-llm-retry` diff --git a/packages/support/llm-replay/README.md b/packages/support/llm-replay/README.md index a8d811f35b..50c4976b8d 100644 --- a/packages/support/llm-replay/README.md +++ b/packages/support/llm-replay/README.md @@ -8,7 +8,7 @@ Its consumers are the ACP snapshot harness in `examples/acp-agent` and the `stre The fixture IS the persisted session log (`/session.jsonl`). Its `assistant/chunk` events carry every `StreamChunk`, so grouping them by `(turn, step)` reconstructs each `stream()` call's chunk sequence (one model call per loop step). Recording is therefore "run the real agent once and harvest the `.jsonl`", done by the snapshot harness — this plugin does not record. A fixture may carry its `request/header` content tokenized to `{{system}}`/`{{tools}}` (the harness pins that content in one scenario and scrubs the rest); replay is indifferent — derivation reads only `assistant/chunk` events and the line-0 session header. -Two failure modes are not reconstructable from `assistant/chunk` alone — a pure throw before any chunk (e.g. an HTTP 401, where the log holds only a `turn/end {error}` and no chunks) and a cancel/hang (timing, not chunk content). A scenario that needs those supplies an optional sidecar (`/replay.override.json`: a `ReplayEntry[]`) that REPLACES the derived script. A `hang` entry may name `readyFile`; replay writes that empty marker after its prefix chunks reach the loop and before it waits for cancellation, so an external driver can cancel deterministically without observing a presentation update. +Two failure modes are not reconstructable from `assistant/chunk` alone — a pure throw before any chunk (e.g. an HTTP 401, where the log holds only a `turn/end {error}` and no chunks) and a cancel/hang (timing, not chunk content). A scenario that needs those supplies an optional sidecar (`/replay.override.json`) that either REPLACES the derived script (a bare `ReplayEntry[]`) or AUGMENTS it (`{ patches: [{ at, entry }] }`: keep every JSONL-derived call, swap only the named 0-based call indexes; `at` equal to the derived length appends — the slot for the retry attempt that follows an injected transient throw). A `hang` entry may name `readyFile`; replay writes that empty marker after its prefix chunks reach the loop and before it waits for cancellation, so an external driver can cancel deterministically without observing a presentation update. ## Nested agents: per-session keying diff --git a/packages/support/llm-replay/src/index.ts b/packages/support/llm-replay/src/index.ts index f9637536f8..e54eabc4b6 100644 --- a/packages/support/llm-replay/src/index.ts +++ b/packages/support/llm-replay/src/index.ts @@ -200,26 +200,63 @@ export function deriveReplayScript(events: SessionEvent[]): ReplayEntry[] { } /** - * Build the replay script for the PRIMARY session: the sidecar override if - * present, otherwise the script derived from the recorded session JSONL. - * Fail-loud if the JSONL fixture is missing (the scenario was never recorded) — - * never silently returns an empty script, so a coverage hole can't masquerade - * as a passing replay. + * One positional patch in an augmentation sidecar: replaces the derived + * entry at call index `at` (0-based) with `entry`, or appends when `at` + * equals the derived length (an extra recorded-after-the-fact call, e.g. the + * retry attempt following an injected transient throw). + */ +export interface ReplayOverridePatch { + /** 0-based call index into the derived script; == length appends. */ + at: number + /** The replacement (or appended) entry at that call position. */ + entry: ReplayEntry +} + +/** + * Override sidecar document: either the legacy whole-script replacement (a + * bare `ReplayEntry[]`) or the augmentation form `{ patches }`, which keeps + * the JSONL-derived script and swaps only the named call indexes — the shape + * for "turn N errors, everything else replays as recorded". + */ +export type ReplayOverrideDoc = ReplayEntry[] | { patches: ReplayOverridePatch[] } + +/** + * Load the PRIMARY session's replay script: the sidecar override when present + * (whole-script replacement or `{ patches }` augmentation over the derived + * script), else the script derived from the session JSONL (fail-loud when the + * fixture is missing). * @param config - the fixture paths; only `file` and `overrideFile` are consulted. - * @returns the primary session's replay entries. + * @returns the resolved primary-session script. */ export function loadReplayScript(config: ReplayConfig): ReplayEntry[] { if (config.overrideFile !== undefined && existsSync(config.overrideFile)) { const parsed: unknown = JSON.parse(readFileSync(config.overrideFile, 'utf8')) - if (!Array.isArray(parsed)) { - throw new Error(`llm-replay: override is not a JSON array: ${config.overrideFile}`) + if (Array.isArray(parsed)) return parsed as ReplayEntry[] + const doc = parsed as { patches?: unknown } + if (typeof parsed !== 'object' || parsed === null || !Array.isArray(doc.patches)) { + throw new Error(`llm-replay: override must be a ReplayEntry[] or { patches: [...] }: ${config.overrideFile}`) } - return parsed as ReplayEntry[] + const script = deriveScriptFromFile(config.file) + for (const patch of doc.patches as ReplayOverridePatch[]) { + if (!Number.isInteger(patch.at) || patch.at < 0 || patch.at > script.length) { + throw new Error( + `llm-replay: override patch index ${String(patch.at)} out of range ` + + `(derived script has ${script.length} call(s); == length appends): ${config.overrideFile}`, + ) + } + script[patch.at] = patch.entry + } + return script } - if (!existsSync(config.file)) { - throw new Error(`llm-replay: fixture not found: ${config.file} — run \`pnpm run test:snapshot:record\` first`) + return deriveScriptFromFile(config.file) +} + +/** Derive the primary script from the session JSONL, failing loud on a missing fixture. */ +function deriveScriptFromFile(file: string): ReplayEntry[] { + if (!existsSync(file)) { + throw new Error(`llm-replay: fixture not found: ${file} — run \`pnpm run test:snapshot:record\` first`) } - return deriveReplayScript(parseSessionLog(readFileSync(config.file, 'utf8'))) + return deriveReplayScript(parseSessionLog(readFileSync(file, 'utf8'))) } /** diff --git a/packages/support/llm-replay/tests/llm-replay.spec.ts b/packages/support/llm-replay/tests/llm-replay.spec.ts index 584a87abf6..1bd8d47405 100644 --- a/packages/support/llm-replay/tests/llm-replay.spec.ts +++ b/packages/support/llm-replay/tests/llm-replay.spec.ts @@ -207,7 +207,52 @@ describe('loadReplayScript', () => { writeFileSync(file, sessionJsonl([]), 'utf8') const overrideFile = join(dir, 'replay.override.json') writeFileSync(overrideFile, '{"not":"array"}', 'utf8') - expect(() => loadReplayScript({ file, overrideFile })).toThrow(/not a JSON array/) + expect(() => loadReplayScript({ file, overrideFile })).toThrow(/ReplayEntry\[\] or \{ patches/) + }) + + it('patches form: swaps the named call index and keeps derived siblings', () => { + const callB: StreamChunk[] = [ + { type: 'block-start', index: 0, blockType: 'text' }, + { type: 'text-delta', index: 0, text: 'two' }, + { type: 'finish', reason: { kind: 'stop' } }, + ] + let seq = 1 + writeFileSync(file, sessionJsonl([ + ...TEXT_CHUNKS.map(c => chunkEvent(seq++, 1, 1, c)), + ...callB.map(c => chunkEvent(seq++, 1, 2, c)), + ]), 'utf8') + const overrideFile = join(dir, 'replay.override.json') + writeFileSync(overrideFile, JSON.stringify({ + patches: [{ at: 0, entry: { kind: 'throw', chunks: [], message: 'transient', code: 'SERVER' } }], + }), 'utf8') + expect(loadReplayScript({ file, overrideFile })).toEqual([ + { kind: 'throw', chunks: [], message: 'transient', code: 'SERVER' }, + { kind: 'chunks', chunks: callB }, + ]) + }) + + it('patches form: at == derived length appends (the retry-attempt slot)', () => { + writeFileSync(file, sessionJsonl(TEXT_CHUNKS.map((c, i) => chunkEvent(i + 1, 1, 1, c))), 'utf8') + const overrideFile = join(dir, 'replay.override.json') + writeFileSync(overrideFile, JSON.stringify({ + patches: [ + { at: 0, entry: { kind: 'throw', chunks: [], message: '429', code: 'RATE_LIMIT' } }, + { at: 1, entry: { kind: 'chunks', chunks: TEXT_CHUNKS } }, + ], + }), 'utf8') + expect(loadReplayScript({ file, overrideFile })).toEqual([ + { kind: 'throw', chunks: [], message: '429', code: 'RATE_LIMIT' }, + { kind: 'chunks', chunks: TEXT_CHUNKS }, + ]) + }) + + it('patches form: an out-of-range index fails loud with the derived length', () => { + writeFileSync(file, sessionJsonl(TEXT_CHUNKS.map((c, i) => chunkEvent(i + 1, 1, 1, c))), 'utf8') + const overrideFile = join(dir, 'replay.override.json') + for (const at of [2, -1, 1.5]) { + writeFileSync(overrideFile, JSON.stringify({ patches: [{ at, entry: { kind: 'hang' } }] }), 'utf8') + expect(() => loadReplayScript({ file, overrideFile })).toThrow(/patch index .* out of range/) + } }) })