Merge master into codex/simp-session-log-representation

This commit is contained in:
Tianyi Cui
2026-07-17 22:33:42 +08:00
310 changed files with 4718 additions and 1903 deletions

View File

@@ -48,7 +48,7 @@ describe('session-log invariants', () => {
session.append('user/message', { content: [{ type: 'text', text: 'hi' }], source: { kind: 'user' } }, { surfaceOp: 'append' })
session.append('step/start', { turn: 1, step: 1 })
session.append('assistant/chunk', { turn: 1, step: 1, chunk: { type: 'text-delta', index: 0, text: 'h' } })
session.append('assistant/message', { turn: 1, step: 1, content: [{ type: 'tool-call', id: CallId('c1'), name: 'echo', arguments: '{}' }] }, { surfaceOp: 'append' })
session.append('assistant/message', { provenance: { provider: 'mock', model: 'mock' }, turn: 1, step: 1, content: [{ type: 'tool-call', id: CallId('c1'), name: 'echo', arguments: '{}' }] }, { surfaceOp: 'append' })
session.append('tool/call', { turn: 1, step: 1, callId: CallId('c1'), name: 'echo', arguments: '{}' })
session.append('tool/result', { turn: 1, step: 1, callId: CallId('c1'), content: [{ type: 'text', text: 'ok' }], isError: false }, { surfaceOp: 'append' })
session.append('step/end', { turn: 1, step: 1 })
@@ -195,7 +195,7 @@ describe('session-log invariants', () => {
expect(() => {
session.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } })
session.append('step/start', { turn: 1, step: 1 })
session.append('assistant/message', { turn: 1, step: 1, content: [
session.append('assistant/message', { provenance: { provider: 'mock', model: 'mock' }, turn: 1, step: 1, content: [
{ type: 'tool-call', id: CallId('crashed'), name: 'bash', arguments: '{}' },
] }, { surfaceOp: 'append' })
session.append('tool/result', {
@@ -251,10 +251,10 @@ describe('session-log invariants', () => {
expect(() => {
session.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } })
session.append('step/start', { turn: 1, step: 1 })
session.append('assistant/message', { turn: 1, step: 1, content: [] }, { surfaceOp: 'append' })
session.append('assistant/message', { provenance: { provider: 'mock', model: 'mock' }, turn: 1, step: 1, content: [] }, { surfaceOp: 'append' })
session.append('step/end', { turn: 1, step: 1 })
session.append('step/start', { turn: 1, step: 2 })
session.append('assistant/message', { turn: 1, step: 2, content: [] }, { surfaceOp: 'append' })
session.append('assistant/message', { provenance: { provider: 'mock', model: 'mock' }, turn: 1, step: 2, content: [] }, { surfaceOp: 'append' })
session.append('step/end', { turn: 1, step: 2 })
session.append('turn/end', { turn: 1, reason: { kind: 'completed' } })
session.append('turn/start', { turn: 2, trigger: { kind: 'message', source: { kind: 'user' } } })
@@ -316,7 +316,7 @@ describe('session-log invariants', () => {
const session = ctx.sessions.create()
session.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } })
session.append('step/start', { turn: 1, step: 1 })
expect(() => session.append('assistant/message', { turn: 1, step: 2, content: [] }, { surfaceOp: 'append' }))
expect(() => session.append('assistant/message', { provenance: { provider: 'mock', model: 'mock' }, turn: 1, step: 2, content: [] }, { surfaceOp: 'append' }))
.toThrow(/open is turn 1\/step 1/)
})
})
@@ -473,7 +473,7 @@ describe('surface contract under the invariants composition', () => {
session.append('step/start', { turn: 1, step: 1 })
expect(() => {
session.append('user/message', { content: [{ type: 'text', text: 'hi' }], source: { kind: 'user' } }, { surfaceOp: 'append' })
session.append('assistant/message', { turn: 1, step: 1, content: [] }, { surfaceOp: 'append', sourceEventSeqs: [1] })
session.append('assistant/message', { provenance: { provider: 'mock', model: 'mock' }, turn: 1, step: 1, content: [] }, { surfaceOp: 'append', sourceEventSeqs: [1] })
}).not.toThrow()
})
@@ -483,7 +483,7 @@ describe('surface contract under the invariants composition', () => {
session.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } })
session.append('step/start', { turn: 1, step: 1 })
session.append('user/message', { content: [{ type: 'text', text: 'a' }], source: { kind: 'user' } }, { surfaceOp: 'append' })
session.append('assistant/message', { turn: 1, step: 1, content: [] }, { surfaceOp: { op: 'replace', start: 2, end: 2 }, sourceEventSeqs: [2] })
session.append('assistant/message', { provenance: { provider: 'mock', model: 'mock' }, turn: 1, step: 1, content: [] }, { surfaceOp: { op: 'replace', start: 2, end: 2 }, sourceEventSeqs: [2] })
// no throw — well-formed replace op
})
@@ -492,7 +492,7 @@ describe('surface contract under the invariants composition', () => {
const session = ctx.sessions.create()
session.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } })
expect(() => {
session.append('assistant/message', { turn: 1, step: 1, content: [] }, { surfaceOp: 'append', sourceEventSeqs: [] })
session.append('assistant/message', { provenance: { provider: 'mock', model: 'mock' }, turn: 1, step: 1, content: [] }, { surfaceOp: 'append', sourceEventSeqs: [] })
}).toThrow(/must not be empty/)
})
@@ -502,7 +502,7 @@ describe('surface contract under the invariants composition', () => {
session.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } })
session.append('user/message', { content: [{ type: 'text', text: 'a' }], source: { kind: 'user' } }, { surfaceOp: 'append' })
expect(() => {
session.append('assistant/message', { turn: 1, step: 1, content: [] }, { surfaceOp: 'append', sourceEventSeqs: [1, 1] })
session.append('assistant/message', { provenance: { provider: 'mock', model: 'mock' }, turn: 1, step: 1, content: [] }, { surfaceOp: 'append', sourceEventSeqs: [1, 1] })
}).toThrow(/must not contain duplicates/)
})
@@ -513,7 +513,7 @@ describe('surface contract under the invariants composition', () => {
// The next event is seq 1. Referencing its own seq fails on "must reference
// earlier events" (the check order is: earlier first, then unknown).
expect(() => {
session.append('assistant/message', { turn: 1, step: 1, content: [] }, { surfaceOp: 'append', sourceEventSeqs: [1] })
session.append('assistant/message', { provenance: { provider: 'mock', model: 'mock' }, turn: 1, step: 1, content: [] }, { surfaceOp: 'append', sourceEventSeqs: [1] })
}).toThrow(/must reference earlier/)
})
@@ -526,7 +526,7 @@ describe('surface contract under the invariants composition', () => {
session.append('step/start', { turn: 1, step: 1 })
// seqs so far: 0, 1. The next event at seq 2 references seq 1 → valid.
expect(() => {
session.append('assistant/message', { turn: 1, step: 1, content: [] }, { surfaceOp: 'append', sourceEventSeqs: [1] })
session.append('assistant/message', { provenance: { provider: 'mock', model: 'mock' }, turn: 1, step: 1, content: [] }, { surfaceOp: 'append', sourceEventSeqs: [1] })
}).not.toThrow()
})
@@ -535,7 +535,7 @@ describe('surface contract under the invariants composition', () => {
const session = ctx.sessions.create()
session.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } })
expect(() => {
session.append('assistant/message', { turn: 1, step: 1, content: [] }, { surfaceOp: 'append', sourceEventSeqs: [99] })
session.append('assistant/message', { provenance: { provider: 'mock', model: 'mock' }, turn: 1, step: 1, content: [] }, { surfaceOp: 'append', sourceEventSeqs: [99] })
}).toThrow(/must reference earlier/)
})
@@ -548,7 +548,7 @@ describe('surface contract under the invariants composition', () => {
session.append('user/message', { content: [{ type: 'text', text: 'b' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) // seq 3
// Reversed range: start seq 3 is at a later surface position than end seq 2.
expect(() => {
session.append('assistant/message', { turn: 1, step: 1, content: [] }, { surfaceOp: { op: 'replace', start: 3, end: 2 }, sourceEventSeqs: [2, 3] })
session.append('assistant/message', { provenance: { provider: 'mock', model: 'mock' }, turn: 1, step: 1, content: [] }, { surfaceOp: { op: 'replace', start: 3, end: 2 }, sourceEventSeqs: [2, 3] })
}).toThrow(/is after end seq 2/)
})
@@ -561,7 +561,7 @@ describe('surface contract under the invariants composition', () => {
session.append('user/message', { content: [{ type: 'text', text: 'b' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) // seq 3
// Replace shadows surface nodes [2, 3] but records provenance for only [2].
expect(() => {
session.append('assistant/message', { turn: 1, step: 1, content: [{ type: 'text', text: 'sum' }] }, { surfaceOp: { op: 'replace', start: 2, end: 3 }, sourceEventSeqs: [2] })
session.append('assistant/message', { provenance: { provider: 'mock', model: 'mock' }, turn: 1, step: 1, content: [{ type: 'text', text: 'sum' }] }, { surfaceOp: { op: 'replace', start: 2, end: 3 }, sourceEventSeqs: [2] })
}).toThrow(/must include every shadowed surface node; missing 3/)
})
@@ -573,7 +573,7 @@ describe('surface contract under the invariants composition', () => {
session.append('user/message', { content: [{ type: 'text', text: 'a' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) // seq 2
session.append('user/message', { content: [{ type: 'text', text: 'b' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) // seq 3
expect(() => {
session.append('assistant/message', { turn: 1, step: 1, content: [{ type: 'text', text: 'sum' }] }, { surfaceOp: { op: 'replace', start: 2, end: 3 }, sourceEventSeqs: [2, 3] })
session.append('assistant/message', { provenance: { provider: 'mock', model: 'mock' }, turn: 1, step: 1, content: [{ type: 'text', text: 'sum' }] }, { surfaceOp: { op: 'replace', start: 2, end: 3 }, sourceEventSeqs: [2, 3] })
}).not.toThrow()
})
@@ -585,7 +585,7 @@ describe('surface contract under the invariants composition', () => {
session.append('user/message', { content: [{ type: 'text', text: 'a' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) // seq 2
// seq 1 (step/start) is a real earlier event but never entered the surface.
expect(() => {
session.append('assistant/message', { turn: 1, step: 1, content: [] }, { surfaceOp: { op: 'replace', start: 1, end: 2 }, sourceEventSeqs: [1, 2] })
session.append('assistant/message', { provenance: { provider: 'mock', model: 'mock' }, turn: 1, step: 1, content: [] }, { surfaceOp: { op: 'replace', start: 1, end: 2 }, sourceEventSeqs: [1, 2] })
}).toThrow(/start seq 1 not found in surface/)
})
@@ -597,7 +597,7 @@ describe('surface contract under the invariants composition', () => {
session.append('user/message', { content: [{ type: 'text', text: 'a' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) // seq 2
// start (2) is on the surface but end (99) never entered it.
expect(() => {
session.append('assistant/message', { turn: 1, step: 1, content: [] }, { surfaceOp: { op: 'replace', start: 2, end: 99 }, sourceEventSeqs: [2] })
session.append('assistant/message', { provenance: { provider: 'mock', model: 'mock' }, turn: 1, step: 1, content: [] }, { surfaceOp: { op: 'replace', start: 2, end: 99 }, sourceEventSeqs: [2] })
}).toThrow(/end seq 99 not found in surface/)
})
@@ -610,11 +610,11 @@ describe('surface contract under the invariants composition', () => {
session.append('user/message', { content: [{ type: 'text', text: 'b' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) // seq 3
// Replace node 2 (position 0) with seq 4 — surface is now [4, 3], so seq 4
// precedes seq 3 in surface order even though 4 > 3 numerically.
session.append('assistant/message', { turn: 1, step: 1, content: [{ type: 'text', text: 's' }] }, { surfaceOp: { op: 'replace', start: 2, end: 2 }, sourceEventSeqs: [2] }) // seq 4
session.append('assistant/message', { provenance: { provider: 'mock', model: 'mock' }, turn: 1, step: 1, content: [{ type: 'text', text: 's' }] }, { surfaceOp: { op: 'replace', start: 2, end: 2 }, sourceEventSeqs: [2] }) // seq 4
// A replace with start=3, end=4 passes the seq check (3 <= 4) but is
// reversed positionally (3 is at pos 1, 4 is at pos 0).
expect(() => {
session.append('assistant/message', { turn: 1, step: 1, content: [] }, { surfaceOp: { op: 'replace', start: 3, end: 4 }, sourceEventSeqs: [3, 4] }) // seq 5
session.append('assistant/message', { provenance: { provider: 'mock', model: 'mock' }, turn: 1, step: 1, content: [] }, { surfaceOp: { op: 'replace', start: 3, end: 4 }, sourceEventSeqs: [3, 4] }) // seq 5
}).toThrow(/is after end seq 4/)
})
@@ -629,9 +629,9 @@ describe('surface contract under the invariants composition', () => {
// head seq (4) is numerically GREATER than the tail seq (3): the surface is
// not seq-ordered. A replace spanning start=4 (pos 0) … end=3 (pos 1) is
// valid positionally and must be accepted even though start seq > end seq.
session.append('assistant/message', { turn: 1, step: 1, content: [{ type: 'text', text: 's' }] }, { surfaceOp: { op: 'replace', start: 2, end: 2 }, sourceEventSeqs: [2] }) // seq 4
session.append('assistant/message', { provenance: { provider: 'mock', model: 'mock' }, turn: 1, step: 1, content: [{ type: 'text', text: 's' }] }, { surfaceOp: { op: 'replace', start: 2, end: 2 }, sourceEventSeqs: [2] }) // seq 4
expect(() => {
session.append('assistant/message', { turn: 1, step: 1, content: [] }, { surfaceOp: { op: 'replace', start: 4, end: 3 }, sourceEventSeqs: [4, 3] }) // seq 5
session.append('assistant/message', { provenance: { provider: 'mock', model: 'mock' }, turn: 1, step: 1, content: [] }, { surfaceOp: { op: 'replace', start: 4, end: 3 }, sourceEventSeqs: [4, 3] }) // seq 5
}).not.toThrow()
})
@@ -643,7 +643,7 @@ describe('surface contract under the invariants composition', () => {
session.append('user/message', { content: [{ type: 'text', text: 'a' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) // seq 2
// A replace with no sourceEventSeqs records no provenance for the node it shadows.
expect(() => {
session.append('assistant/message', { turn: 1, step: 1, content: [] }, { surfaceOp: { op: 'replace', start: 2, end: 2 } })
session.append('assistant/message', { provenance: { provider: 'mock', model: 'mock' }, turn: 1, step: 1, content: [] }, { surfaceOp: { op: 'replace', start: 2, end: 2 } })
}).toThrow(/must include every shadowed surface node; missing 2/)
})
@@ -654,7 +654,7 @@ describe('surface contract under the invariants composition', () => {
{ type: 'step/start' as const, seq: 1, time: 0, data: { turn: 1, step: 1 } },
{ type: 'user/message' as const, seq: 2, time: 0, data: { content: [{ type: 'text' as const, text: 'a' }], source: { kind: 'user' as const } }, surfaceOp: 'append' as const },
{ type: 'user/message' as const, seq: 3, time: 0, data: { content: [{ type: 'text' as const, text: 'b' }], source: { kind: 'user' as const } }, surfaceOp: 'append' as const },
{ type: 'assistant/message' as const, seq: 4, time: 0, data: { turn: 1, step: 1, content: [{ type: 'text' as const, text: 'sum' }] }, surfaceOp: { op: 'replace' as const, start: 2, end: 3 }, sourceEventSeqs: [2] },
{ type: 'assistant/message' as const, seq: 4, time: 0, data: { turn: 1, step: 1, content: [{ type: 'text' as const, text: 'sum' }], provenance: { provider: 'mock', model: 'mock' } }, surfaceOp: { op: 'replace' as const, start: 2, end: 3 }, sourceEventSeqs: [2] },
]
expect(() => ctx.sessions.create(undefined, { seed: badSeed })).toThrow(/must include every shadowed surface node; missing 3/)
})
@@ -670,7 +670,7 @@ describe('request-reconstruction cross-check (llm/stream)', () => {
session.append('user/message', { content: [{ type: 'text', text: 'hi' }], source: { kind: 'user' } }, { surfaceOp: 'append' })
const boundary = session.deriveMessages()
session.append('step/start', { turn: 1, step: 1 })
session.append('request/header', { header: { config: { model: 'm' } }, reason: 'initial' })
session.append('request/header', { header: { config: { provider: 'mock', model: 'm' } }, reason: 'initial' })
return { ctx, session, boundary }
}
@@ -700,7 +700,7 @@ describe('request-reconstruction cross-check (llm/stream)', () => {
it('expects the folded header\'s session prefix ahead of the derivation (prefix + derived)', async () => {
const { ctx, session, boundary } = await requestSetup()
const prefix = { role: 'user' as const, content: [{ type: 'text' as const, text: '<system-reminder>catalog</system-reminder>' }] }
session.append('request/header', { header: { config: { model: 'm' }, messagePrefix: [prefix] }, reason: 'change' })
session.append('request/header', { header: { config: { provider: 'mock', model: 'm' }, messagePrefix: [prefix] }, reason: 'change' })
// The prefixed request matches the fold…
const prefixed = Object.freeze({ model: 'm', messages: Object.freeze([prefix, ...boundary]), sessionId: session.id })
expect(() => { dispatch(ctx, prefixed) }).not.toThrow()
@@ -769,7 +769,7 @@ describe('request cross-check ordering (prepend)', () => {
session.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } })
session.append('user/message', { content: [{ type: 'text', text: 'hi' }], source: { kind: 'user' } }, { surfaceOp: 'append' })
session.append('step/start', { turn: 1, step: 1 })
session.append('request/header', { header: { config: { model: 'm' } }, reason: 'initial' })
session.append('request/header', { header: { config: { provider: 'mock', model: 'm' } }, reason: 'initial' })
const divergent = Object.freeze({
model: 'm',

View File

@@ -1,6 +1,6 @@
# @deepseek-ai/dsh-llm-replay
A replay LLM plugin for keyless snapshot tests. It installs a single `llm/stream` waterfall listener that short-circuits the waterfall (never calls `next()`) and yields model streams reconstructed from a recorded **session JSONL** fixture so a test can boot the real agent against a fixed model transcript with no API key.
A replay LLM plugin for keyless snapshot tests. It yields model streams reconstructed from a recorded **session JSONL** fixture, so a test can boot the real agent against a fixed model transcript with no API key. With `providers` configured it registers a replay-only adapter whose catalog is visible to clients such as ACP editors; without `providers` it installs the catch-all `llm/stream` waterfall used by tests that do not need discovery.
Its consumer is the ACP snapshot harness in `examples/acp-agent`, which loads this plugin (via `cordis.snapshot.yml`) in place of a real LLM adapter. The package exists so its derive/parse/replay logic falls under the per-file 100% coverage gate on `packages/*/src` (the same logic, while it lived under `examples/`, was outside the gate).
@@ -23,10 +23,18 @@ Replay keys every call by its calling session id (`GenerateOptions.sessionId`, s
| `file` | string | `$DSH_SNAPSHOT_FILE` | Path to the primary (parent) `session.jsonl` fixture. Required (config or env). |
| `overrideFile` | string | `$DSH_SNAPSHOT_OVERRIDE` | Optional path to a `ReplayEntry[]` sidecar that replaces the PRIMARY session's derived script. |
| `childFiles` | string[] | `$DSH_SNAPSHOT_CHILD_FILES` (path-delimited) | Recorded subagent child-session logs for a nested scenario; empty for a single-session scenario. |
| `providers` | `ReplayProviderConfig[]` | — | Optional replay-only provider and model catalog. Configured routes dispatch through the replay adapter and never perform provider I/O. |
```yaml
- id: llm-replay
name: '@deepseek-ai/dsh-llm-replay'
config:
providers:
- id: deepseek
name: DeepSeek
models:
- id: deepseek-v4-flash
- id: deepseek-v4-pro
# file/overrideFile/childFiles default to $DSH_SNAPSHOT_FILE /
# $DSH_SNAPSHOT_OVERRIDE / $DSH_SNAPSHOT_CHILD_FILES, set by the snapshot
# harness per scenario.
@@ -34,11 +42,11 @@ Replay keys every call by its calling session id (`GenerateOptions.sessionId`, s
## Exports
- `installLlmReplay(ctx, config)` — install the `llm/stream` listener; returns the disposer (HMR safety). Use this in tests to drive replay without the Loader or env vars.
- `installLlmReplay(ctx, config)` — install the configured replay adapter or catch-all `llm/stream` listener; returns the disposer (HMR safety). Use this in tests to drive replay without the Loader or env vars.
- `loadSessionScripts(config)` — resolve the ordered `SessionScript[]` (primary + children) for a scenario, ready to bind to live sessions in first-call order.
- `loadReplayScript(config)` — resolve the `ReplayEntry[]` for the PRIMARY session only (sidecar override if present, else derived from the JSONL; fail-loud if the fixture is missing).
- `deriveReplayScript(events)` / `parseSessionLog(text)` / `parseSessionHeader(text)` — the pure helpers that turn a recorded session log into a script and read its header `id`/`createdAt`. A derived group must end in a `finish` chunk; a group without one is the fingerprint of a thrown `stream()` and must instead be expressed via an override sidecar.
- Types `ReplayEntry` / `SessionScript` / `ReplayConfig` / `Config`.
- Types `ReplayEntry` / `SessionScript` / `ReplayConfig` / `ReplayProviderConfig` / `ReplayModelConfig` / `Config`.
## Plugin export shape

View File

@@ -10,8 +10,8 @@ import { existsSync, readFileSync } from 'node:fs'
import { delimiter as pathDelimiter } from 'node:path'
import type { Context } from 'cordis'
import type { SessionEvent } from '@deepseek-ai/dsh-session'
import type { GenerateOptions, StreamChunk } from '@deepseek-ai/dsh-llm'
import { LlmError, assertNever } from '@deepseek-ai/dsh-llm'
import type { GenerateOptions, LlmModelInfo, LlmProviderInfo, StreamChunk } from '@deepseek-ai/dsh-llm'
import { LlmAdapter, LlmError, assertNever } from '@deepseek-ai/dsh-llm'
/**
* One recorded model call. `throw` may replay prefix chunks before failing;
@@ -23,6 +23,26 @@ export type ReplayEntry =
| { kind: 'throw'; chunks: StreamChunk[]; message: string; code: string; status?: number }
| { kind: 'hang' }
/** One model exposed by a replay-only provider catalog. */
export interface ReplayModelConfig {
/** Model id used for replay requests. */
id: string
/** Selector label; defaults to {@link id}. */
name?: string
/** Optional selector description. */
description?: string
}
/** One provider route exposed by the replay adapter. */
export interface ReplayProviderConfig {
/** Provider route used for replay requests. */
id: string
/** Selector label; defaults to {@link id}. */
name?: string
/** Advisory models exposed to clients such as ACP editors. */
models?: ReplayModelConfig[]
}
/** Resolved plugin configuration. */
export interface ReplayConfig {
/**
@@ -45,6 +65,12 @@ export interface ReplayConfig {
* for a single-session scenario.
*/
childFiles?: string[]
/**
* Optional provider catalog. When non-empty, replay registers an adapter for
* these routes; when absent or empty, it retains the catch-all waterfall used
* by tests that do not need discovery.
*/
providers?: ReplayProviderConfig[]
}
/**
@@ -203,6 +229,42 @@ export function loadSessionScripts(config: ReplayConfig): SessionScript[] {
return [primary, ...children]
}
/** Replay adapter that makes a configured provider catalog discoverable without provider I/O. */
class ReplayAdapter extends LlmAdapter {
private readonly providers: ReadonlyMap<string, ReplayProviderConfig>
constructor(
providers: readonly ReplayProviderConfig[],
private readonly replay: (options: GenerateOptions) => AsyncIterable<StreamChunk>,
) {
super()
this.providers = new Map(providers.map(provider => [provider.id, provider]))
}
override providerInfo(provider: string): LlmProviderInfo {
const configured = this.providers.get(provider)
/* v8 ignore next -- LlmService only asks about routes registered from this same map. */
if (configured === undefined) return super.providerInfo(provider)
return { id: provider, name: configured.name ?? provider }
}
override listModels(provider: string): Promise<readonly LlmModelInfo[]> {
const configured = this.providers.get(provider)
/* v8 ignore next -- LlmService only asks about routes registered from this same map. */
if (configured === undefined) return Promise.resolve([])
return Promise.resolve((configured.models ?? []).map(model => ({
provider,
id: model.id,
name: model.name ?? model.id,
...model.description === undefined ? {} : { description: model.description },
})))
}
override stream(options: GenerateOptions): AsyncIterable<StreamChunk> {
return this.replay(options)
}
}
/** Yield a recorded stream back, honoring abort like a real adapter. */
async function* replayEntry(entry: ReplayEntry, signal: AbortSignal | undefined): AsyncIterable<StreamChunk> {
switch (entry.kind) {
@@ -243,12 +305,14 @@ async function* replayEntry(entry: ReplayEntry, signal: AbortSignal | undefined)
/**
* Install per-session positional replay. A newly seen live session takes the
* next ordered recorded script, then advances its own cursor synchronously at
* invocation time; calls without `sessionId` share one anonymous session.
* Returns the effect disposer for HMR-safe removal.
* invocation time; calls without `sessionId` share one anonymous session. A
* non-empty provider catalog registers a routed replay adapter; otherwise a
* catch-all waterfall intercepts requests. Returns the effect disposer for
* HMR-safe removal.
*
* @param ctx - the context whose `llm/stream` waterfall the listener short-circuits.
* @param ctx - the context whose LLM service receives the replay route or waterfall.
* @param config - the resolved fixture paths (env-var defaulting is `apply`'s job).
* @returns the `ctx.on` disposer that removes the listener.
* @returns the disposer that removes the registered adapter or listener.
*/
export function installLlmReplay(ctx: Context, config: ReplayConfig): () => void {
const scripts = loadSessionScripts(config)
@@ -258,7 +322,7 @@ export function installLlmReplay(ctx: Context, config: ReplayConfig): () => void
const bound = new Map<string, { entries: ReplayEntry[]; cursor: number }>()
let nextScript = 0
const ANON = '\0anon\0' // the key for a call that carries no sessionId
return ctx.on('llm/stream', (options: GenerateOptions, _next) => {
const replay = (options: GenerateOptions): AsyncIterable<StreamChunk> => {
const key = options.sessionId ?? ANON
let state = bound.get(key)
let unrecorded = false
@@ -296,7 +360,12 @@ export function installLlmReplay(ctx: Context, config: ReplayConfig): () => void
}
yield* replayEntry(entry, options.signal)
})()
})
}
const providers = config.providers ?? []
if (providers.length > 0) {
return ctx.llm.registerAdapter(providers.map(provider => provider.id), new ReplayAdapter(providers, replay))
}
return ctx.on('llm/stream', (options: GenerateOptions, _next) => replay(options))
}
export const name = 'llm-replay'
@@ -314,6 +383,8 @@ export interface Config {
* a nested-agent scenario; absent/empty for a single-session scenario.
*/
childFiles?: string[]
/** Optional replay-only provider catalog; absent or empty selects catch-all waterfall replay. */
providers?: ReplayProviderConfig[]
}
export function apply(ctx: Context, config: Config = {}): void {
@@ -329,5 +400,6 @@ export function apply(ctx: Context, config: Config = {}): void {
file,
...overrideFile !== undefined && overrideFile.length > 0 ? { overrideFile } : {},
...childFiles.length > 0 ? { childFiles } : {},
...config.providers !== undefined ? { providers: config.providers } : {},
})
}

View File

@@ -198,7 +198,7 @@ describe('loadReplayScript', () => {
})
})
describe('installLlmReplay (through the real waterfall)', () => {
describe('installLlmReplay (through the real LlmService)', () => {
function writeLog(...calls: StreamChunk[][]): void {
let seq = 1
const events: SessionEvent[] = []
@@ -214,7 +214,41 @@ describe('installLlmReplay (through the real waterfall)', () => {
await ctx.plugin(LlmService)
// No adapter registered for 'm' — replay must not reach it.
installLlmReplay(ctx, { file })
expect(await drain(ctx.llm.stream({ model: 'm', messages: [] }))).toEqual(TEXT_CHUNKS)
expect(await drain(ctx.llm.stream({ provider: 'm', model: 'm', messages: [] }))).toEqual(TEXT_CHUNKS)
})
it('registers a replay-only provider catalog when configured', async () => {
writeLog(TEXT_CHUNKS)
const ctx = new Context()
await ctx.plugin(LlmService)
const dispose = installLlmReplay(ctx, {
file,
providers: [
{
id: 'deepseek',
name: 'DeepSeek',
models: [
{ id: 'flash' },
{ id: 'pro', name: 'Pro', description: 'Larger model' },
],
},
{ id: 'empty' },
],
})
expect(ctx.llm.listProviders()).toEqual([
{ id: 'deepseek', name: 'DeepSeek' },
{ id: 'empty', name: 'empty' },
])
await expect(ctx.llm.listModels('deepseek')).resolves.toEqual([
{ provider: 'deepseek', id: 'flash', name: 'flash' },
{ provider: 'deepseek', id: 'pro', name: 'Pro', description: 'Larger model' },
])
await expect(ctx.llm.listModels('empty')).resolves.toEqual([])
expect(await drain(ctx.llm.stream({ provider: 'deepseek', model: 'pro', messages: [] }))).toEqual(TEXT_CHUNKS)
dispose()
expect(ctx.llm.listProviders()).toEqual([])
})
it('serves the Nth call the Nth derived entry (positional)', async () => {
@@ -227,8 +261,8 @@ describe('installLlmReplay (through the real waterfall)', () => {
const ctx = new Context()
await ctx.plugin(LlmService)
installLlmReplay(ctx, { file })
expect(await drain(ctx.llm.stream({ model: 'm', messages: [] }))).toEqual(TEXT_CHUNKS)
expect(await drain(ctx.llm.stream({ model: 'm', messages: [] }))).toEqual(second)
expect(await drain(ctx.llm.stream({ provider: 'm', model: 'm', messages: [] }))).toEqual(TEXT_CHUNKS)
expect(await drain(ctx.llm.stream({ provider: 'm', model: 'm', messages: [] }))).toEqual(second)
})
it('replays a sidecar throw-entry as an LlmError with code/status, after its prefix chunks', async () => {
@@ -244,7 +278,7 @@ describe('installLlmReplay (through the real waterfall)', () => {
const seen: StreamChunk[] = []
await expect((async () => {
for await (const c of ctx.llm.stream({ model: 'm', messages: [] })) seen.push(c)
for await (const c of ctx.llm.stream({ provider: 'm', model: 'm', messages: [] })) seen.push(c)
})()).rejects.toMatchObject({ message: 'unauthorized', code: 'AUTH', status: 401 })
expect(seen).toEqual(partial)
})
@@ -258,7 +292,7 @@ describe('installLlmReplay (through the real waterfall)', () => {
installLlmReplay(ctx, { file, overrideFile })
const controller = new AbortController()
const iterator = ctx.llm.stream({ model: 'm', messages: [], signal: controller.signal })[Symbol.asyncIterator]()
const iterator = ctx.llm.stream({ provider: 'm', model: 'm', messages: [], signal: controller.signal })[Symbol.asyncIterator]()
// Deterministically consume the two pre-hang chunks (no sleep), then abort
// and assert the next pull rejects — event-driven, per the no-sleeps rule.
expect((await iterator.next()).value).toMatchObject({ type: 'block-start' })
@@ -272,8 +306,8 @@ describe('installLlmReplay (through the real waterfall)', () => {
const ctx = new Context()
await ctx.plugin(LlmService)
installLlmReplay(ctx, { file })
await drain(ctx.llm.stream({ model: 'm', messages: [] }))
await expect(drain(ctx.llm.stream({ model: 'm', messages: [] }))).rejects.toThrow(/exhausted/)
await drain(ctx.llm.stream({ provider: 'm', model: 'm', messages: [] }))
await expect(drain(ctx.llm.stream({ provider: 'm', model: 'm', messages: [] }))).rejects.toThrow(/exhausted/)
})
it('aborts mid-replay when the signal is already set', async () => {
@@ -283,7 +317,7 @@ describe('installLlmReplay (through the real waterfall)', () => {
installLlmReplay(ctx, { file })
const controller = new AbortController()
controller.abort()
await expect(drain(ctx.llm.stream({ model: 'm', messages: [], signal: controller.signal })))
await expect(drain(ctx.llm.stream({ provider: 'm', model: 'm', messages: [], signal: controller.signal })))
.rejects.toThrow('aborted')
})
@@ -305,11 +339,11 @@ describe('installLlmReplay (through the real waterfall)', () => {
}, { inject: ['llm'] }))
// While installed, replay short-circuits to the derived fixture ('hi').
expect(await drain(ctx.llm.stream({ model: 'm', messages: [] }))).toEqual(TEXT_CHUNKS)
expect(await drain(ctx.llm.stream({ provider: 'm', model: 'm', messages: [] }))).toEqual(TEXT_CHUNKS)
await fiber.dispose()
// After dispose the listener is gone; the call reaches the real adapter.
expect(await drain(ctx.llm.stream({ model: 'm', messages: [] })))
expect(await drain(ctx.llm.stream({ provider: 'm', model: 'm', messages: [] })))
.toEqual([{ type: 'finish', reason: { kind: 'stop' } }])
})
@@ -321,7 +355,7 @@ describe('installLlmReplay (through the real waterfall)', () => {
const ctx = new Context()
await ctx.plugin(LlmService)
installLlmReplay(ctx, { file, overrideFile })
await expect(drain(ctx.llm.stream({ model: 'm', messages: [] })))
await expect(drain(ctx.llm.stream({ provider: 'm', model: 'm', messages: [] })))
.rejects.toThrow(/llm-replay replay entry/)
})
@@ -333,7 +367,7 @@ describe('installLlmReplay (through the real waterfall)', () => {
await ctx.plugin(LlmService)
installLlmReplay(ctx, { file, overrideFile })
const controller = new AbortController()
const iterator = ctx.llm.stream({ model: 'm', messages: [], signal: controller.signal })[Symbol.asyncIterator]()
const iterator = ctx.llm.stream({ provider: 'm', model: 'm', messages: [], signal: controller.signal })[Symbol.asyncIterator]()
// Consume the two pre-hang chunks, then start the third pull so the generator
// is parked inside the await (signal NOT yet aborted — exercises the
// addEventListener('abort') registration), and only THEN abort.
@@ -359,7 +393,7 @@ describe('installLlmReplay (through the real waterfall)', () => {
controller.abort()
// Already aborted: the throw-entry's prefix loop surfaces 'aborted' before
// it can reach the recorded LlmError.
await expect(drain(ctx.llm.stream({ model: 'm', messages: [], signal: controller.signal })))
await expect(drain(ctx.llm.stream({ provider: 'm', model: 'm', messages: [], signal: controller.signal })))
.rejects.toThrow('aborted')
})
@@ -373,7 +407,7 @@ describe('installLlmReplay (through the real waterfall)', () => {
const controller = new AbortController()
controller.abort()
// The two pre-hang chunks still flow; the abort surfaces at the await.
const iterator = ctx.llm.stream({ model: 'm', messages: [], signal: controller.signal })[Symbol.asyncIterator]()
const iterator = ctx.llm.stream({ provider: 'm', model: 'm', messages: [], signal: controller.signal })[Symbol.asyncIterator]()
await iterator.next()
await iterator.next()
await expect(iterator.next()).rejects.toThrow('aborted')
@@ -503,7 +537,7 @@ describe('installLlmReplay (per-session keying)', () => {
]
const live = (id: string): GenerateOptions =>
({ model: 'm', messages: [], sessionId: id as NonNullable<GenerateOptions['sessionId']> })
({ provider: 'm', model: 'm', messages: [], sessionId: id as NonNullable<GenerateOptions['sessionId']> })
it('routes each live session to its own script by FIRST-CALL order', async () => {
const parentFile = writeSession('session.jsonl', { id: 'rec-parent', createdAt: 100 }, [TEXT_CHUNKS])
@@ -540,7 +574,7 @@ describe('installLlmReplay (per-session keying)', () => {
await ctx.plugin(LlmService)
installLlmReplay(ctx, { file: parentFile })
// No sessionId at all — the legacy single-session path.
expect(await drain(ctx.llm.stream({ model: 'm', messages: [] }))).toEqual(TEXT_CHUNKS)
expect(await drain(ctx.llm.stream({ provider: 'm', model: 'm', messages: [] }))).toEqual(TEXT_CHUNKS)
})
it('fails loud when more distinct live sessions call than were recorded', async () => {
@@ -574,12 +608,13 @@ describe('apply (the plugin entry)', () => {
expect(inject).toEqual(['llm'])
})
it('installs replay from an explicit config.file', async () => {
it('installs replay and its catalog from explicit config', async () => {
writeFileSync(file, sessionJsonl(TEXT_CHUNKS.map((c, i) => chunkEvent(i + 1, 1, 1, c))), 'utf8')
const ctx = new Context()
await ctx.plugin(LlmService)
apply(ctx, { file })
expect(await drain(ctx.llm.stream({ model: 'm', messages: [] }))).toEqual(TEXT_CHUNKS)
apply(ctx, { file, providers: [{ id: 'm', models: [{ id: 'm' }] }] })
expect(ctx.llm.listProviders()).toEqual([{ id: 'm', name: 'm' }])
expect(await drain(ctx.llm.stream({ provider: 'm', model: 'm', messages: [] }))).toEqual(TEXT_CHUNKS)
})
it('falls back to $DSH_SNAPSHOT_FILE / $DSH_SNAPSHOT_OVERRIDE when config is empty', async () => {
@@ -591,7 +626,7 @@ describe('apply (the plugin entry)', () => {
const ctx = new Context()
await ctx.plugin(LlmService)
apply(ctx)
expect(await drain(ctx.llm.stream({ model: 'm', messages: [] }))).toEqual(TEXT_CHUNKS)
expect(await drain(ctx.llm.stream({ provider: 'm', model: 'm', messages: [] }))).toEqual(TEXT_CHUNKS)
})
it('uses only the file when no override path is configured or in the env', async () => {
@@ -601,7 +636,7 @@ describe('apply (the plugin entry)', () => {
const ctx = new Context()
await ctx.plugin(LlmService)
apply(ctx)
expect(await drain(ctx.llm.stream({ model: 'm', messages: [] }))).toEqual(TEXT_CHUNKS)
expect(await drain(ctx.llm.stream({ provider: 'm', model: 'm', messages: [] }))).toEqual(TEXT_CHUNKS)
})
it('throws when no fixture path is given by config or env', async () => {
@@ -631,7 +666,7 @@ describe('apply (the plugin entry)', () => {
await ctx.plugin(LlmService)
apply(ctx, { file, childFiles: [childFile] })
const live = (id: string): GenerateOptions =>
({ model: 'm', messages: [], sessionId: id as NonNullable<GenerateOptions['sessionId']> })
({ provider: 'm', model: 'm', messages: [], sessionId: id as NonNullable<GenerateOptions['sessionId']> })
expect(await drain(ctx.llm.stream(live('A')))).toEqual(TEXT_CHUNKS)
expect(await drain(ctx.llm.stream(live('B')))).toEqual(childSecond)
})
@@ -651,7 +686,7 @@ describe('apply (the plugin entry)', () => {
await ctx.plugin(LlmService)
apply(ctx)
const live = (id: string): GenerateOptions =>
({ model: 'm', messages: [], sessionId: id as NonNullable<GenerateOptions['sessionId']> })
({ provider: 'm', model: 'm', messages: [], sessionId: id as NonNullable<GenerateOptions['sessionId']> })
expect(await drain(ctx.llm.stream(live('A')))).toEqual(TEXT_CHUNKS)
expect(await drain(ctx.llm.stream(live('B')))).toEqual(childChunks)
})
@@ -663,6 +698,6 @@ describe('apply (the plugin entry)', () => {
const ctx = new Context()
await ctx.plugin(LlmService)
apply(ctx)
expect(await drain(ctx.llm.stream({ model: 'm', messages: [] }))).toEqual(TEXT_CHUNKS)
expect(await drain(ctx.llm.stream({ provider: 'm', model: 'm', messages: [] }))).toEqual(TEXT_CHUNKS)
})
})