Merge master into codex/simp-session-log-representation
This commit is contained in:
@@ -48,7 +48,7 @@ describe('session-log invariants', () => {
|
||||
session.append('user/message', { content: [{ type: 'text', text: 'hi' }], source: { kind: 'user' } }, { surfaceOp: 'append' })
|
||||
session.append('step/start', { turn: 1, step: 1 })
|
||||
session.append('assistant/chunk', { turn: 1, step: 1, chunk: { type: 'text-delta', index: 0, text: 'h' } })
|
||||
session.append('assistant/message', { turn: 1, step: 1, content: [{ type: 'tool-call', id: CallId('c1'), name: 'echo', arguments: '{}' }] }, { surfaceOp: 'append' })
|
||||
session.append('assistant/message', { provenance: { provider: 'mock', model: 'mock' }, turn: 1, step: 1, content: [{ type: 'tool-call', id: CallId('c1'), name: 'echo', arguments: '{}' }] }, { surfaceOp: 'append' })
|
||||
session.append('tool/call', { turn: 1, step: 1, callId: CallId('c1'), name: 'echo', arguments: '{}' })
|
||||
session.append('tool/result', { turn: 1, step: 1, callId: CallId('c1'), content: [{ type: 'text', text: 'ok' }], isError: false }, { surfaceOp: 'append' })
|
||||
session.append('step/end', { turn: 1, step: 1 })
|
||||
@@ -195,7 +195,7 @@ describe('session-log invariants', () => {
|
||||
expect(() => {
|
||||
session.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } })
|
||||
session.append('step/start', { turn: 1, step: 1 })
|
||||
session.append('assistant/message', { turn: 1, step: 1, content: [
|
||||
session.append('assistant/message', { provenance: { provider: 'mock', model: 'mock' }, turn: 1, step: 1, content: [
|
||||
{ type: 'tool-call', id: CallId('crashed'), name: 'bash', arguments: '{}' },
|
||||
] }, { surfaceOp: 'append' })
|
||||
session.append('tool/result', {
|
||||
@@ -251,10 +251,10 @@ describe('session-log invariants', () => {
|
||||
expect(() => {
|
||||
session.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } })
|
||||
session.append('step/start', { turn: 1, step: 1 })
|
||||
session.append('assistant/message', { turn: 1, step: 1, content: [] }, { surfaceOp: 'append' })
|
||||
session.append('assistant/message', { provenance: { provider: 'mock', model: 'mock' }, turn: 1, step: 1, content: [] }, { surfaceOp: 'append' })
|
||||
session.append('step/end', { turn: 1, step: 1 })
|
||||
session.append('step/start', { turn: 1, step: 2 })
|
||||
session.append('assistant/message', { turn: 1, step: 2, content: [] }, { surfaceOp: 'append' })
|
||||
session.append('assistant/message', { provenance: { provider: 'mock', model: 'mock' }, turn: 1, step: 2, content: [] }, { surfaceOp: 'append' })
|
||||
session.append('step/end', { turn: 1, step: 2 })
|
||||
session.append('turn/end', { turn: 1, reason: { kind: 'completed' } })
|
||||
session.append('turn/start', { turn: 2, trigger: { kind: 'message', source: { kind: 'user' } } })
|
||||
@@ -316,7 +316,7 @@ describe('session-log invariants', () => {
|
||||
const session = ctx.sessions.create()
|
||||
session.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } })
|
||||
session.append('step/start', { turn: 1, step: 1 })
|
||||
expect(() => session.append('assistant/message', { turn: 1, step: 2, content: [] }, { surfaceOp: 'append' }))
|
||||
expect(() => session.append('assistant/message', { provenance: { provider: 'mock', model: 'mock' }, turn: 1, step: 2, content: [] }, { surfaceOp: 'append' }))
|
||||
.toThrow(/open is turn 1\/step 1/)
|
||||
})
|
||||
})
|
||||
@@ -473,7 +473,7 @@ describe('surface contract under the invariants composition', () => {
|
||||
session.append('step/start', { turn: 1, step: 1 })
|
||||
expect(() => {
|
||||
session.append('user/message', { content: [{ type: 'text', text: 'hi' }], source: { kind: 'user' } }, { surfaceOp: 'append' })
|
||||
session.append('assistant/message', { turn: 1, step: 1, content: [] }, { surfaceOp: 'append', sourceEventSeqs: [1] })
|
||||
session.append('assistant/message', { provenance: { provider: 'mock', model: 'mock' }, turn: 1, step: 1, content: [] }, { surfaceOp: 'append', sourceEventSeqs: [1] })
|
||||
}).not.toThrow()
|
||||
})
|
||||
|
||||
@@ -483,7 +483,7 @@ describe('surface contract under the invariants composition', () => {
|
||||
session.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } })
|
||||
session.append('step/start', { turn: 1, step: 1 })
|
||||
session.append('user/message', { content: [{ type: 'text', text: 'a' }], source: { kind: 'user' } }, { surfaceOp: 'append' })
|
||||
session.append('assistant/message', { turn: 1, step: 1, content: [] }, { surfaceOp: { op: 'replace', start: 2, end: 2 }, sourceEventSeqs: [2] })
|
||||
session.append('assistant/message', { provenance: { provider: 'mock', model: 'mock' }, turn: 1, step: 1, content: [] }, { surfaceOp: { op: 'replace', start: 2, end: 2 }, sourceEventSeqs: [2] })
|
||||
// no throw — well-formed replace op
|
||||
})
|
||||
|
||||
@@ -492,7 +492,7 @@ describe('surface contract under the invariants composition', () => {
|
||||
const session = ctx.sessions.create()
|
||||
session.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } })
|
||||
expect(() => {
|
||||
session.append('assistant/message', { turn: 1, step: 1, content: [] }, { surfaceOp: 'append', sourceEventSeqs: [] })
|
||||
session.append('assistant/message', { provenance: { provider: 'mock', model: 'mock' }, turn: 1, step: 1, content: [] }, { surfaceOp: 'append', sourceEventSeqs: [] })
|
||||
}).toThrow(/must not be empty/)
|
||||
})
|
||||
|
||||
@@ -502,7 +502,7 @@ describe('surface contract under the invariants composition', () => {
|
||||
session.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } })
|
||||
session.append('user/message', { content: [{ type: 'text', text: 'a' }], source: { kind: 'user' } }, { surfaceOp: 'append' })
|
||||
expect(() => {
|
||||
session.append('assistant/message', { turn: 1, step: 1, content: [] }, { surfaceOp: 'append', sourceEventSeqs: [1, 1] })
|
||||
session.append('assistant/message', { provenance: { provider: 'mock', model: 'mock' }, turn: 1, step: 1, content: [] }, { surfaceOp: 'append', sourceEventSeqs: [1, 1] })
|
||||
}).toThrow(/must not contain duplicates/)
|
||||
})
|
||||
|
||||
@@ -513,7 +513,7 @@ describe('surface contract under the invariants composition', () => {
|
||||
// The next event is seq 1. Referencing its own seq fails on "must reference
|
||||
// earlier events" (the check order is: earlier first, then unknown).
|
||||
expect(() => {
|
||||
session.append('assistant/message', { turn: 1, step: 1, content: [] }, { surfaceOp: 'append', sourceEventSeqs: [1] })
|
||||
session.append('assistant/message', { provenance: { provider: 'mock', model: 'mock' }, turn: 1, step: 1, content: [] }, { surfaceOp: 'append', sourceEventSeqs: [1] })
|
||||
}).toThrow(/must reference earlier/)
|
||||
})
|
||||
|
||||
@@ -526,7 +526,7 @@ describe('surface contract under the invariants composition', () => {
|
||||
session.append('step/start', { turn: 1, step: 1 })
|
||||
// seqs so far: 0, 1. The next event at seq 2 references seq 1 → valid.
|
||||
expect(() => {
|
||||
session.append('assistant/message', { turn: 1, step: 1, content: [] }, { surfaceOp: 'append', sourceEventSeqs: [1] })
|
||||
session.append('assistant/message', { provenance: { provider: 'mock', model: 'mock' }, turn: 1, step: 1, content: [] }, { surfaceOp: 'append', sourceEventSeqs: [1] })
|
||||
}).not.toThrow()
|
||||
})
|
||||
|
||||
@@ -535,7 +535,7 @@ describe('surface contract under the invariants composition', () => {
|
||||
const session = ctx.sessions.create()
|
||||
session.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } })
|
||||
expect(() => {
|
||||
session.append('assistant/message', { turn: 1, step: 1, content: [] }, { surfaceOp: 'append', sourceEventSeqs: [99] })
|
||||
session.append('assistant/message', { provenance: { provider: 'mock', model: 'mock' }, turn: 1, step: 1, content: [] }, { surfaceOp: 'append', sourceEventSeqs: [99] })
|
||||
}).toThrow(/must reference earlier/)
|
||||
})
|
||||
|
||||
@@ -548,7 +548,7 @@ describe('surface contract under the invariants composition', () => {
|
||||
session.append('user/message', { content: [{ type: 'text', text: 'b' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) // seq 3
|
||||
// Reversed range: start seq 3 is at a later surface position than end seq 2.
|
||||
expect(() => {
|
||||
session.append('assistant/message', { turn: 1, step: 1, content: [] }, { surfaceOp: { op: 'replace', start: 3, end: 2 }, sourceEventSeqs: [2, 3] })
|
||||
session.append('assistant/message', { provenance: { provider: 'mock', model: 'mock' }, turn: 1, step: 1, content: [] }, { surfaceOp: { op: 'replace', start: 3, end: 2 }, sourceEventSeqs: [2, 3] })
|
||||
}).toThrow(/is after end seq 2/)
|
||||
})
|
||||
|
||||
@@ -561,7 +561,7 @@ describe('surface contract under the invariants composition', () => {
|
||||
session.append('user/message', { content: [{ type: 'text', text: 'b' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) // seq 3
|
||||
// Replace shadows surface nodes [2, 3] but records provenance for only [2].
|
||||
expect(() => {
|
||||
session.append('assistant/message', { turn: 1, step: 1, content: [{ type: 'text', text: 'sum' }] }, { surfaceOp: { op: 'replace', start: 2, end: 3 }, sourceEventSeqs: [2] })
|
||||
session.append('assistant/message', { provenance: { provider: 'mock', model: 'mock' }, turn: 1, step: 1, content: [{ type: 'text', text: 'sum' }] }, { surfaceOp: { op: 'replace', start: 2, end: 3 }, sourceEventSeqs: [2] })
|
||||
}).toThrow(/must include every shadowed surface node; missing 3/)
|
||||
})
|
||||
|
||||
@@ -573,7 +573,7 @@ describe('surface contract under the invariants composition', () => {
|
||||
session.append('user/message', { content: [{ type: 'text', text: 'a' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) // seq 2
|
||||
session.append('user/message', { content: [{ type: 'text', text: 'b' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) // seq 3
|
||||
expect(() => {
|
||||
session.append('assistant/message', { turn: 1, step: 1, content: [{ type: 'text', text: 'sum' }] }, { surfaceOp: { op: 'replace', start: 2, end: 3 }, sourceEventSeqs: [2, 3] })
|
||||
session.append('assistant/message', { provenance: { provider: 'mock', model: 'mock' }, turn: 1, step: 1, content: [{ type: 'text', text: 'sum' }] }, { surfaceOp: { op: 'replace', start: 2, end: 3 }, sourceEventSeqs: [2, 3] })
|
||||
}).not.toThrow()
|
||||
})
|
||||
|
||||
@@ -585,7 +585,7 @@ describe('surface contract under the invariants composition', () => {
|
||||
session.append('user/message', { content: [{ type: 'text', text: 'a' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) // seq 2
|
||||
// seq 1 (step/start) is a real earlier event but never entered the surface.
|
||||
expect(() => {
|
||||
session.append('assistant/message', { turn: 1, step: 1, content: [] }, { surfaceOp: { op: 'replace', start: 1, end: 2 }, sourceEventSeqs: [1, 2] })
|
||||
session.append('assistant/message', { provenance: { provider: 'mock', model: 'mock' }, turn: 1, step: 1, content: [] }, { surfaceOp: { op: 'replace', start: 1, end: 2 }, sourceEventSeqs: [1, 2] })
|
||||
}).toThrow(/start seq 1 not found in surface/)
|
||||
})
|
||||
|
||||
@@ -597,7 +597,7 @@ describe('surface contract under the invariants composition', () => {
|
||||
session.append('user/message', { content: [{ type: 'text', text: 'a' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) // seq 2
|
||||
// start (2) is on the surface but end (99) never entered it.
|
||||
expect(() => {
|
||||
session.append('assistant/message', { turn: 1, step: 1, content: [] }, { surfaceOp: { op: 'replace', start: 2, end: 99 }, sourceEventSeqs: [2] })
|
||||
session.append('assistant/message', { provenance: { provider: 'mock', model: 'mock' }, turn: 1, step: 1, content: [] }, { surfaceOp: { op: 'replace', start: 2, end: 99 }, sourceEventSeqs: [2] })
|
||||
}).toThrow(/end seq 99 not found in surface/)
|
||||
})
|
||||
|
||||
@@ -610,11 +610,11 @@ describe('surface contract under the invariants composition', () => {
|
||||
session.append('user/message', { content: [{ type: 'text', text: 'b' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) // seq 3
|
||||
// Replace node 2 (position 0) with seq 4 — surface is now [4, 3], so seq 4
|
||||
// precedes seq 3 in surface order even though 4 > 3 numerically.
|
||||
session.append('assistant/message', { turn: 1, step: 1, content: [{ type: 'text', text: 's' }] }, { surfaceOp: { op: 'replace', start: 2, end: 2 }, sourceEventSeqs: [2] }) // seq 4
|
||||
session.append('assistant/message', { provenance: { provider: 'mock', model: 'mock' }, turn: 1, step: 1, content: [{ type: 'text', text: 's' }] }, { surfaceOp: { op: 'replace', start: 2, end: 2 }, sourceEventSeqs: [2] }) // seq 4
|
||||
// A replace with start=3, end=4 passes the seq check (3 <= 4) but is
|
||||
// reversed positionally (3 is at pos 1, 4 is at pos 0).
|
||||
expect(() => {
|
||||
session.append('assistant/message', { turn: 1, step: 1, content: [] }, { surfaceOp: { op: 'replace', start: 3, end: 4 }, sourceEventSeqs: [3, 4] }) // seq 5
|
||||
session.append('assistant/message', { provenance: { provider: 'mock', model: 'mock' }, turn: 1, step: 1, content: [] }, { surfaceOp: { op: 'replace', start: 3, end: 4 }, sourceEventSeqs: [3, 4] }) // seq 5
|
||||
}).toThrow(/is after end seq 4/)
|
||||
})
|
||||
|
||||
@@ -629,9 +629,9 @@ describe('surface contract under the invariants composition', () => {
|
||||
// head seq (4) is numerically GREATER than the tail seq (3): the surface is
|
||||
// not seq-ordered. A replace spanning start=4 (pos 0) … end=3 (pos 1) is
|
||||
// valid positionally and must be accepted even though start seq > end seq.
|
||||
session.append('assistant/message', { turn: 1, step: 1, content: [{ type: 'text', text: 's' }] }, { surfaceOp: { op: 'replace', start: 2, end: 2 }, sourceEventSeqs: [2] }) // seq 4
|
||||
session.append('assistant/message', { provenance: { provider: 'mock', model: 'mock' }, turn: 1, step: 1, content: [{ type: 'text', text: 's' }] }, { surfaceOp: { op: 'replace', start: 2, end: 2 }, sourceEventSeqs: [2] }) // seq 4
|
||||
expect(() => {
|
||||
session.append('assistant/message', { turn: 1, step: 1, content: [] }, { surfaceOp: { op: 'replace', start: 4, end: 3 }, sourceEventSeqs: [4, 3] }) // seq 5
|
||||
session.append('assistant/message', { provenance: { provider: 'mock', model: 'mock' }, turn: 1, step: 1, content: [] }, { surfaceOp: { op: 'replace', start: 4, end: 3 }, sourceEventSeqs: [4, 3] }) // seq 5
|
||||
}).not.toThrow()
|
||||
})
|
||||
|
||||
@@ -643,7 +643,7 @@ describe('surface contract under the invariants composition', () => {
|
||||
session.append('user/message', { content: [{ type: 'text', text: 'a' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) // seq 2
|
||||
// A replace with no sourceEventSeqs records no provenance for the node it shadows.
|
||||
expect(() => {
|
||||
session.append('assistant/message', { turn: 1, step: 1, content: [] }, { surfaceOp: { op: 'replace', start: 2, end: 2 } })
|
||||
session.append('assistant/message', { provenance: { provider: 'mock', model: 'mock' }, turn: 1, step: 1, content: [] }, { surfaceOp: { op: 'replace', start: 2, end: 2 } })
|
||||
}).toThrow(/must include every shadowed surface node; missing 2/)
|
||||
})
|
||||
|
||||
@@ -654,7 +654,7 @@ describe('surface contract under the invariants composition', () => {
|
||||
{ type: 'step/start' as const, seq: 1, time: 0, data: { turn: 1, step: 1 } },
|
||||
{ type: 'user/message' as const, seq: 2, time: 0, data: { content: [{ type: 'text' as const, text: 'a' }], source: { kind: 'user' as const } }, surfaceOp: 'append' as const },
|
||||
{ type: 'user/message' as const, seq: 3, time: 0, data: { content: [{ type: 'text' as const, text: 'b' }], source: { kind: 'user' as const } }, surfaceOp: 'append' as const },
|
||||
{ type: 'assistant/message' as const, seq: 4, time: 0, data: { turn: 1, step: 1, content: [{ type: 'text' as const, text: 'sum' }] }, surfaceOp: { op: 'replace' as const, start: 2, end: 3 }, sourceEventSeqs: [2] },
|
||||
{ type: 'assistant/message' as const, seq: 4, time: 0, data: { turn: 1, step: 1, content: [{ type: 'text' as const, text: 'sum' }], provenance: { provider: 'mock', model: 'mock' } }, surfaceOp: { op: 'replace' as const, start: 2, end: 3 }, sourceEventSeqs: [2] },
|
||||
]
|
||||
expect(() => ctx.sessions.create(undefined, { seed: badSeed })).toThrow(/must include every shadowed surface node; missing 3/)
|
||||
})
|
||||
@@ -670,7 +670,7 @@ describe('request-reconstruction cross-check (llm/stream)', () => {
|
||||
session.append('user/message', { content: [{ type: 'text', text: 'hi' }], source: { kind: 'user' } }, { surfaceOp: 'append' })
|
||||
const boundary = session.deriveMessages()
|
||||
session.append('step/start', { turn: 1, step: 1 })
|
||||
session.append('request/header', { header: { config: { model: 'm' } }, reason: 'initial' })
|
||||
session.append('request/header', { header: { config: { provider: 'mock', model: 'm' } }, reason: 'initial' })
|
||||
return { ctx, session, boundary }
|
||||
}
|
||||
|
||||
@@ -700,7 +700,7 @@ describe('request-reconstruction cross-check (llm/stream)', () => {
|
||||
it('expects the folded header\'s session prefix ahead of the derivation (prefix + derived)', async () => {
|
||||
const { ctx, session, boundary } = await requestSetup()
|
||||
const prefix = { role: 'user' as const, content: [{ type: 'text' as const, text: '<system-reminder>catalog</system-reminder>' }] }
|
||||
session.append('request/header', { header: { config: { model: 'm' }, messagePrefix: [prefix] }, reason: 'change' })
|
||||
session.append('request/header', { header: { config: { provider: 'mock', model: 'm' }, messagePrefix: [prefix] }, reason: 'change' })
|
||||
// The prefixed request matches the fold…
|
||||
const prefixed = Object.freeze({ model: 'm', messages: Object.freeze([prefix, ...boundary]), sessionId: session.id })
|
||||
expect(() => { dispatch(ctx, prefixed) }).not.toThrow()
|
||||
@@ -769,7 +769,7 @@ describe('request cross-check ordering (prepend)', () => {
|
||||
session.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } })
|
||||
session.append('user/message', { content: [{ type: 'text', text: 'hi' }], source: { kind: 'user' } }, { surfaceOp: 'append' })
|
||||
session.append('step/start', { turn: 1, step: 1 })
|
||||
session.append('request/header', { header: { config: { model: 'm' } }, reason: 'initial' })
|
||||
session.append('request/header', { header: { config: { provider: 'mock', model: 'm' } }, reason: 'initial' })
|
||||
|
||||
const divergent = Object.freeze({
|
||||
model: 'm',
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# @deepseek-ai/dsh-llm-replay
|
||||
|
||||
A replay LLM plugin for keyless snapshot tests. It installs a single `llm/stream` waterfall listener that short-circuits the waterfall (never calls `next()`) and yields model streams reconstructed from a recorded **session JSONL** fixture — so a test can boot the real agent against a fixed model transcript with no API key.
|
||||
A replay LLM plugin for keyless snapshot tests. It yields model streams reconstructed from a recorded **session JSONL** fixture, so a test can boot the real agent against a fixed model transcript with no API key. With `providers` configured it registers a replay-only adapter whose catalog is visible to clients such as ACP editors; without `providers` it installs the catch-all `llm/stream` waterfall used by tests that do not need discovery.
|
||||
|
||||
Its consumer is the ACP snapshot harness in `examples/acp-agent`, which loads this plugin (via `cordis.snapshot.yml`) in place of a real LLM adapter. The package exists so its derive/parse/replay logic falls under the per-file 100% coverage gate on `packages/*/src` (the same logic, while it lived under `examples/`, was outside the gate).
|
||||
|
||||
@@ -23,10 +23,18 @@ Replay keys every call by its calling session id (`GenerateOptions.sessionId`, s
|
||||
| `file` | string | `$DSH_SNAPSHOT_FILE` | Path to the primary (parent) `session.jsonl` fixture. Required (config or env). |
|
||||
| `overrideFile` | string | `$DSH_SNAPSHOT_OVERRIDE` | Optional path to a `ReplayEntry[]` sidecar that replaces the PRIMARY session's derived script. |
|
||||
| `childFiles` | string[] | `$DSH_SNAPSHOT_CHILD_FILES` (path-delimited) | Recorded subagent child-session logs for a nested scenario; empty for a single-session scenario. |
|
||||
| `providers` | `ReplayProviderConfig[]` | — | Optional replay-only provider and model catalog. Configured routes dispatch through the replay adapter and never perform provider I/O. |
|
||||
|
||||
```yaml
|
||||
- id: llm-replay
|
||||
name: '@deepseek-ai/dsh-llm-replay'
|
||||
config:
|
||||
providers:
|
||||
- id: deepseek
|
||||
name: DeepSeek
|
||||
models:
|
||||
- id: deepseek-v4-flash
|
||||
- id: deepseek-v4-pro
|
||||
# file/overrideFile/childFiles default to $DSH_SNAPSHOT_FILE /
|
||||
# $DSH_SNAPSHOT_OVERRIDE / $DSH_SNAPSHOT_CHILD_FILES, set by the snapshot
|
||||
# harness per scenario.
|
||||
@@ -34,11 +42,11 @@ Replay keys every call by its calling session id (`GenerateOptions.sessionId`, s
|
||||
|
||||
## Exports
|
||||
|
||||
- `installLlmReplay(ctx, config)` — install the `llm/stream` listener; returns the disposer (HMR safety). Use this in tests to drive replay without the Loader or env vars.
|
||||
- `installLlmReplay(ctx, config)` — install the configured replay adapter or catch-all `llm/stream` listener; returns the disposer (HMR safety). Use this in tests to drive replay without the Loader or env vars.
|
||||
- `loadSessionScripts(config)` — resolve the ordered `SessionScript[]` (primary + children) for a scenario, ready to bind to live sessions in first-call order.
|
||||
- `loadReplayScript(config)` — resolve the `ReplayEntry[]` for the PRIMARY session only (sidecar override if present, else derived from the JSONL; fail-loud if the fixture is missing).
|
||||
- `deriveReplayScript(events)` / `parseSessionLog(text)` / `parseSessionHeader(text)` — the pure helpers that turn a recorded session log into a script and read its header `id`/`createdAt`. A derived group must end in a `finish` chunk; a group without one is the fingerprint of a thrown `stream()` and must instead be expressed via an override sidecar.
|
||||
- Types `ReplayEntry` / `SessionScript` / `ReplayConfig` / `Config`.
|
||||
- Types `ReplayEntry` / `SessionScript` / `ReplayConfig` / `ReplayProviderConfig` / `ReplayModelConfig` / `Config`.
|
||||
|
||||
## Plugin export shape
|
||||
|
||||
|
||||
@@ -10,8 +10,8 @@ import { existsSync, readFileSync } from 'node:fs'
|
||||
import { delimiter as pathDelimiter } from 'node:path'
|
||||
import type { Context } from 'cordis'
|
||||
import type { SessionEvent } from '@deepseek-ai/dsh-session'
|
||||
import type { GenerateOptions, StreamChunk } from '@deepseek-ai/dsh-llm'
|
||||
import { LlmError, assertNever } from '@deepseek-ai/dsh-llm'
|
||||
import type { GenerateOptions, LlmModelInfo, LlmProviderInfo, StreamChunk } from '@deepseek-ai/dsh-llm'
|
||||
import { LlmAdapter, LlmError, assertNever } from '@deepseek-ai/dsh-llm'
|
||||
|
||||
/**
|
||||
* One recorded model call. `throw` may replay prefix chunks before failing;
|
||||
@@ -23,6 +23,26 @@ export type ReplayEntry =
|
||||
| { kind: 'throw'; chunks: StreamChunk[]; message: string; code: string; status?: number }
|
||||
| { kind: 'hang' }
|
||||
|
||||
/** One model exposed by a replay-only provider catalog. */
|
||||
export interface ReplayModelConfig {
|
||||
/** Model id used for replay requests. */
|
||||
id: string
|
||||
/** Selector label; defaults to {@link id}. */
|
||||
name?: string
|
||||
/** Optional selector description. */
|
||||
description?: string
|
||||
}
|
||||
|
||||
/** One provider route exposed by the replay adapter. */
|
||||
export interface ReplayProviderConfig {
|
||||
/** Provider route used for replay requests. */
|
||||
id: string
|
||||
/** Selector label; defaults to {@link id}. */
|
||||
name?: string
|
||||
/** Advisory models exposed to clients such as ACP editors. */
|
||||
models?: ReplayModelConfig[]
|
||||
}
|
||||
|
||||
/** Resolved plugin configuration. */
|
||||
export interface ReplayConfig {
|
||||
/**
|
||||
@@ -45,6 +65,12 @@ export interface ReplayConfig {
|
||||
* for a single-session scenario.
|
||||
*/
|
||||
childFiles?: string[]
|
||||
/**
|
||||
* Optional provider catalog. When non-empty, replay registers an adapter for
|
||||
* these routes; when absent or empty, it retains the catch-all waterfall used
|
||||
* by tests that do not need discovery.
|
||||
*/
|
||||
providers?: ReplayProviderConfig[]
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -203,6 +229,42 @@ export function loadSessionScripts(config: ReplayConfig): SessionScript[] {
|
||||
return [primary, ...children]
|
||||
}
|
||||
|
||||
/** Replay adapter that makes a configured provider catalog discoverable without provider I/O. */
|
||||
class ReplayAdapter extends LlmAdapter {
|
||||
private readonly providers: ReadonlyMap<string, ReplayProviderConfig>
|
||||
|
||||
constructor(
|
||||
providers: readonly ReplayProviderConfig[],
|
||||
private readonly replay: (options: GenerateOptions) => AsyncIterable<StreamChunk>,
|
||||
) {
|
||||
super()
|
||||
this.providers = new Map(providers.map(provider => [provider.id, provider]))
|
||||
}
|
||||
|
||||
override providerInfo(provider: string): LlmProviderInfo {
|
||||
const configured = this.providers.get(provider)
|
||||
/* v8 ignore next -- LlmService only asks about routes registered from this same map. */
|
||||
if (configured === undefined) return super.providerInfo(provider)
|
||||
return { id: provider, name: configured.name ?? provider }
|
||||
}
|
||||
|
||||
override listModels(provider: string): Promise<readonly LlmModelInfo[]> {
|
||||
const configured = this.providers.get(provider)
|
||||
/* v8 ignore next -- LlmService only asks about routes registered from this same map. */
|
||||
if (configured === undefined) return Promise.resolve([])
|
||||
return Promise.resolve((configured.models ?? []).map(model => ({
|
||||
provider,
|
||||
id: model.id,
|
||||
name: model.name ?? model.id,
|
||||
...model.description === undefined ? {} : { description: model.description },
|
||||
})))
|
||||
}
|
||||
|
||||
override stream(options: GenerateOptions): AsyncIterable<StreamChunk> {
|
||||
return this.replay(options)
|
||||
}
|
||||
}
|
||||
|
||||
/** Yield a recorded stream back, honoring abort like a real adapter. */
|
||||
async function* replayEntry(entry: ReplayEntry, signal: AbortSignal | undefined): AsyncIterable<StreamChunk> {
|
||||
switch (entry.kind) {
|
||||
@@ -243,12 +305,14 @@ async function* replayEntry(entry: ReplayEntry, signal: AbortSignal | undefined)
|
||||
/**
|
||||
* Install per-session positional replay. A newly seen live session takes the
|
||||
* next ordered recorded script, then advances its own cursor synchronously at
|
||||
* invocation time; calls without `sessionId` share one anonymous session.
|
||||
* Returns the effect disposer for HMR-safe removal.
|
||||
* invocation time; calls without `sessionId` share one anonymous session. A
|
||||
* non-empty provider catalog registers a routed replay adapter; otherwise a
|
||||
* catch-all waterfall intercepts requests. Returns the effect disposer for
|
||||
* HMR-safe removal.
|
||||
*
|
||||
* @param ctx - the context whose `llm/stream` waterfall the listener short-circuits.
|
||||
* @param ctx - the context whose LLM service receives the replay route or waterfall.
|
||||
* @param config - the resolved fixture paths (env-var defaulting is `apply`'s job).
|
||||
* @returns the `ctx.on` disposer that removes the listener.
|
||||
* @returns the disposer that removes the registered adapter or listener.
|
||||
*/
|
||||
export function installLlmReplay(ctx: Context, config: ReplayConfig): () => void {
|
||||
const scripts = loadSessionScripts(config)
|
||||
@@ -258,7 +322,7 @@ export function installLlmReplay(ctx: Context, config: ReplayConfig): () => void
|
||||
const bound = new Map<string, { entries: ReplayEntry[]; cursor: number }>()
|
||||
let nextScript = 0
|
||||
const ANON = '\0anon\0' // the key for a call that carries no sessionId
|
||||
return ctx.on('llm/stream', (options: GenerateOptions, _next) => {
|
||||
const replay = (options: GenerateOptions): AsyncIterable<StreamChunk> => {
|
||||
const key = options.sessionId ?? ANON
|
||||
let state = bound.get(key)
|
||||
let unrecorded = false
|
||||
@@ -296,7 +360,12 @@ export function installLlmReplay(ctx: Context, config: ReplayConfig): () => void
|
||||
}
|
||||
yield* replayEntry(entry, options.signal)
|
||||
})()
|
||||
})
|
||||
}
|
||||
const providers = config.providers ?? []
|
||||
if (providers.length > 0) {
|
||||
return ctx.llm.registerAdapter(providers.map(provider => provider.id), new ReplayAdapter(providers, replay))
|
||||
}
|
||||
return ctx.on('llm/stream', (options: GenerateOptions, _next) => replay(options))
|
||||
}
|
||||
|
||||
export const name = 'llm-replay'
|
||||
@@ -314,6 +383,8 @@ export interface Config {
|
||||
* a nested-agent scenario; absent/empty for a single-session scenario.
|
||||
*/
|
||||
childFiles?: string[]
|
||||
/** Optional replay-only provider catalog; absent or empty selects catch-all waterfall replay. */
|
||||
providers?: ReplayProviderConfig[]
|
||||
}
|
||||
|
||||
export function apply(ctx: Context, config: Config = {}): void {
|
||||
@@ -329,5 +400,6 @@ export function apply(ctx: Context, config: Config = {}): void {
|
||||
file,
|
||||
...overrideFile !== undefined && overrideFile.length > 0 ? { overrideFile } : {},
|
||||
...childFiles.length > 0 ? { childFiles } : {},
|
||||
...config.providers !== undefined ? { providers: config.providers } : {},
|
||||
})
|
||||
}
|
||||
|
||||
@@ -198,7 +198,7 @@ describe('loadReplayScript', () => {
|
||||
})
|
||||
})
|
||||
|
||||
describe('installLlmReplay (through the real waterfall)', () => {
|
||||
describe('installLlmReplay (through the real LlmService)', () => {
|
||||
function writeLog(...calls: StreamChunk[][]): void {
|
||||
let seq = 1
|
||||
const events: SessionEvent[] = []
|
||||
@@ -214,7 +214,41 @@ describe('installLlmReplay (through the real waterfall)', () => {
|
||||
await ctx.plugin(LlmService)
|
||||
// No adapter registered for 'm' — replay must not reach it.
|
||||
installLlmReplay(ctx, { file })
|
||||
expect(await drain(ctx.llm.stream({ model: 'm', messages: [] }))).toEqual(TEXT_CHUNKS)
|
||||
expect(await drain(ctx.llm.stream({ provider: 'm', model: 'm', messages: [] }))).toEqual(TEXT_CHUNKS)
|
||||
})
|
||||
|
||||
it('registers a replay-only provider catalog when configured', async () => {
|
||||
writeLog(TEXT_CHUNKS)
|
||||
const ctx = new Context()
|
||||
await ctx.plugin(LlmService)
|
||||
const dispose = installLlmReplay(ctx, {
|
||||
file,
|
||||
providers: [
|
||||
{
|
||||
id: 'deepseek',
|
||||
name: 'DeepSeek',
|
||||
models: [
|
||||
{ id: 'flash' },
|
||||
{ id: 'pro', name: 'Pro', description: 'Larger model' },
|
||||
],
|
||||
},
|
||||
{ id: 'empty' },
|
||||
],
|
||||
})
|
||||
|
||||
expect(ctx.llm.listProviders()).toEqual([
|
||||
{ id: 'deepseek', name: 'DeepSeek' },
|
||||
{ id: 'empty', name: 'empty' },
|
||||
])
|
||||
await expect(ctx.llm.listModels('deepseek')).resolves.toEqual([
|
||||
{ provider: 'deepseek', id: 'flash', name: 'flash' },
|
||||
{ provider: 'deepseek', id: 'pro', name: 'Pro', description: 'Larger model' },
|
||||
])
|
||||
await expect(ctx.llm.listModels('empty')).resolves.toEqual([])
|
||||
expect(await drain(ctx.llm.stream({ provider: 'deepseek', model: 'pro', messages: [] }))).toEqual(TEXT_CHUNKS)
|
||||
|
||||
dispose()
|
||||
expect(ctx.llm.listProviders()).toEqual([])
|
||||
})
|
||||
|
||||
it('serves the Nth call the Nth derived entry (positional)', async () => {
|
||||
@@ -227,8 +261,8 @@ describe('installLlmReplay (through the real waterfall)', () => {
|
||||
const ctx = new Context()
|
||||
await ctx.plugin(LlmService)
|
||||
installLlmReplay(ctx, { file })
|
||||
expect(await drain(ctx.llm.stream({ model: 'm', messages: [] }))).toEqual(TEXT_CHUNKS)
|
||||
expect(await drain(ctx.llm.stream({ model: 'm', messages: [] }))).toEqual(second)
|
||||
expect(await drain(ctx.llm.stream({ provider: 'm', model: 'm', messages: [] }))).toEqual(TEXT_CHUNKS)
|
||||
expect(await drain(ctx.llm.stream({ provider: 'm', model: 'm', messages: [] }))).toEqual(second)
|
||||
})
|
||||
|
||||
it('replays a sidecar throw-entry as an LlmError with code/status, after its prefix chunks', async () => {
|
||||
@@ -244,7 +278,7 @@ describe('installLlmReplay (through the real waterfall)', () => {
|
||||
|
||||
const seen: StreamChunk[] = []
|
||||
await expect((async () => {
|
||||
for await (const c of ctx.llm.stream({ model: 'm', messages: [] })) seen.push(c)
|
||||
for await (const c of ctx.llm.stream({ provider: 'm', model: 'm', messages: [] })) seen.push(c)
|
||||
})()).rejects.toMatchObject({ message: 'unauthorized', code: 'AUTH', status: 401 })
|
||||
expect(seen).toEqual(partial)
|
||||
})
|
||||
@@ -258,7 +292,7 @@ describe('installLlmReplay (through the real waterfall)', () => {
|
||||
installLlmReplay(ctx, { file, overrideFile })
|
||||
|
||||
const controller = new AbortController()
|
||||
const iterator = ctx.llm.stream({ model: 'm', messages: [], signal: controller.signal })[Symbol.asyncIterator]()
|
||||
const iterator = ctx.llm.stream({ provider: 'm', model: 'm', messages: [], signal: controller.signal })[Symbol.asyncIterator]()
|
||||
// Deterministically consume the two pre-hang chunks (no sleep), then abort
|
||||
// and assert the next pull rejects — event-driven, per the no-sleeps rule.
|
||||
expect((await iterator.next()).value).toMatchObject({ type: 'block-start' })
|
||||
@@ -272,8 +306,8 @@ describe('installLlmReplay (through the real waterfall)', () => {
|
||||
const ctx = new Context()
|
||||
await ctx.plugin(LlmService)
|
||||
installLlmReplay(ctx, { file })
|
||||
await drain(ctx.llm.stream({ model: 'm', messages: [] }))
|
||||
await expect(drain(ctx.llm.stream({ model: 'm', messages: [] }))).rejects.toThrow(/exhausted/)
|
||||
await drain(ctx.llm.stream({ provider: 'm', model: 'm', messages: [] }))
|
||||
await expect(drain(ctx.llm.stream({ provider: 'm', model: 'm', messages: [] }))).rejects.toThrow(/exhausted/)
|
||||
})
|
||||
|
||||
it('aborts mid-replay when the signal is already set', async () => {
|
||||
@@ -283,7 +317,7 @@ describe('installLlmReplay (through the real waterfall)', () => {
|
||||
installLlmReplay(ctx, { file })
|
||||
const controller = new AbortController()
|
||||
controller.abort()
|
||||
await expect(drain(ctx.llm.stream({ model: 'm', messages: [], signal: controller.signal })))
|
||||
await expect(drain(ctx.llm.stream({ provider: 'm', model: 'm', messages: [], signal: controller.signal })))
|
||||
.rejects.toThrow('aborted')
|
||||
})
|
||||
|
||||
@@ -305,11 +339,11 @@ describe('installLlmReplay (through the real waterfall)', () => {
|
||||
}, { inject: ['llm'] }))
|
||||
|
||||
// While installed, replay short-circuits to the derived fixture ('hi').
|
||||
expect(await drain(ctx.llm.stream({ model: 'm', messages: [] }))).toEqual(TEXT_CHUNKS)
|
||||
expect(await drain(ctx.llm.stream({ provider: 'm', model: 'm', messages: [] }))).toEqual(TEXT_CHUNKS)
|
||||
|
||||
await fiber.dispose()
|
||||
// After dispose the listener is gone; the call reaches the real adapter.
|
||||
expect(await drain(ctx.llm.stream({ model: 'm', messages: [] })))
|
||||
expect(await drain(ctx.llm.stream({ provider: 'm', model: 'm', messages: [] })))
|
||||
.toEqual([{ type: 'finish', reason: { kind: 'stop' } }])
|
||||
})
|
||||
|
||||
@@ -321,7 +355,7 @@ describe('installLlmReplay (through the real waterfall)', () => {
|
||||
const ctx = new Context()
|
||||
await ctx.plugin(LlmService)
|
||||
installLlmReplay(ctx, { file, overrideFile })
|
||||
await expect(drain(ctx.llm.stream({ model: 'm', messages: [] })))
|
||||
await expect(drain(ctx.llm.stream({ provider: 'm', model: 'm', messages: [] })))
|
||||
.rejects.toThrow(/llm-replay replay entry/)
|
||||
})
|
||||
|
||||
@@ -333,7 +367,7 @@ describe('installLlmReplay (through the real waterfall)', () => {
|
||||
await ctx.plugin(LlmService)
|
||||
installLlmReplay(ctx, { file, overrideFile })
|
||||
const controller = new AbortController()
|
||||
const iterator = ctx.llm.stream({ model: 'm', messages: [], signal: controller.signal })[Symbol.asyncIterator]()
|
||||
const iterator = ctx.llm.stream({ provider: 'm', model: 'm', messages: [], signal: controller.signal })[Symbol.asyncIterator]()
|
||||
// Consume the two pre-hang chunks, then start the third pull so the generator
|
||||
// is parked inside the await (signal NOT yet aborted — exercises the
|
||||
// addEventListener('abort') registration), and only THEN abort.
|
||||
@@ -359,7 +393,7 @@ describe('installLlmReplay (through the real waterfall)', () => {
|
||||
controller.abort()
|
||||
// Already aborted: the throw-entry's prefix loop surfaces 'aborted' before
|
||||
// it can reach the recorded LlmError.
|
||||
await expect(drain(ctx.llm.stream({ model: 'm', messages: [], signal: controller.signal })))
|
||||
await expect(drain(ctx.llm.stream({ provider: 'm', model: 'm', messages: [], signal: controller.signal })))
|
||||
.rejects.toThrow('aborted')
|
||||
})
|
||||
|
||||
@@ -373,7 +407,7 @@ describe('installLlmReplay (through the real waterfall)', () => {
|
||||
const controller = new AbortController()
|
||||
controller.abort()
|
||||
// The two pre-hang chunks still flow; the abort surfaces at the await.
|
||||
const iterator = ctx.llm.stream({ model: 'm', messages: [], signal: controller.signal })[Symbol.asyncIterator]()
|
||||
const iterator = ctx.llm.stream({ provider: 'm', model: 'm', messages: [], signal: controller.signal })[Symbol.asyncIterator]()
|
||||
await iterator.next()
|
||||
await iterator.next()
|
||||
await expect(iterator.next()).rejects.toThrow('aborted')
|
||||
@@ -503,7 +537,7 @@ describe('installLlmReplay (per-session keying)', () => {
|
||||
]
|
||||
|
||||
const live = (id: string): GenerateOptions =>
|
||||
({ model: 'm', messages: [], sessionId: id as NonNullable<GenerateOptions['sessionId']> })
|
||||
({ provider: 'm', model: 'm', messages: [], sessionId: id as NonNullable<GenerateOptions['sessionId']> })
|
||||
|
||||
it('routes each live session to its own script by FIRST-CALL order', async () => {
|
||||
const parentFile = writeSession('session.jsonl', { id: 'rec-parent', createdAt: 100 }, [TEXT_CHUNKS])
|
||||
@@ -540,7 +574,7 @@ describe('installLlmReplay (per-session keying)', () => {
|
||||
await ctx.plugin(LlmService)
|
||||
installLlmReplay(ctx, { file: parentFile })
|
||||
// No sessionId at all — the legacy single-session path.
|
||||
expect(await drain(ctx.llm.stream({ model: 'm', messages: [] }))).toEqual(TEXT_CHUNKS)
|
||||
expect(await drain(ctx.llm.stream({ provider: 'm', model: 'm', messages: [] }))).toEqual(TEXT_CHUNKS)
|
||||
})
|
||||
|
||||
it('fails loud when more distinct live sessions call than were recorded', async () => {
|
||||
@@ -574,12 +608,13 @@ describe('apply (the plugin entry)', () => {
|
||||
expect(inject).toEqual(['llm'])
|
||||
})
|
||||
|
||||
it('installs replay from an explicit config.file', async () => {
|
||||
it('installs replay and its catalog from explicit config', async () => {
|
||||
writeFileSync(file, sessionJsonl(TEXT_CHUNKS.map((c, i) => chunkEvent(i + 1, 1, 1, c))), 'utf8')
|
||||
const ctx = new Context()
|
||||
await ctx.plugin(LlmService)
|
||||
apply(ctx, { file })
|
||||
expect(await drain(ctx.llm.stream({ model: 'm', messages: [] }))).toEqual(TEXT_CHUNKS)
|
||||
apply(ctx, { file, providers: [{ id: 'm', models: [{ id: 'm' }] }] })
|
||||
expect(ctx.llm.listProviders()).toEqual([{ id: 'm', name: 'm' }])
|
||||
expect(await drain(ctx.llm.stream({ provider: 'm', model: 'm', messages: [] }))).toEqual(TEXT_CHUNKS)
|
||||
})
|
||||
|
||||
it('falls back to $DSH_SNAPSHOT_FILE / $DSH_SNAPSHOT_OVERRIDE when config is empty', async () => {
|
||||
@@ -591,7 +626,7 @@ describe('apply (the plugin entry)', () => {
|
||||
const ctx = new Context()
|
||||
await ctx.plugin(LlmService)
|
||||
apply(ctx)
|
||||
expect(await drain(ctx.llm.stream({ model: 'm', messages: [] }))).toEqual(TEXT_CHUNKS)
|
||||
expect(await drain(ctx.llm.stream({ provider: 'm', model: 'm', messages: [] }))).toEqual(TEXT_CHUNKS)
|
||||
})
|
||||
|
||||
it('uses only the file when no override path is configured or in the env', async () => {
|
||||
@@ -601,7 +636,7 @@ describe('apply (the plugin entry)', () => {
|
||||
const ctx = new Context()
|
||||
await ctx.plugin(LlmService)
|
||||
apply(ctx)
|
||||
expect(await drain(ctx.llm.stream({ model: 'm', messages: [] }))).toEqual(TEXT_CHUNKS)
|
||||
expect(await drain(ctx.llm.stream({ provider: 'm', model: 'm', messages: [] }))).toEqual(TEXT_CHUNKS)
|
||||
})
|
||||
|
||||
it('throws when no fixture path is given by config or env', async () => {
|
||||
@@ -631,7 +666,7 @@ describe('apply (the plugin entry)', () => {
|
||||
await ctx.plugin(LlmService)
|
||||
apply(ctx, { file, childFiles: [childFile] })
|
||||
const live = (id: string): GenerateOptions =>
|
||||
({ model: 'm', messages: [], sessionId: id as NonNullable<GenerateOptions['sessionId']> })
|
||||
({ provider: 'm', model: 'm', messages: [], sessionId: id as NonNullable<GenerateOptions['sessionId']> })
|
||||
expect(await drain(ctx.llm.stream(live('A')))).toEqual(TEXT_CHUNKS)
|
||||
expect(await drain(ctx.llm.stream(live('B')))).toEqual(childSecond)
|
||||
})
|
||||
@@ -651,7 +686,7 @@ describe('apply (the plugin entry)', () => {
|
||||
await ctx.plugin(LlmService)
|
||||
apply(ctx)
|
||||
const live = (id: string): GenerateOptions =>
|
||||
({ model: 'm', messages: [], sessionId: id as NonNullable<GenerateOptions['sessionId']> })
|
||||
({ provider: 'm', model: 'm', messages: [], sessionId: id as NonNullable<GenerateOptions['sessionId']> })
|
||||
expect(await drain(ctx.llm.stream(live('A')))).toEqual(TEXT_CHUNKS)
|
||||
expect(await drain(ctx.llm.stream(live('B')))).toEqual(childChunks)
|
||||
})
|
||||
@@ -663,6 +698,6 @@ describe('apply (the plugin entry)', () => {
|
||||
const ctx = new Context()
|
||||
await ctx.plugin(LlmService)
|
||||
apply(ctx)
|
||||
expect(await drain(ctx.llm.stream({ model: 'm', messages: [] }))).toEqual(TEXT_CHUNKS)
|
||||
expect(await drain(ctx.llm.stream({ provider: 'm', model: 'm', messages: [] }))).toEqual(TEXT_CHUNKS)
|
||||
})
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user