Merge branch 'master' into worktree/llm-mock-fault-server
This commit is contained in:
@@ -53,7 +53,7 @@ Every scenario compares `stdout.expected.jsonl` with cwd-rooted separators canon
|
||||
|
||||
The example also ships a `cordis.snapshot.yml` replay overlay next to its `cordis.yml` (the bin swaps them under `DSH_SNAPSHOT=replay` — [single-source replay config Agent Note](../../../.agents/notes/implemented/testing/2026-07-04-single-source-acp-replay-config.md)); replay fixtures are served by [`dsh-llm-replay`](../llm-replay/README.md), which this package points at via the `DSH_SNAPSHOT_*` env vars it sets on the child. `pnpm run test:snapshot:record` calls the live LLM and rewrites the recorded scenarios' model fixtures; `pnpm run test:snapshot:refresh` stays keyless, runs the replay overlay, and rewrites stdout, comparable session-log expected outputs, and each pin's prompt and tool-schema sidecars from the committed model scripts. Fixture roles, record/replay/refresh semantics, and scenario-table fields are documented on `Scenario` and in the [snapshot Agent Note](../../../.agents/notes/implemented/testing/2026-06-19-acp-snapshot-tests.md).
|
||||
|
||||
Constraints: `suite.ts` imports vitest, so the package entry is importable only inside a vitest run (the launcher, harness, and normalizers have no such dependency but ship from the same entry). ACP-specific by design — the launcher speaks the SDK's `ClientSideConnection`. Input scripts cover initialization, fresh-session creation, text prompting, cancellation, expected RPC failures, and durable turn-boundary waits. Permission round-trips are a FIFO queue of option-kind selections (`allow_once`, `reject_once`, …) mapped to the agent-issued `optionId`; an absent or exhausted queue answers `cancelled`, and an unoffered kind rejects the run.
|
||||
Constraints: `suite.ts` imports vitest, so the package entry is importable only inside a vitest run (the launcher, harness, and normalizers have no such dependency but ship from the same entry). The launcher and suite factory are ACP-specific by design — the launcher speaks the SDK's `ClientSideConnection` — while the normalizers are transport-neutral session-log/text helpers also consumed by the TUI snapshot suite and the web browser e2e lane. Input scripts cover initialization, fresh-session creation, text prompting, cancellation, expected RPC failures, and durable turn-boundary waits. Permission round-trips are a FIFO queue of option-kind selections (`allow_once`, `reject_once`, …) mapped to the agent-issued `optionId`; an absent or exhausted queue answers `cancelled`, and an unoffered kind rejects the run.
|
||||
|
||||
## Model Experience
|
||||
|
||||
|
||||
@@ -11,11 +11,17 @@ const CWD = '{{cwd}}'
|
||||
const SYSTEM = '{{system}}'
|
||||
const TOOLS = '{{tools}}'
|
||||
const MESSAGE_PREFIX = '{{messagePrefix}}'
|
||||
const EVENT_TIME = '{{eventTime}}'
|
||||
const EVENT_OMITTED_BYTES = '{{eventOmittedBytes}}'
|
||||
|
||||
/** A cwd-rooted path after volatile cwd replacement, through its last separator-delimited segment. */
|
||||
const CWD_ROOTED_PATH_RE = /\{\{cwd\}\}(?:[\\/][^\s<>"'`]+)+/g
|
||||
const PATH_TAG_RE = /(<path>)([^<]*)(<\/path>)/g
|
||||
const ADDITIONAL_INSTRUCTIONS_PATH_RE = /(Additional instructions from: )([^\r\n]+)/g
|
||||
const EMBEDDED_EVENT_TIME_RE = /^( "time": )\d+(?=,\r?$)/gm
|
||||
const EVENT_READ_OMITTED_BYTES_RE = /(\r?\n\r?\n\(Omitted )\d+( bytes\.)/g
|
||||
const EVENT_READ_TARGET_REGION_RE
|
||||
= /^Session [^\r\n]+ — [^\r\n]+\r?\nTarget event seq \d+:\r?\n```json\r?\n\{\r?\n[\s\S]*?(?=\r?\n```(?:\r?\n|$)|\r?\n\r?\n\(Omitted )/
|
||||
|
||||
/** A UUID v4 string, the shape `randomUUID()` produces for session ids. */
|
||||
const UUID_RE = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/gi
|
||||
@@ -77,6 +83,16 @@ function scrubString(value: string, ctx: NormalizeContext, cwdPathMode: CwdPathM
|
||||
}
|
||||
out = out.replace(LOCAL_SPILL_PATH_RE, (_match, name: string) => `{{spillLocator:${name}}}`)
|
||||
out = out.replace(SNAPSHOT_SPILL_PATH_RE, (_match, name: string) => `{{spillLocator:${name}}}`)
|
||||
// Exact event-read results render the target as pretty JSON inside a
|
||||
// distinctive envelope. Restrict time scrubbing to that fenced target so
|
||||
// neighbor, model, bash, and unrelated tool text remains regression-visible.
|
||||
if (EVENT_READ_TARGET_REGION_RE.test(out)) {
|
||||
out = out.replace(
|
||||
EVENT_READ_TARGET_REGION_RE,
|
||||
target => target.replace(EMBEDDED_EVENT_TIME_RE, `$1${EVENT_TIME}`),
|
||||
)
|
||||
out = out.replace(EVENT_READ_OMITTED_BYTES_RE, `$1${EVENT_OMITTED_BYTES}$2`)
|
||||
}
|
||||
for (const id of ctx.sessionIds) out = out.split(id).join(SESSION_ID)
|
||||
out = out.replace(UUID_RE, SESSION_ID)
|
||||
return out
|
||||
|
||||
@@ -123,6 +123,56 @@ Additional instructions from: nested\AGENTS.md`,
|
||||
expect(out).not.toContain('"id"')
|
||||
})
|
||||
|
||||
it('stabilizes only the top-level event timestamp and spill byte count in event-read text', () => {
|
||||
const raw = JSON.stringify({
|
||||
jsonrpc: '2.0',
|
||||
method: 'session/update',
|
||||
params: {
|
||||
update: {
|
||||
sessionUpdate: 'tool_call_update',
|
||||
content: [{
|
||||
type: 'content',
|
||||
content: {
|
||||
type: 'text',
|
||||
text: 'Session prior — title\nTarget event seq 4:\n```json\n{\n "seq": 4,\n "time": 1784876275593,\n "data": {\n "time": 31337,\n "note": "model-visible"\n }\n}\n```\n\nAfter:\n "time": 424242,\n neighbor semantic text\n\n(Omitted 39387 bytes. Full formatted result stored at: /tmp/result.txt.)',
|
||||
},
|
||||
}],
|
||||
},
|
||||
},
|
||||
})
|
||||
const out = normalizeStdout(raw, ctx)
|
||||
expect(out).toContain('\\"time\\": {{eventTime}}')
|
||||
expect(out).toContain('\\"time\\": 31337')
|
||||
expect(out).toContain('\\"time\\": 424242')
|
||||
expect(out).toContain('Omitted {{eventOmittedBytes}} bytes')
|
||||
expect(out).not.toContain('1784876275593')
|
||||
expect(out).not.toContain('39387')
|
||||
})
|
||||
|
||||
it('preserves event-like timestamps in unrelated output text', () => {
|
||||
const raw = JSON.stringify({
|
||||
jsonrpc: '2.0',
|
||||
method: 'session/update',
|
||||
params: {
|
||||
update: {
|
||||
sessionUpdate: 'tool_call_update',
|
||||
content: [{
|
||||
type: 'content',
|
||||
content: {
|
||||
type: 'text',
|
||||
text: 'bash output:\n```json\n{\n "time": 1784876275593,\n "data": {}\n}\n```\n\n(Omitted 39387 bytes. Full formatted result stored at: /tmp/result.txt.)',
|
||||
},
|
||||
}],
|
||||
},
|
||||
},
|
||||
})
|
||||
const out = normalizeStdout(raw, ctx)
|
||||
expect(out).toContain('1784876275593')
|
||||
expect(out).toContain('39387')
|
||||
expect(out).not.toContain('{{eventTime}}')
|
||||
expect(out).not.toContain('{{eventOmittedBytes}}')
|
||||
})
|
||||
|
||||
it('throws on a non-JSON stdout line (the purity check)', () => {
|
||||
const raw = `${JSON.stringify({ jsonrpc: '2.0', id: 1 })}\noops a log leaked\n`
|
||||
expect(() => normalizeStdout(raw, ctx)).toThrow()
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
A replay LLM plugin for keyless snapshot tests. It yields model streams reconstructed from a recorded **session JSONL** fixture, so a test can boot the real agent against a fixed model transcript with no API key. With `providers` configured it registers a replay-only adapter whose catalog is available to scenarios that exercise model discovery; without `providers` it installs the catch-all `llm/stream` waterfall used by tests that do not need discovery.
|
||||
|
||||
Its consumers are the ACP snapshot harness in `examples/acp-agent` and the `stream-json` snapshot in `examples/headless-agent`; each loads this plugin in place of a real LLM adapter. Keeping derivation and replay here places that logic under the per-file 100% coverage gate on `packages/*/src`.
|
||||
Its consumers are the ACP, headless `stream-json`, and TUI snapshot suites plus the web browser e2e lane. Loader-driven suites mount this plugin in place of a real LLM adapter; the web lane installs it directly to retain the teardown consumption handle. Keeping derivation and replay here places that logic under the per-file 100% coverage gate on `packages/*/src`.
|
||||
|
||||
## How the fixture works
|
||||
|
||||
@@ -24,6 +24,7 @@ Replay keys every call by its calling session id (`GenerateOptions.sessionId`, s
|
||||
| `overrideFile` | string | `$DSH_SNAPSHOT_OVERRIDE` | Optional path to a `ReplayEntry[]` sidecar that replaces the PRIMARY session's derived script. |
|
||||
| `childFiles` | string[] | `$DSH_SNAPSHOT_CHILD_FILES` (path-delimited) | Recorded subagent child-session logs for a nested scenario; empty for a single-session scenario. |
|
||||
| `providers` | `ReplayProviderConfig[]` | — | Optional replay-only provider and model catalog. Each model may publish `contextWindow`; configured routes dispatch through the replay adapter and never perform provider I/O. |
|
||||
| `paceMs` | number | — (burst) | Optional per-chunk delay in ms so downstream transports (e.g. the web SSE mux observed by a real browser) see genuinely incremental delivery. A realism knob only — tests must not depend on it for correctness. Non-negative integer; abort during a pace wait cancels the stream promptly. |
|
||||
|
||||
```yaml
|
||||
- id: llm-replay
|
||||
@@ -43,11 +44,11 @@ Replay keys every call by its calling session id (`GenerateOptions.sessionId`, s
|
||||
|
||||
## Exports
|
||||
|
||||
- `installLlmReplay(ctx, config)` — install the configured replay adapter or catch-all `llm/stream` listener; returns the disposer (HMR safety). Use this in tests to drive replay without the Loader or env vars.
|
||||
- `installLlmReplay(ctx, config)` — install the configured replay adapter or catch-all `llm/stream` listener; returns a `ReplayHandle` (`dispose()` for HMR safety plus `assertConsumed()`, the teardown check that every recorded script bound to a live session and every bound cursor drained — turning a scenario that silently drove fewer model calls than recorded into a crisp diagnostic). Use this in tests to drive replay without the Loader or env vars.
|
||||
- `loadSessionScripts(config)` — resolve the ordered `SessionScript[]` (primary + children) for a scenario, ready to bind to live sessions in first-call order.
|
||||
- `loadReplayScript(config)` — resolve the `ReplayEntry[]` for the PRIMARY session only (sidecar override if present, else derived from the JSONL; fail-loud if the fixture is missing).
|
||||
- `deriveReplayScript(events)` / `parseSessionLog(text)` / `parseSessionHeader(text)` — the pure helpers that turn a recorded session log into a script and read its header `id`/`createdAt`. A derived group must end in a `finish` chunk; a group without one is the fingerprint of a thrown `stream()` and must instead be expressed via an override sidecar.
|
||||
- Types `ReplayEntry` / `SessionScript` / `ReplayConfig` / `ReplayProviderConfig` / `ReplayModelConfig` / `Config`.
|
||||
- Types `ReplayEntry` / `SessionScript` / `ReplayConfig` / `ReplayProviderConfig` / `ReplayModelConfig` / `ReplayHandle` / `Config`.
|
||||
|
||||
## Plugin export shape
|
||||
|
||||
|
||||
@@ -78,6 +78,32 @@ export interface ReplayConfig {
|
||||
* by tests that do not need discovery.
|
||||
*/
|
||||
providers?: ReplayProviderConfig[]
|
||||
/**
|
||||
* Optional per-chunk pacing delay in milliseconds: each replayed chunk waits
|
||||
* this long before yielding, so a downstream transport (e.g. the web SSE
|
||||
* mux observed by a browser) sees genuinely incremental delivery. A realism
|
||||
* knob only — correctness must never depend on it. Absent or `0` keeps
|
||||
* today's synchronous burst yield. Must be a non-negative finite integer;
|
||||
* aborting mid-wait cancels the stream like any other abort.
|
||||
*/
|
||||
paceMs?: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle returned by {@link installLlmReplay}: removal plus the end-of-run
|
||||
* consumption check that turns silent fixture underruns (a scenario that
|
||||
* issued fewer calls than recorded, or never bound a recorded child script)
|
||||
* into a crisp diagnostic at teardown.
|
||||
*/
|
||||
export interface ReplayHandle {
|
||||
/** Remove the registered adapter or waterfall listener (HMR safety). Freestanding closure — safe to destructure. */
|
||||
dispose(this: void): void
|
||||
/**
|
||||
* Throw unless every recorded script was bound to a live session and every
|
||||
* bound cursor consumed its full entry list. Call at scenario teardown.
|
||||
* Freestanding closure — safe to destructure.
|
||||
*/
|
||||
assertConsumed(this: void): void
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -281,12 +307,32 @@ class ReplayAdapter extends LlmAdapter {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Wait `paceMs` between chunk yields, aborting the wait (and the stream) the
|
||||
* moment the signal fires — a paced replay must cancel as promptly as a burst
|
||||
* one.
|
||||
*/
|
||||
function paceDelay(paceMs: number, signal: AbortSignal | undefined): Promise<void> {
|
||||
return new Promise<void>((resolve, reject) => {
|
||||
const timer = setTimeout(() => {
|
||||
signal?.removeEventListener('abort', onAbort)
|
||||
resolve()
|
||||
}, paceMs)
|
||||
const onAbort = (): void => {
|
||||
clearTimeout(timer)
|
||||
reject(new Error('aborted'))
|
||||
}
|
||||
signal?.addEventListener('abort', onAbort, { once: true })
|
||||
})
|
||||
}
|
||||
|
||||
/** Yield a recorded stream back, honoring abort like a real adapter. */
|
||||
async function* replayEntry(entry: ReplayEntry, signal: AbortSignal | undefined): AsyncIterable<StreamChunk> {
|
||||
async function* replayEntry(entry: ReplayEntry, signal: AbortSignal | undefined, paceMs: number): AsyncIterable<StreamChunk> {
|
||||
switch (entry.kind) {
|
||||
case 'chunks':
|
||||
for (const chunk of entry.chunks) {
|
||||
if (signal?.aborted) throw new Error('aborted')
|
||||
if (paceMs > 0) await paceDelay(paceMs, signal)
|
||||
yield chunk
|
||||
}
|
||||
return
|
||||
@@ -297,6 +343,7 @@ async function* replayEntry(entry: ReplayEntry, signal: AbortSignal | undefined)
|
||||
// mid-stream STREAM_CLOSED after partial chunks).
|
||||
for (const chunk of entry.chunks) {
|
||||
if (signal?.aborted) throw new Error('aborted')
|
||||
if (paceMs > 0) await paceDelay(paceMs, signal)
|
||||
yield chunk
|
||||
}
|
||||
throw new LlmError(entry.message, entry.code)
|
||||
@@ -324,14 +371,17 @@ async function* replayEntry(entry: ReplayEntry, signal: AbortSignal | undefined)
|
||||
* next ordered recorded script, then advances its own cursor synchronously at
|
||||
* invocation time; calls without `sessionId` share one anonymous session. A
|
||||
* non-empty provider catalog registers a routed replay adapter; otherwise a
|
||||
* catch-all waterfall intercepts requests. Returns the effect disposer for
|
||||
* HMR-safe removal.
|
||||
* catch-all waterfall intercepts requests.
|
||||
*
|
||||
* @param ctx - the context whose LLM service receives the replay route or waterfall.
|
||||
* @param config - the resolved fixture paths (env-var defaulting is `apply`'s job).
|
||||
* @returns the disposer that removes the registered adapter or listener.
|
||||
* @returns the {@link ReplayHandle} carrying the disposer and the teardown consumption check.
|
||||
*/
|
||||
export function installLlmReplay(ctx: Context, config: ReplayConfig): () => void {
|
||||
export function installLlmReplay(ctx: Context, config: ReplayConfig): ReplayHandle {
|
||||
const paceMs = config.paceMs ?? 0
|
||||
if (!Number.isInteger(paceMs) || paceMs < 0) {
|
||||
throw new Error(`llm-replay: paceMs must be a non-negative integer, got ${String(config.paceMs)}`)
|
||||
}
|
||||
const scripts = loadSessionScripts(config)
|
||||
// Live-session → its bound script + cursor. A new live session id claims the
|
||||
// next not-yet-bound script (scripts are in bind order); `nextScript` is the
|
||||
@@ -375,14 +425,31 @@ export function installLlmReplay(ctx: Context, config: ReplayConfig): () => void
|
||||
+ `but its script has only ${boundState.entries.length}; re-record the scenario`,
|
||||
)
|
||||
}
|
||||
yield* replayEntry(entry, options.signal)
|
||||
yield* replayEntry(entry, options.signal, paceMs)
|
||||
})()
|
||||
}
|
||||
const providers = config.providers ?? []
|
||||
if (providers.length > 0) {
|
||||
return ctx.llm.registerAdapter(providers.map(provider => provider.id), new ReplayAdapter(providers, replay))
|
||||
const dispose = providers.length > 0
|
||||
? ctx.llm.registerAdapter(providers.map(provider => provider.id), new ReplayAdapter(providers, replay))
|
||||
: ctx.on('llm/stream', (options: GenerateOptions, _next) => replay(options))
|
||||
return {
|
||||
dispose,
|
||||
assertConsumed(): void {
|
||||
const problems: string[] = []
|
||||
if (nextScript < scripts.length) {
|
||||
problems.push(`${scripts.length - nextScript} recorded script(s) never bound to a live session`)
|
||||
}
|
||||
for (const [key, state] of bound) {
|
||||
if (state.cursor < state.entries.length) {
|
||||
const who = key === ANON ? 'the anonymous session' : `session ${key}`
|
||||
problems.push(`${who} consumed ${state.cursor}/${state.entries.length} recorded call(s)`)
|
||||
}
|
||||
}
|
||||
if (problems.length > 0) {
|
||||
throw new Error(`llm-replay: fixture not fully consumed — ${problems.join('; ')}; the scenario drove fewer model calls than recorded`)
|
||||
}
|
||||
},
|
||||
}
|
||||
return ctx.on('llm/stream', (options: GenerateOptions, _next) => replay(options))
|
||||
}
|
||||
|
||||
export const name = 'llm-replay'
|
||||
@@ -402,6 +469,8 @@ export interface Config {
|
||||
childFiles?: string[]
|
||||
/** Optional replay-only provider catalog; absent or empty selects catch-all waterfall replay. */
|
||||
providers?: ReplayProviderConfig[]
|
||||
/** Optional per-chunk pacing delay in ms (see {@link ReplayConfig.paceMs}); absent keeps burst yield. */
|
||||
paceMs?: number
|
||||
}
|
||||
|
||||
export function apply(ctx: Context, config: Config = {}): void {
|
||||
@@ -418,5 +487,6 @@ export function apply(ctx: Context, config: Config = {}): void {
|
||||
...overrideFile !== undefined && overrideFile.length > 0 ? { overrideFile } : {},
|
||||
...childFiles.length > 0 ? { childFiles } : {},
|
||||
...config.providers !== undefined ? { providers: config.providers } : {},
|
||||
...config.paceMs !== undefined ? { paceMs: config.paceMs } : {},
|
||||
})
|
||||
}
|
||||
|
||||
@@ -234,7 +234,7 @@ describe('installLlmReplay (through the real LlmService)', () => {
|
||||
writeLog(TEXT_CHUNKS)
|
||||
const ctx = new Context()
|
||||
await ctx.plugin(LlmService)
|
||||
const dispose = installLlmReplay(ctx, {
|
||||
const { dispose } = installLlmReplay(ctx, {
|
||||
file,
|
||||
providers: [
|
||||
{
|
||||
@@ -431,6 +431,92 @@ describe('installLlmReplay (through the real LlmService)', () => {
|
||||
await iterator.next()
|
||||
await expect(iterator.next()).rejects.toThrow('aborted')
|
||||
})
|
||||
|
||||
it('rejects a paceMs that is not a non-negative integer', async () => {
|
||||
writeLog(TEXT_CHUNKS)
|
||||
const ctx = new Context()
|
||||
await ctx.plugin(LlmService)
|
||||
expect(() => installLlmReplay(ctx, { file, paceMs: -1 })).toThrow(/paceMs/)
|
||||
expect(() => installLlmReplay(ctx, { file, paceMs: 1.5 })).toThrow(/paceMs/)
|
||||
})
|
||||
|
||||
it('paces chunk yields when paceMs is set (each chunk waits at least the pace)', async () => {
|
||||
writeLog(TEXT_CHUNKS)
|
||||
const ctx = new Context()
|
||||
await ctx.plugin(LlmService)
|
||||
installLlmReplay(ctx, { file, paceMs: 10 })
|
||||
const started = performance.now()
|
||||
const chunks = await drain(ctx.llm.stream({ provider: 'm', model: 'm', messages: [] }))
|
||||
expect(chunks).toEqual(TEXT_CHUNKS)
|
||||
// N chunks × 10ms; allow generous scheduling slack, assert the floor only.
|
||||
expect(performance.now() - started).toBeGreaterThanOrEqual(TEXT_CHUNKS.length * 10 - 5)
|
||||
})
|
||||
|
||||
it('aborting DURING a pace wait cancels the stream promptly', async () => {
|
||||
writeLog(TEXT_CHUNKS)
|
||||
const ctx = new Context()
|
||||
await ctx.plugin(LlmService)
|
||||
installLlmReplay(ctx, { file, paceMs: 60_000 })
|
||||
const controller = new AbortController()
|
||||
const pending = drain(ctx.llm.stream({ provider: 'm', model: 'm', messages: [], signal: controller.signal }))
|
||||
// Let the generator park inside the pace timer, then abort — the reject
|
||||
// must come from the abort listener, not the (distant) timer.
|
||||
await new Promise(r => setImmediate(r))
|
||||
controller.abort()
|
||||
await expect(pending).rejects.toThrow('aborted')
|
||||
})
|
||||
|
||||
it('assertConsumed passes only after every recorded call replayed', async () => {
|
||||
writeLog(TEXT_CHUNKS, TEXT_CHUNKS)
|
||||
const ctx = new Context()
|
||||
await ctx.plugin(LlmService)
|
||||
const handle = installLlmReplay(ctx, { file })
|
||||
await drain(ctx.llm.stream({ provider: 'm', model: 'm', messages: [] }))
|
||||
// One of two recorded calls consumed — the underrun must name the gap.
|
||||
expect(() => { handle.assertConsumed() }).toThrow(/consumed 1\/2 recorded call/)
|
||||
await drain(ctx.llm.stream({ provider: 'm', model: 'm', messages: [] }))
|
||||
expect(() => { handle.assertConsumed() }).not.toThrow()
|
||||
})
|
||||
|
||||
it('paces a throw-entry prefix too (the recorded partial streams at the same cadence)', async () => {
|
||||
writeFileSync(file, sessionJsonl([]), 'utf8')
|
||||
const overrideFile = join(dir, 'replay.override.json')
|
||||
const partial: StreamChunk[] = [{ type: 'block-start', index: 0, blockType: 'text' }]
|
||||
writeFileSync(overrideFile, JSON.stringify([
|
||||
{ kind: 'throw', chunks: partial, message: 'boom', code: 'STREAM_CLOSED' },
|
||||
]), 'utf8')
|
||||
const ctx = new Context()
|
||||
await ctx.plugin(LlmService)
|
||||
installLlmReplay(ctx, { file, overrideFile, paceMs: 10 })
|
||||
const started = performance.now()
|
||||
await expect(drain(ctx.llm.stream({ provider: 'm', model: 'm', messages: [] }))).rejects.toThrow('boom')
|
||||
expect(performance.now() - started).toBeGreaterThanOrEqual(5)
|
||||
})
|
||||
|
||||
it('assertConsumed names an underrunning identified session by its id', async () => {
|
||||
writeLog(TEXT_CHUNKS, TEXT_CHUNKS)
|
||||
const ctx = new Context()
|
||||
await ctx.plugin(LlmService)
|
||||
const handle = installLlmReplay(ctx, { file })
|
||||
const sessionId = 'live-underrun' as NonNullable<GenerateOptions['sessionId']>
|
||||
await drain(ctx.llm.stream({ provider: 'm', model: 'm', messages: [], sessionId }))
|
||||
expect(() => { handle.assertConsumed() }).toThrow(/session live-underrun consumed 1\/2/)
|
||||
})
|
||||
|
||||
it('assertConsumed reports recorded scripts no live session ever bound', async () => {
|
||||
writeLog(TEXT_CHUNKS)
|
||||
const childFile = join(dir, 'session.1.jsonl')
|
||||
writeFileSync(childFile, sessionJsonl(
|
||||
TEXT_CHUNKS.map((chunk, i) => chunkEvent(i + 1, 1, 1, chunk)),
|
||||
{ id: 'child', createdAt: 10 },
|
||||
), 'utf8')
|
||||
const ctx = new Context()
|
||||
await ctx.plugin(LlmService)
|
||||
const handle = installLlmReplay(ctx, { file, childFiles: [childFile] })
|
||||
await drain(ctx.llm.stream({ provider: 'm', model: 'm', messages: [], sessionId: 'live-parent' as NonNullable<GenerateOptions['sessionId']> }))
|
||||
// The child script never bound: the scenario drove fewer sessions than recorded.
|
||||
expect(() => { handle.assertConsumed() }).toThrow(/1 recorded script\(s\) never bound/)
|
||||
})
|
||||
})
|
||||
|
||||
describe('parseSessionHeader', () => {
|
||||
@@ -631,7 +717,7 @@ describe('apply (the plugin entry)', () => {
|
||||
writeFileSync(file, sessionJsonl(TEXT_CHUNKS.map((c, i) => chunkEvent(i + 1, 1, 1, c))), 'utf8')
|
||||
const ctx = new Context()
|
||||
await ctx.plugin(LlmService)
|
||||
apply(ctx, { file, providers: [{ id: 'm', models: [{ id: 'm' }] }] })
|
||||
apply(ctx, { file, providers: [{ id: 'm', models: [{ id: 'm' }] }], paceMs: 1 })
|
||||
expect(ctx.llm.listProviders()).toEqual([{ id: 'm', name: 'm' }])
|
||||
expect(await drain(ctx.llm.stream({ provider: 'm', model: 'm', messages: [] }))).toEqual(TEXT_CHUNKS)
|
||||
})
|
||||
|
||||
@@ -5,7 +5,7 @@ import {
|
||||
resolveExampleMode,
|
||||
} from '@deepseek-ai/dsh-loader-smoke'
|
||||
|
||||
const SRC_BIN = '/repo/packages/examples/tui-demo/src/bin.ts'
|
||||
const SRC_BIN = '/repo/packages/examples/cli-demo/src/bin.ts'
|
||||
const TSCONFIG = '/repo/tsconfig.json'
|
||||
|
||||
const originalMode = process.env[EXAMPLE_MODE_ENV]
|
||||
@@ -65,7 +65,7 @@ describe('resolveExampleLaunch', () => {
|
||||
env: { DSH_HOME: '/tmp/home' },
|
||||
})
|
||||
expect(args).not.toContain('--import')
|
||||
expect(args).toContain('/repo/packages/examples/tui-demo/lib/bin.js')
|
||||
expect(args).toContain('/repo/packages/examples/cli-demo/lib/bin.js')
|
||||
expect(args.slice(-2)).toEqual(['--config', './cordis.yml'])
|
||||
expect(env.TSX_TSCONFIG_PATH).toBeUndefined()
|
||||
expect(env.DSH_HOME).toBe('/tmp/home')
|
||||
@@ -100,6 +100,6 @@ describe('resolveExampleLaunch', () => {
|
||||
it('defaults the mode from the environment', () => {
|
||||
process.env[EXAMPLE_MODE_ENV] = 'lib'
|
||||
const { args } = resolveExampleLaunch({ srcBin: SRC_BIN })
|
||||
expect(args).toContain('/repo/packages/examples/tui-demo/lib/bin.js')
|
||||
expect(args).toContain('/repo/packages/examples/cli-demo/lib/bin.js')
|
||||
})
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user