feat(acp): advertise and switch llm models
This commit is contained in:
@@ -1,6 +1,6 @@
|
||||
# @deepseek-ai/dsh-llm-replay
|
||||
|
||||
A replay LLM plugin for keyless snapshot tests. It installs a single `llm/stream` waterfall listener that short-circuits the waterfall (never calls `next()`) and yields model streams reconstructed from a recorded **session JSONL** fixture — so a test can boot the real agent against a fixed model transcript with no API key.
|
||||
A replay LLM plugin for keyless snapshot tests. It yields model streams reconstructed from a recorded **session JSONL** fixture, so a test can boot the real agent against a fixed model transcript with no API key. With `providers` configured it registers a replay-only adapter whose catalog is visible to clients such as ACP editors; without `providers` it installs the catch-all `llm/stream` waterfall used by tests that do not need discovery.
|
||||
|
||||
Its consumer is the ACP snapshot harness in `examples/acp-agent`, which loads this plugin (via `cordis.snapshot.yml`) in place of a real LLM adapter. The package exists so its derive/parse/replay logic falls under the per-file 100% coverage gate on `packages/*/src` (the same logic, while it lived under `examples/`, was outside the gate).
|
||||
|
||||
@@ -23,10 +23,18 @@ Replay keys every call by its calling session id (`GenerateOptions.sessionId`, s
|
||||
| `file` | string | `$DSH_SNAPSHOT_FILE` | Path to the primary (parent) `session.jsonl` fixture. Required (config or env). |
|
||||
| `overrideFile` | string | `$DSH_SNAPSHOT_OVERRIDE` | Optional path to a `ReplayEntry[]` sidecar that replaces the PRIMARY session's derived script. |
|
||||
| `childFiles` | string[] | `$DSH_SNAPSHOT_CHILD_FILES` (path-delimited) | Recorded subagent child-session logs for a nested scenario; empty for a single-session scenario. |
|
||||
| `providers` | `ReplayProviderConfig[]` | — | Optional replay-only provider and model catalog. Configured routes dispatch through the replay adapter and never perform provider I/O. |
|
||||
|
||||
```yaml
|
||||
- id: llm-replay
|
||||
name: '@deepseek-ai/dsh-llm-replay'
|
||||
config:
|
||||
providers:
|
||||
- id: deepseek
|
||||
name: DeepSeek
|
||||
models:
|
||||
- id: deepseek-v4-flash
|
||||
- id: deepseek-v4-pro
|
||||
# file/overrideFile/childFiles default to $DSH_SNAPSHOT_FILE /
|
||||
# $DSH_SNAPSHOT_OVERRIDE / $DSH_SNAPSHOT_CHILD_FILES, set by the snapshot
|
||||
# harness per scenario.
|
||||
@@ -34,11 +42,11 @@ Replay keys every call by its calling session id (`GenerateOptions.sessionId`, s
|
||||
|
||||
## Exports
|
||||
|
||||
- `installLlmReplay(ctx, config)` — install the `llm/stream` listener; returns the disposer (HMR safety). Use this in tests to drive replay without the Loader or env vars.
|
||||
- `installLlmReplay(ctx, config)` — install the configured replay adapter or catch-all `llm/stream` listener; returns the disposer (HMR safety). Use this in tests to drive replay without the Loader or env vars.
|
||||
- `loadSessionScripts(config)` — resolve the ordered `SessionScript[]` (primary + children) for a scenario, ready to bind to live sessions in first-call order.
|
||||
- `loadReplayScript(config)` — resolve the `ReplayEntry[]` for the PRIMARY session only (sidecar override if present, else derived from the JSONL; fail-loud if the fixture is missing).
|
||||
- `deriveReplayScript(events)` / `parseSessionLog(text)` / `parseSessionHeader(text)` — the pure helpers that turn a recorded session log into a script and read its header `id`/`createdAt`. A derived group must end in a `finish` chunk; a group without one is the fingerprint of a thrown `stream()` and must instead be expressed via an override sidecar.
|
||||
- Types `ReplayEntry` / `SessionScript` / `ReplayConfig` / `Config`.
|
||||
- Types `ReplayEntry` / `SessionScript` / `ReplayConfig` / `ReplayProviderConfig` / `ReplayModelConfig` / `Config`.
|
||||
|
||||
## Plugin export shape
|
||||
|
||||
|
||||
@@ -10,8 +10,8 @@ import { existsSync, readFileSync } from 'node:fs'
|
||||
import { delimiter as pathDelimiter } from 'node:path'
|
||||
import type { Context } from 'cordis'
|
||||
import type { SessionEvent } from '@deepseek-ai/dsh-session'
|
||||
import type { GenerateOptions, StreamChunk } from '@deepseek-ai/dsh-llm'
|
||||
import { LlmError, assertNever } from '@deepseek-ai/dsh-llm'
|
||||
import type { GenerateOptions, LlmModelInfo, LlmProviderInfo, StreamChunk } from '@deepseek-ai/dsh-llm'
|
||||
import { LlmAdapter, LlmError, assertNever } from '@deepseek-ai/dsh-llm'
|
||||
|
||||
/**
|
||||
* One recorded model call. `throw` may replay prefix chunks before failing;
|
||||
@@ -23,6 +23,26 @@ export type ReplayEntry =
|
||||
| { kind: 'throw'; chunks: StreamChunk[]; message: string; code: string; status?: number }
|
||||
| { kind: 'hang' }
|
||||
|
||||
/** One model exposed by a replay-only provider catalog. */
|
||||
export interface ReplayModelConfig {
|
||||
/** Model id used for replay requests. */
|
||||
id: string
|
||||
/** Selector label; defaults to {@link id}. */
|
||||
name?: string
|
||||
/** Optional selector description. */
|
||||
description?: string
|
||||
}
|
||||
|
||||
/** One provider route exposed by the replay adapter. */
|
||||
export interface ReplayProviderConfig {
|
||||
/** Provider route used for replay requests. */
|
||||
id: string
|
||||
/** Selector label; defaults to {@link id}. */
|
||||
name?: string
|
||||
/** Advisory models exposed to clients such as ACP editors. */
|
||||
models?: ReplayModelConfig[]
|
||||
}
|
||||
|
||||
/** Resolved plugin configuration. */
|
||||
export interface ReplayConfig {
|
||||
/**
|
||||
@@ -45,6 +65,12 @@ export interface ReplayConfig {
|
||||
* for a single-session scenario.
|
||||
*/
|
||||
childFiles?: string[]
|
||||
/**
|
||||
* Optional provider catalog. When non-empty, replay registers an adapter for
|
||||
* these routes; when absent or empty, it retains the catch-all waterfall used
|
||||
* by tests that do not need discovery.
|
||||
*/
|
||||
providers?: ReplayProviderConfig[]
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -203,6 +229,42 @@ export function loadSessionScripts(config: ReplayConfig): SessionScript[] {
|
||||
return [primary, ...children]
|
||||
}
|
||||
|
||||
/** Replay adapter that makes a configured provider catalog discoverable without provider I/O. */
|
||||
class ReplayAdapter extends LlmAdapter {
|
||||
private readonly providers: ReadonlyMap<string, ReplayProviderConfig>
|
||||
|
||||
constructor(
|
||||
providers: readonly ReplayProviderConfig[],
|
||||
private readonly replay: (options: GenerateOptions) => AsyncIterable<StreamChunk>,
|
||||
) {
|
||||
super()
|
||||
this.providers = new Map(providers.map(provider => [provider.id, provider]))
|
||||
}
|
||||
|
||||
override providerInfo(provider: string): LlmProviderInfo {
|
||||
const configured = this.providers.get(provider)
|
||||
/* v8 ignore next -- LlmService only asks about routes registered from this same map. */
|
||||
if (configured === undefined) return super.providerInfo(provider)
|
||||
return { id: provider, name: configured.name ?? provider }
|
||||
}
|
||||
|
||||
override listModels(provider: string): Promise<readonly LlmModelInfo[]> {
|
||||
const configured = this.providers.get(provider)
|
||||
/* v8 ignore next -- LlmService only asks about routes registered from this same map. */
|
||||
if (configured === undefined) return Promise.resolve([])
|
||||
return Promise.resolve((configured.models ?? []).map(model => ({
|
||||
provider,
|
||||
id: model.id,
|
||||
name: model.name ?? model.id,
|
||||
...model.description === undefined ? {} : { description: model.description },
|
||||
})))
|
||||
}
|
||||
|
||||
override stream(options: GenerateOptions): AsyncIterable<StreamChunk> {
|
||||
return this.replay(options)
|
||||
}
|
||||
}
|
||||
|
||||
/** Yield a recorded stream back, honoring abort like a real adapter. */
|
||||
async function* replayEntry(entry: ReplayEntry, signal: AbortSignal | undefined): AsyncIterable<StreamChunk> {
|
||||
switch (entry.kind) {
|
||||
@@ -243,12 +305,14 @@ async function* replayEntry(entry: ReplayEntry, signal: AbortSignal | undefined)
|
||||
/**
|
||||
* Install per-session positional replay. A newly seen live session takes the
|
||||
* next ordered recorded script, then advances its own cursor synchronously at
|
||||
* invocation time; calls without `sessionId` share one anonymous session.
|
||||
* Returns the effect disposer for HMR-safe removal.
|
||||
* invocation time; calls without `sessionId` share one anonymous session. A
|
||||
* non-empty provider catalog registers a routed replay adapter; otherwise a
|
||||
* catch-all waterfall intercepts requests. Returns the effect disposer for
|
||||
* HMR-safe removal.
|
||||
*
|
||||
* @param ctx - the context whose `llm/stream` waterfall the listener short-circuits.
|
||||
* @param ctx - the context whose LLM service receives the replay route or waterfall.
|
||||
* @param config - the resolved fixture paths (env-var defaulting is `apply`'s job).
|
||||
* @returns the `ctx.on` disposer that removes the listener.
|
||||
* @returns the disposer that removes the registered adapter or listener.
|
||||
*/
|
||||
export function installLlmReplay(ctx: Context, config: ReplayConfig): () => void {
|
||||
const scripts = loadSessionScripts(config)
|
||||
@@ -258,7 +322,7 @@ export function installLlmReplay(ctx: Context, config: ReplayConfig): () => void
|
||||
const bound = new Map<string, { entries: ReplayEntry[]; cursor: number }>()
|
||||
let nextScript = 0
|
||||
const ANON = '\0anon\0' // the key for a call that carries no sessionId
|
||||
return ctx.on('llm/stream', (options: GenerateOptions, _next) => {
|
||||
const replay = (options: GenerateOptions): AsyncIterable<StreamChunk> => {
|
||||
const key = options.sessionId ?? ANON
|
||||
let state = bound.get(key)
|
||||
let unrecorded = false
|
||||
@@ -296,7 +360,12 @@ export function installLlmReplay(ctx: Context, config: ReplayConfig): () => void
|
||||
}
|
||||
yield* replayEntry(entry, options.signal)
|
||||
})()
|
||||
})
|
||||
}
|
||||
const providers = config.providers ?? []
|
||||
if (providers.length > 0) {
|
||||
return ctx.llm.registerAdapter(providers.map(provider => provider.id), new ReplayAdapter(providers, replay))
|
||||
}
|
||||
return ctx.on('llm/stream', (options: GenerateOptions, _next) => replay(options))
|
||||
}
|
||||
|
||||
export const name = 'llm-replay'
|
||||
@@ -314,6 +383,8 @@ export interface Config {
|
||||
* a nested-agent scenario; absent/empty for a single-session scenario.
|
||||
*/
|
||||
childFiles?: string[]
|
||||
/** Optional replay-only provider catalog; absent or empty selects catch-all waterfall replay. */
|
||||
providers?: ReplayProviderConfig[]
|
||||
}
|
||||
|
||||
export function apply(ctx: Context, config: Config = {}): void {
|
||||
@@ -329,5 +400,6 @@ export function apply(ctx: Context, config: Config = {}): void {
|
||||
file,
|
||||
...overrideFile !== undefined && overrideFile.length > 0 ? { overrideFile } : {},
|
||||
...childFiles.length > 0 ? { childFiles } : {},
|
||||
...config.providers !== undefined ? { providers: config.providers } : {},
|
||||
})
|
||||
}
|
||||
|
||||
@@ -198,7 +198,7 @@ describe('loadReplayScript', () => {
|
||||
})
|
||||
})
|
||||
|
||||
describe('installLlmReplay (through the real waterfall)', () => {
|
||||
describe('installLlmReplay (through the real LlmService)', () => {
|
||||
function writeLog(...calls: StreamChunk[][]): void {
|
||||
let seq = 1
|
||||
const events: SessionEvent[] = []
|
||||
@@ -217,6 +217,40 @@ describe('installLlmReplay (through the real waterfall)', () => {
|
||||
expect(await drain(ctx.llm.stream({ provider: 'm', model: 'm', messages: [] }))).toEqual(TEXT_CHUNKS)
|
||||
})
|
||||
|
||||
it('registers a replay-only provider catalog when configured', async () => {
|
||||
writeLog(TEXT_CHUNKS)
|
||||
const ctx = new Context()
|
||||
await ctx.plugin(LlmService)
|
||||
const dispose = installLlmReplay(ctx, {
|
||||
file,
|
||||
providers: [
|
||||
{
|
||||
id: 'deepseek',
|
||||
name: 'DeepSeek',
|
||||
models: [
|
||||
{ id: 'flash' },
|
||||
{ id: 'pro', name: 'Pro', description: 'Larger model' },
|
||||
],
|
||||
},
|
||||
{ id: 'empty' },
|
||||
],
|
||||
})
|
||||
|
||||
expect(ctx.llm.listProviders()).toEqual([
|
||||
{ id: 'deepseek', name: 'DeepSeek' },
|
||||
{ id: 'empty', name: 'empty' },
|
||||
])
|
||||
await expect(ctx.llm.listModels('deepseek')).resolves.toEqual([
|
||||
{ provider: 'deepseek', id: 'flash', name: 'flash' },
|
||||
{ provider: 'deepseek', id: 'pro', name: 'Pro', description: 'Larger model' },
|
||||
])
|
||||
await expect(ctx.llm.listModels('empty')).resolves.toEqual([])
|
||||
expect(await drain(ctx.llm.stream({ provider: 'deepseek', model: 'pro', messages: [] }))).toEqual(TEXT_CHUNKS)
|
||||
|
||||
dispose()
|
||||
expect(ctx.llm.listProviders()).toEqual([])
|
||||
})
|
||||
|
||||
it('serves the Nth call the Nth derived entry (positional)', async () => {
|
||||
const second: StreamChunk[] = [
|
||||
{ type: 'block-start', index: 0, blockType: 'text' },
|
||||
@@ -574,11 +608,12 @@ describe('apply (the plugin entry)', () => {
|
||||
expect(inject).toEqual(['llm'])
|
||||
})
|
||||
|
||||
it('installs replay from an explicit config.file', async () => {
|
||||
it('installs replay and its catalog from explicit config', async () => {
|
||||
writeFileSync(file, sessionJsonl(TEXT_CHUNKS.map((c, i) => chunkEvent(i + 1, 1, 1, c))), 'utf8')
|
||||
const ctx = new Context()
|
||||
await ctx.plugin(LlmService)
|
||||
apply(ctx, { file })
|
||||
apply(ctx, { file, providers: [{ id: 'm', models: [{ id: 'm' }] }] })
|
||||
expect(ctx.llm.listProviders()).toEqual([{ id: 'm', name: 'm' }])
|
||||
expect(await drain(ctx.llm.stream({ provider: 'm', model: 'm', messages: [] }))).toEqual(TEXT_CHUNKS)
|
||||
})
|
||||
|
||||
|
||||
Reference in New Issue
Block a user