Merge remote-tracking branch 'origin/master' into codex/skill-system
This commit is contained in:
Yichen Jiang
471
packages/support/acp-snapshot/src/harness.ts
Normal file
471
packages/support/acp-snapshot/src/harness.ts
Normal file
@@ -0,0 +1,471 @@
|
||||
/**
|
||||
* Shared subprocess harness for ACP snapshot suites. A library module driven by
|
||||
* the suite factory in ./suite.ts (and directly by harness-level specs); each
|
||||
* example's `*.snapshot.ts` names its own agent-under-test paths.
|
||||
*
|
||||
* It boots the REAL agent bin subprocess via the cordis Loader (so the
|
||||
* export-shape bug class stays guarded — see docs/postmortem/0001), drives it
|
||||
* over real ACP JSON-RPC stdio with a deterministic input script, tees raw
|
||||
* stdout (for the golden + a purity check) into an SDK `ClientSideConnection`,
|
||||
* and — in record mode — harvests the persisted session JSONL after a graceful
|
||||
* shutdown flush. The pure normalizers in ./normalize.ts turn the captured
|
||||
* stdout frames and the session-log events into stable, snapshot-able text.
|
||||
*
|
||||
* See docs/rfc/implemented/testing/2026-06-19-acp-snapshot-tests.md.
|
||||
*
|
||||
* @module @deepseek-ai/dsh-acp-snapshot/harness
|
||||
*/
|
||||
|
||||
import { spawn, type ChildProcessWithoutNullStreams } from 'node:child_process'
|
||||
import { cp, mkdtemp, readFile, readdir, rm } from 'node:fs/promises'
|
||||
import { existsSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join, delimiter } from 'node:path'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import { Readable, Writable } from 'node:stream'
|
||||
import {
|
||||
ClientSideConnection,
|
||||
ndJsonStream,
|
||||
PROTOCOL_VERSION,
|
||||
type Agent as AcpAgent,
|
||||
type Client,
|
||||
type RequestPermissionRequest,
|
||||
type RequestPermissionResponse,
|
||||
type SessionNotification,
|
||||
} from '@agentclientprotocol/sdk'
|
||||
|
||||
// Resolve tsx's ESM loader to an ABSOLUTE path once: the child runs with its
|
||||
// cwd in a temp dir OUTSIDE the repo, where a bare `--import tsx` would not
|
||||
// resolve from node_modules. import.meta.resolve gives this package's tsx
|
||||
// regardless of the child cwd.
|
||||
const tsxLoader = fileURLToPath(import.meta.resolve('tsx'))
|
||||
|
||||
/**
|
||||
* The agent composition a scenario runs against: which bin to boot and which
|
||||
* leaf config it loads. All paths are ABSOLUTE — the subprocess cwd is a temp
|
||||
* dir outside the repo, so relative resolution would miss; a suite resolves
|
||||
* them from its own `import.meta.url`.
|
||||
*/
|
||||
export interface AgentUnderTest {
|
||||
/** The agent bin entry (e.g. `packages/ui/acp-agent/src/bin.ts`), run unbuilt via tsx. */
|
||||
binScript: string
|
||||
/**
|
||||
* The example's live `cordis.yml`. Under `DSH_SNAPSHOT=replay` the bin swaps
|
||||
* it for the sibling `cordis.snapshot.yml` (the keyless replay overlay), so
|
||||
* one path serves both modes.
|
||||
*/
|
||||
configPath: string
|
||||
/**
|
||||
* The repo-root tsconfig whose `paths` map resolves the unbuilt workspace
|
||||
* imports. Passed to the child as `TSX_TSCONFIG_PATH`: tsx finds a tsconfig
|
||||
* by searching UP from the child's cwd — a temp dir outside the repo — so
|
||||
* without the explicit pin the dsh-* imports fail before the bin writes a
|
||||
* byte.
|
||||
*/
|
||||
tsconfigPath: string
|
||||
}
|
||||
|
||||
/**
|
||||
* One step of a scenario's deterministic input script (`input.json`). The
|
||||
* harness interprets these in order. `newSession` captures the server-issued
|
||||
* (random) session id into a `{{sessionId}}` variable that later steps
|
||||
* reference, since a committed file cannot know the id in advance.
|
||||
*
|
||||
* `promptAndCancel` sends a prompt WITHOUT awaiting its response, waits until
|
||||
* the client observes the first streamed `agent_message_chunk` (so the emitted
|
||||
* frames deterministically precede the cancellation), then cancels the turn —
|
||||
* the only way to exercise a cancel deterministically (a plain `prompt` step
|
||||
* awaits the response, which a cancel/hang scenario would block on forever).
|
||||
*/
|
||||
export type InputStep =
|
||||
| { op: 'initialize'; terminalOutput?: boolean }
|
||||
| { op: 'newSession' }
|
||||
| { op: 'newSessionExpectError'; additionalDirectories?: string[] }
|
||||
| { op: 'prompt'; text: string }
|
||||
| { op: 'promptExpectError'; text: string }
|
||||
| { op: 'promptAndCancel'; text: string }
|
||||
| { op: 'cancel' }
|
||||
|
||||
/** A scenario's `input.json`: an ordered list of input steps. */
|
||||
export interface InputScript {
|
||||
steps: InputStep[]
|
||||
/**
|
||||
* Ordered answers for the agent's `session/request_permission` round-trips,
|
||||
* consumed FIFO — the Nth request gets the Nth answer. Each answer selects
|
||||
* by option KIND: option ids are agent-issued randoms a committed script
|
||||
* cannot know, while kinds are the ACP-stable vocabulary, so the client maps
|
||||
* kind → the offered `optionId` at answer time. A request beyond the queue
|
||||
* (or with no queue at all) is answered `cancelled` — the stub behavior a
|
||||
* scenario without approvals relies on. A scripted kind the request does
|
||||
* not offer REJECTS the run: the scenario scripted an impossible click,
|
||||
* and {@link runScenario} throws once the in-flight step settles (the
|
||||
* agent itself just sees `cancelled`, so it cannot absorb the bug).
|
||||
*/
|
||||
permissionAnswers?: PermissionAnswer[]
|
||||
}
|
||||
|
||||
/** One scripted answer to a permission request: which offered option kind to select. */
|
||||
export interface PermissionAnswer {
|
||||
/** The `PermissionOption.kind` to select (`allow_once`, `reject_always`, …). */
|
||||
kind: 'allow_once' | 'allow_always' | 'reject_once' | 'reject_always'
|
||||
}
|
||||
|
||||
/** One harvested session log plus the identifying facts off its header line. */
|
||||
export interface HarvestedLog {
|
||||
/** The recorded session id (header `id`). */
|
||||
id: string
|
||||
/** Session creation time (header `createdAt`) — the child-ordering key. */
|
||||
createdAt: number
|
||||
/** The parent session id, if this log is a subagent child (header `parentSession`). */
|
||||
parentSession?: string
|
||||
/** The full `.jsonl` file content. */
|
||||
content: string
|
||||
}
|
||||
|
||||
/** The result of running a scenario: raw stdout + the harvested session log(s). */
|
||||
export interface RunResult {
|
||||
/** Raw stdout bytes (decoded utf8), every newline-delimited JSON-RPC frame. */
|
||||
rawStdout: string
|
||||
/** stderr (for diagnostics on failure). */
|
||||
stderr: string
|
||||
/** The session id the server issued (undefined if no session was created). */
|
||||
sessionId?: string
|
||||
/** The temp cwd the session ran in (the bash workspace). */
|
||||
cwd: string
|
||||
/**
|
||||
* Every persisted session log harvested after the run, ordered primary-first:
|
||||
* the top-level (parent) session — the one with no `parentSession` — then each
|
||||
* subagent child by ascending `createdAt`. A single-session scenario harvests
|
||||
* exactly one; a nested-agent scenario harvests the parent plus one per child.
|
||||
*/
|
||||
sessionLogs: HarvestedLog[]
|
||||
}
|
||||
|
||||
/** How to run one scenario: the agent to boot, the mode, and the fixture wiring. */
|
||||
export interface RunOptions {
|
||||
/** The agent composition to boot. */
|
||||
agent: AgentUnderTest
|
||||
/** `replay` (default, keyless) or `record` (real API, harvests the log). */
|
||||
mode: 'replay' | 'record'
|
||||
/** The recorded session JSONL fixture path (replay reads it; record writes near it). */
|
||||
fixtureFile: string
|
||||
/** Optional sidecar override path (replay). */
|
||||
overrideFile?: string
|
||||
/**
|
||||
* Recorded SUBAGENT child-session fixture paths (replay). A nested-agent
|
||||
* scenario ships one per child (`session.1.jsonl`, …); the harness forwards
|
||||
* them to `dsh-llm-replay` via `$DSH_SNAPSHOT_CHILD_FILES` so each child
|
||||
* session replays from its own recorded script. Empty for single-session
|
||||
* scenarios. Ignored in record mode (children are harvested, not replayed).
|
||||
*/
|
||||
childFiles?: string[]
|
||||
/**
|
||||
* Optional `<scenario>/workspace/` directory whose contents are copied into
|
||||
* the temp cwd BEFORE the run — the standard way to seed files the agent
|
||||
* operates on (a file to read, edit, or grep). Absent for scenarios that
|
||||
* start from an empty workspace.
|
||||
*/
|
||||
workspaceDir?: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Run a scenario end-to-end against a freshly-spawned subprocess. Owns the
|
||||
* child and its temp dirs; always tears them down. Returns the captured stdout
|
||||
* and (record mode) the harvested session-log path.
|
||||
*
|
||||
* @param input The scenario's input script (steps + optional permission answers).
|
||||
* @param opts The agent to boot, the mode, and the fixture wiring.
|
||||
* @returns The captured stdout/stderr, session id, temp cwd, and harvested logs.
|
||||
*/
|
||||
export async function runScenario(input: InputScript, opts: RunOptions): Promise<RunResult> {
|
||||
const cwd = await mkdtemp(join(tmpdir(), 'acp-snap-cwd-'))
|
||||
const sessionsRoot = await mkdtemp(join(tmpdir(), 'acp-snap-sessions-'))
|
||||
// Everything past the temp-dir creation runs under a try/finally that always
|
||||
// removes both dirs — so a failure in workspace seeding, spawn, or any step
|
||||
// never leaks them (the "e2e tests own their resources" rule).
|
||||
let child: ChildProcessWithoutNullStreams | undefined
|
||||
let sessionId: string | undefined
|
||||
let sessionLogs: HarvestedLog[] = []
|
||||
const rawBuffers: Buffer[] = []
|
||||
const stderrChunks: string[] = []
|
||||
try {
|
||||
// Seed the workspace if the scenario ships one (a file the agent reads/edits).
|
||||
// Copied into the temp cwd so the agent's bash tools see it; the goldens
|
||||
// normalize the cwd, so the seeded paths stay stable across runs.
|
||||
if (opts.workspaceDir !== undefined && existsSync(opts.workspaceDir)) {
|
||||
await cp(opts.workspaceDir, cwd, { recursive: true })
|
||||
}
|
||||
const env: NodeJS.ProcessEnv = {
|
||||
...process.env,
|
||||
TSX_TSCONFIG_PATH: opts.agent.tsconfigPath,
|
||||
DSH_SNAPSHOT: opts.mode,
|
||||
DSH_SNAPSHOT_FILE: opts.fixtureFile,
|
||||
DSH_SNAPSHOT_SESSIONS_ROOT: sessionsRoot,
|
||||
DSH_HOME: join(cwd, '.dsh'),
|
||||
DSH_AGENTS_HOME: join(cwd, '.agents'),
|
||||
...opts.overrideFile !== undefined ? { DSH_SNAPSHOT_OVERRIDE: opts.overrideFile } : {},
|
||||
...opts.childFiles !== undefined && opts.childFiles.length > 0
|
||||
? { DSH_SNAPSHOT_CHILD_FILES: opts.childFiles.join(delimiter) }
|
||||
: {},
|
||||
}
|
||||
|
||||
child = spawn(
|
||||
process.execPath,
|
||||
['--import', tsxLoader, opts.agent.binScript, opts.agent.configPath],
|
||||
{ cwd, env, stdio: ['pipe', 'pipe', 'pipe'] },
|
||||
)
|
||||
|
||||
child.stderr.setEncoding('utf8')
|
||||
child.stderr.on('data', (c: string) => stderrChunks.push(c))
|
||||
|
||||
// Tee raw stdout: accumulate the bytes for the golden + purity check, and ALSO
|
||||
// feed the same bytes to the SDK client through a passthrough. Buffer the raw
|
||||
// bytes (not per-chunk utf8 strings) and decode once at the end, so a
|
||||
// multibyte sequence split across two 'data' events can't corrupt the golden.
|
||||
const passthrough = new Readable({ read() {} })
|
||||
child.stdout.on('data', (buf: Buffer) => {
|
||||
rawBuffers.push(buf)
|
||||
passthrough.push(buf)
|
||||
})
|
||||
child.stdout.on('end', () => passthrough.push(null))
|
||||
|
||||
const stream = ndJsonStream(
|
||||
Writable.toWeb(child.stdin) as WritableStream<Uint8Array>,
|
||||
Readable.toWeb(passthrough) as ReadableStream<Uint8Array>,
|
||||
)
|
||||
// Watcher so a step can block until the client OBSERVES a particular
|
||||
// session/update — used by promptAndCancel to pin frame order (send cancel
|
||||
// only after the streamed agent_message_chunk has arrived, so those frames
|
||||
// deterministically precede the cancelled prompt response).
|
||||
const updateWaiters: { match: (u: SessionNotification['update']) => boolean; resolve: () => void }[] = []
|
||||
const waitForUpdate = (match: (u: SessionNotification['update']) => boolean): Promise<void> =>
|
||||
new Promise<void>(resolve => updateWaiters.push({ match, resolve }))
|
||||
|
||||
// Permission answers are consumed FIFO across the whole run; exhaustion
|
||||
// falls back to `cancelled` so approval-free scenarios keep the plain stub.
|
||||
const permissionQueue = [...input.permissionAnswers ?? []]
|
||||
// A scenario bug detected inside a client callback (a scripted permission
|
||||
// kind the agent never offered). It cannot fail the run from in there: a
|
||||
// callback throw only becomes a JSON-RPC error RESPONSE to the agent, and
|
||||
// a tolerant agent treats that as a denial and carries on — the run (or
|
||||
// worse, a record) would absorb the impossible click silently. So the
|
||||
// callback answers `cancelled` (a well-defined path for the agent),
|
||||
// captures the error here, and the step loop fails the run on it.
|
||||
let scriptError: Error | undefined
|
||||
const makeClient = (_agent: AcpAgent): Client => ({
|
||||
sessionUpdate(params: SessionNotification): Promise<void> {
|
||||
for (let i = updateWaiters.length - 1; i >= 0; i--) {
|
||||
const waiter = updateWaiters[i]
|
||||
// The index is always in-bounds (i only decreases; splice removes at
|
||||
// i, so lower entries stay valid); the guard satisfies
|
||||
// noUncheckedIndexedAccess.
|
||||
/* v8 ignore next 1 -- unreachable in-bounds guard, see above */
|
||||
if (waiter === undefined) continue
|
||||
if (waiter.match(params.update)) {
|
||||
updateWaiters.splice(i, 1)
|
||||
waiter.resolve()
|
||||
}
|
||||
}
|
||||
return Promise.resolve()
|
||||
},
|
||||
requestPermission(params: RequestPermissionRequest): Promise<RequestPermissionResponse> {
|
||||
const answer = permissionQueue.shift()
|
||||
if (answer === undefined) return Promise.resolve({ outcome: { outcome: 'cancelled' } })
|
||||
const option = params.options.find(o => o.kind === answer.kind)
|
||||
if (option === undefined) {
|
||||
// The scenario scripted a click the agent never offered — a scenario
|
||||
// bug. Captured (last one wins; same bug class either way) and
|
||||
// answered `cancelled`; the step loop rejects the run on it.
|
||||
scriptError = new Error(
|
||||
`snapshot-harness: scripted permission answer ${answer.kind} not among `
|
||||
+ `the offered options [${params.options.map(o => o.kind).join(', ')}]`,
|
||||
)
|
||||
return Promise.resolve({ outcome: { outcome: 'cancelled' } })
|
||||
}
|
||||
return Promise.resolve({ outcome: { outcome: 'selected', optionId: option.optionId } })
|
||||
},
|
||||
})
|
||||
const client = new ClientSideConnection(makeClient, stream)
|
||||
|
||||
for (const step of input.steps) {
|
||||
await runStep(client, step, cwd, waitForUpdate, () => sessionId, (id) => { sessionId = id })
|
||||
// A permission exchange happens while a step's request is in flight, so
|
||||
// by the time the step settles any script bug it exposed is captured —
|
||||
// fail the run HERE, as a harness error, rather than hoping the agent's
|
||||
// reaction to the answer perturbs the transcript.
|
||||
if (scriptError !== undefined) throw scriptError
|
||||
}
|
||||
// Done driving: close stdin so the server disposes gracefully (flushing
|
||||
// persistence) and exits. Then await exit so the harvested log is complete.
|
||||
child.stdin.end()
|
||||
await waitForExit(child)
|
||||
// Harvest EVERY persisted log (parent + any subagent children) while the
|
||||
// temp dirs still exist, ordered primary-first.
|
||||
sessionLogs = await harvestSessionLogs(sessionsRoot)
|
||||
} finally {
|
||||
// Failure-safe teardown: kill a still-running child and drop the temp dirs
|
||||
// even if seeding/spawn/a step/harvest threw, so a flaky run never leaks a
|
||||
// process or dir. `child` is undefined only if spawn itself threw.
|
||||
if (child !== undefined && child.exitCode === null && child.signalCode === null) {
|
||||
child.kill('SIGKILL')
|
||||
await waitForExit(child)
|
||||
}
|
||||
await rm(cwd, { recursive: true, force: true })
|
||||
await rm(sessionsRoot, { recursive: true, force: true })
|
||||
}
|
||||
|
||||
return {
|
||||
rawStdout: Buffer.concat(rawBuffers).toString('utf8'),
|
||||
stderr: stderrChunks.join(''),
|
||||
cwd,
|
||||
...sessionId !== undefined ? { sessionId } : {},
|
||||
sessionLogs,
|
||||
}
|
||||
}
|
||||
|
||||
/** Drive one input step over the client connection. */
|
||||
async function runStep(
|
||||
client: ClientSideConnection,
|
||||
step: InputStep,
|
||||
cwd: string,
|
||||
waitForUpdate: (match: (u: SessionNotification['update']) => boolean) => Promise<void>,
|
||||
getSessionId: () => string | undefined,
|
||||
setSessionId: (id: string) => void,
|
||||
): Promise<void> {
|
||||
switch (step.op) {
|
||||
case 'initialize':
|
||||
await client.initialize({
|
||||
protocolVersion: PROTOCOL_VERSION,
|
||||
clientCapabilities: step.terminalOutput === true ? { _meta: { terminal_output: true } } : {},
|
||||
})
|
||||
return
|
||||
case 'newSession': {
|
||||
const { sessionId } = await client.newSession({ cwd, mcpServers: [] })
|
||||
setSessionId(sessionId)
|
||||
return
|
||||
}
|
||||
case 'newSessionExpectError': {
|
||||
// The bridge rejects a session/new that widens the workspace scope
|
||||
// (non-empty additionalDirectories / mcpServers — unimplemented). The SDK
|
||||
// surfaces that as a rejected RPC; swallow it so the run completes and the
|
||||
// error frame is captured in the transcript.
|
||||
await client.newSession({
|
||||
cwd,
|
||||
mcpServers: [],
|
||||
...step.additionalDirectories !== undefined ? { additionalDirectories: step.additionalDirectories } : {},
|
||||
}).then(
|
||||
() => { throw new Error('snapshot-harness: expected session/new to be rejected but it succeeded') },
|
||||
() => { /* expected: the bridge rejected the unsupported workspace scope */ },
|
||||
)
|
||||
return
|
||||
}
|
||||
case 'prompt': {
|
||||
const sessionId = getSessionId()
|
||||
if (sessionId === undefined) throw new Error('snapshot-harness: prompt before newSession')
|
||||
await client.prompt({ sessionId, prompt: [{ type: 'text', text: step.text }] })
|
||||
return
|
||||
}
|
||||
case 'promptExpectError': {
|
||||
const sessionId = getSessionId()
|
||||
if (sessionId === undefined) throw new Error('snapshot-harness: promptExpectError before newSession')
|
||||
// The model fails this turn (a recorded provider error), so the bridge
|
||||
// answers the prompt with a JSON-RPC error and the SDK rejects. That
|
||||
// rejection IS the expected editor experience — swallow it so the run
|
||||
// completes and the stdout transcript (the error frame) is captured.
|
||||
await client.prompt({ sessionId, prompt: [{ type: 'text', text: step.text }] })
|
||||
.then(() => { throw new Error('snapshot-harness: expected the prompt to fail but it succeeded') },
|
||||
() => { /* expected: the turn failed and the bridge returned an error */ })
|
||||
return
|
||||
}
|
||||
case 'promptAndCancel': {
|
||||
const sessionId = getSessionId()
|
||||
if (sessionId === undefined) throw new Error('snapshot-harness: promptAndCancel before newSession')
|
||||
// Dispatch the prompt WITHOUT awaiting (a hang fixture never resolves on
|
||||
// its own). To pin frame order deterministically, wait until the client
|
||||
// has OBSERVED the hang's streamed agent_message_chunk before cancelling —
|
||||
// so those update frames always precede the cancelled prompt response in
|
||||
// the transcript (without this, the late chunk and the response race).
|
||||
// Then cancel and await the prompt, which the bridge settles as
|
||||
// `cancelled` once the abort propagates.
|
||||
const promptDone = client.prompt({ sessionId, prompt: [{ type: 'text', text: step.text }] })
|
||||
await waitForUpdate(u => u.sessionUpdate === 'agent_message_chunk')
|
||||
await client.cancel({ sessionId })
|
||||
await promptDone
|
||||
return
|
||||
}
|
||||
case 'cancel': {
|
||||
const sessionId = getSessionId()
|
||||
if (sessionId === undefined) throw new Error('snapshot-harness: cancel before newSession')
|
||||
await client.cancel({ sessionId })
|
||||
return
|
||||
}
|
||||
default:
|
||||
throw new Error(`snapshot-harness: unknown input op ${JSON.stringify(step)}`)
|
||||
}
|
||||
}
|
||||
|
||||
/** Resolve once the child process exits (any code/signal). */
|
||||
function waitForExit(child: ChildProcessWithoutNullStreams): Promise<void> {
|
||||
// Race guard: both call sites run within one synchronous frame of
|
||||
// stdin.end()/kill(), so the exit event cannot have been delivered yet;
|
||||
// kept for any future caller that awaits in between.
|
||||
/* v8 ignore next 1 -- unreachable race guard, see above */
|
||||
if (child.exitCode !== null || child.signalCode !== null) return Promise.resolve()
|
||||
return new Promise<void>(resolve => child.once('exit', () => { resolve() }))
|
||||
}
|
||||
|
||||
/**
|
||||
* Harvest EVERY persisted `.jsonl` session log under a sessions root, parse each
|
||||
* header line, and return them ordered primary-first: the top-level session (no
|
||||
* `parentSession`) leads, then each subagent child by ascending `createdAt`.
|
||||
*
|
||||
* The JSONL backend lays sessions out as `<root>/<cwd-bucket>/<encoded-id>.jsonl`
|
||||
* (one bucket per cwd), so a parent and its same-cwd in-process child land in
|
||||
* the SAME bucket — collecting all files across all buckets catches both (a
|
||||
* first-match short-circuit would silently drop the child). Returns `[]` if no
|
||||
* log was produced (a no-session scenario).
|
||||
*/
|
||||
async function harvestSessionLogs(root: string): Promise<HarvestedLog[]> {
|
||||
let cwdDirs: string[]
|
||||
try {
|
||||
cwdDirs = await readdir(root)
|
||||
} catch {
|
||||
return []
|
||||
}
|
||||
const logs: HarvestedLog[] = []
|
||||
for (const dir of cwdDirs) {
|
||||
const sub = join(root, dir)
|
||||
let files: string[]
|
||||
try {
|
||||
files = await readdir(sub)
|
||||
} catch {
|
||||
continue
|
||||
}
|
||||
for (const f of files) {
|
||||
if (!f.endsWith('.jsonl')) continue
|
||||
const content = await readFile(join(sub, f), 'utf8')
|
||||
const firstLine = content.split('\n').find(line => line.trim().length > 0) ?? '{}'
|
||||
const header = JSON.parse(firstLine) as { id?: unknown; createdAt?: unknown; parentSession?: unknown }
|
||||
logs.push({
|
||||
id: typeof header.id === 'string' ? header.id : '',
|
||||
createdAt: typeof header.createdAt === 'number' ? header.createdAt : 0,
|
||||
...typeof header.parentSession === 'string' ? { parentSession: header.parentSession } : {},
|
||||
content,
|
||||
})
|
||||
}
|
||||
}
|
||||
// Primary (no parentSession) first, then children by ascending createdAt. A
|
||||
// scenario has exactly one top-level session. In the synchronous cut sibling
|
||||
// children are created strictly sequentially, so their createdAt values are
|
||||
// strictly ordered; the recordedId tiebreak only keeps a degenerate
|
||||
// same-millisecond collision (unreachable here) deterministic. This harvest
|
||||
// order must match the replay load order in dsh-llm-replay's loadSessionScripts
|
||||
// so session.<n>.jsonl maps to the same child on record and replay — replay
|
||||
// re-sorts childFiles by the same key, so the two stay consistent.
|
||||
logs.sort((a, b) => {
|
||||
const ap = a.parentSession === undefined ? 0 : 1
|
||||
const bp = b.parentSession === undefined ? 0 : 1
|
||||
return ap - bp || a.createdAt - b.createdAt || a.id.localeCompare(b.id)
|
||||
})
|
||||
return logs
|
||||
}
|
||||
38
packages/support/acp-snapshot/src/index.ts
Normal file
38
packages/support/acp-snapshot/src/index.ts
Normal file
@@ -0,0 +1,38 @@
|
||||
/**
|
||||
* ACP snapshot suite kit — the shared machinery behind the keyless snapshot
|
||||
* tier (`pnpm run test:snapshot`). Three layers, composable per example:
|
||||
* the subprocess scenario harness ({@link runScenario}), the pure golden
|
||||
* normalizers ({@link normalizeStdout} / {@link normalizeSessionLog} /
|
||||
* {@link scrubRequestHeaders}), and the suite factory
|
||||
* ({@link defineAcpSnapshotSuite}) that registers a scenario table as a full
|
||||
* describe/it tree. An example's `*.snapshot.ts` supplies only its
|
||||
* {@link AgentUnderTest} paths, its snapshots directory, and its
|
||||
* {@link Scenario} table.
|
||||
*
|
||||
* NOTE: ./suite.ts imports vitest, so this package is importable only inside a
|
||||
* vitest run — a support-tier constraint stated in the README.
|
||||
*
|
||||
* @module @deepseek-ai/dsh-acp-snapshot
|
||||
*/
|
||||
|
||||
export {
|
||||
runScenario,
|
||||
type AgentUnderTest,
|
||||
type HarvestedLog,
|
||||
type InputScript,
|
||||
type InputStep,
|
||||
type PermissionAnswer,
|
||||
type RunOptions,
|
||||
type RunResult,
|
||||
} from './harness.ts'
|
||||
export {
|
||||
normalizeSessionLog,
|
||||
normalizeStdout,
|
||||
scrubRequestHeaders,
|
||||
type NormalizeContext,
|
||||
} from './normalize.ts'
|
||||
export {
|
||||
defineAcpSnapshotSuite,
|
||||
type Scenario,
|
||||
type SnapshotSuiteOptions,
|
||||
} from './suite.ts'
|
||||
213
packages/support/acp-snapshot/src/normalize.ts
Normal file
213
packages/support/acp-snapshot/src/normalize.ts
Normal file
@@ -0,0 +1,213 @@
|
||||
/**
|
||||
* Pure normalizers for the ACP snapshot goldens. They replace the
|
||||
* non-deterministic values in the two captured surfaces — the stdout JSON-RPC
|
||||
* transcript and the persisted session JSONL — with stable tokens, so a golden
|
||||
* compare reflects behavior, not run-to-run noise. Kept dependency-free and
|
||||
* side-effect-free so they unit-test trivially.
|
||||
*
|
||||
* Scrubbed: `randomUUID()` session ids → `{{sessionId}}`; the temp `mkdtemp`
|
||||
* cwd → `{{cwd}}` (it appears in terminal-card `_meta` and the log header);
|
||||
* JSON-RPC request `id` → a stable per-transcript sequence; the log's per-event
|
||||
* `time` (epoch ms) and header `createdAt` → 0; a `hook/result` event's
|
||||
* `durationMs` (wall-clock hook runtime) → 0. NOT scrubbed: the log's `seq`
|
||||
* (deterministic — `seq = log.length`, part of the event-log contract).
|
||||
*
|
||||
* A separate, composable normalizer — {@link scrubRequestHeaders} — replaces
|
||||
* the bulky request-header CONTENT (the composed system prompt, the tool
|
||||
* schema list, and the session prefix) with
|
||||
* `{{system}}`/`{{tools}}`/`{{messagePrefix}}` tokens. It is deliberately NOT
|
||||
* folded into {@link normalizeSessionLog}: each suite's one header-pinning
|
||||
* scenario compares that content verbatim, every other scenario composes the
|
||||
* scrub in (the `pinsHeader` flag on the scenario table, consumed by the suite
|
||||
* factory in ./suite.ts; see the pinned-header RFC,
|
||||
* docs/rfc/implemented/testing/2026-07-06-pin-request-header-content-in-one-scenario.md).
|
||||
*
|
||||
* See docs/rfc/implemented/testing/2026-06-19-acp-snapshot-tests.md.
|
||||
*
|
||||
* @module @deepseek-ai/dsh-acp-snapshot/normalize
|
||||
*/
|
||||
|
||||
const SESSION_ID = '{{sessionId}}'
|
||||
const CWD = '{{cwd}}'
|
||||
const SYSTEM = '{{system}}'
|
||||
const TOOLS = '{{tools}}'
|
||||
const MESSAGE_PREFIX = '{{messagePrefix}}'
|
||||
|
||||
/** A UUID v4 string, the shape `randomUUID()` produces for session ids. */
|
||||
const UUID_RE = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/gi
|
||||
|
||||
/** Inputs the normalizers need to recognize a run's volatile values. */
|
||||
export interface NormalizeContext {
|
||||
/** The session id(s) the run issued — replaced with `{{sessionId}}`. */
|
||||
sessionIds: string[]
|
||||
/** The temp cwd the run used — replaced with `{{cwd}}`. */
|
||||
cwd: string
|
||||
}
|
||||
|
||||
/** Replace cwd, session ids, and any stray UUID with stable tokens in a string. */
|
||||
function scrubString(value: string, ctx: NormalizeContext): string {
|
||||
let out = value
|
||||
// cwd first (longest, most specific), then explicit session ids, then any
|
||||
// residual UUID (covers ids that appear in places we didn't enumerate).
|
||||
out = out.split(ctx.cwd).join(CWD)
|
||||
for (const id of ctx.sessionIds) out = out.split(id).join(SESSION_ID)
|
||||
out = out.replace(UUID_RE, SESSION_ID)
|
||||
return out
|
||||
}
|
||||
|
||||
/** Recursively scrub a parsed JSON value (strings replaced; structure kept). */
|
||||
function scrubValue(value: unknown, ctx: NormalizeContext): unknown {
|
||||
if (typeof value === 'string') return scrubString(value, ctx)
|
||||
if (Array.isArray(value)) return value.map(v => scrubValue(v, ctx))
|
||||
if (value !== null && typeof value === 'object') {
|
||||
const out: Record<string, unknown> = {}
|
||||
for (const [k, v] of Object.entries(value)) out[k] = scrubValue(v, ctx)
|
||||
return out
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize a raw stdout transcript (newline-delimited JSON-RPC frames) into a
|
||||
* stable golden in the SAME shape as the wire: one compact JSON frame per line
|
||||
* (NDJSON), with the JSON-RPC `id` rewritten to a per-transcript sequence
|
||||
* (1, 2, 3, …) and all volatile strings scrubbed. Throws if any non-empty line
|
||||
* is not valid JSON — that doubles as the stdout-purity check (no logger leaked
|
||||
* onto the protocol).
|
||||
*
|
||||
* @param rawStdout The captured stdout bytes, decoded utf8.
|
||||
* @param ctx The run's volatile values to scrub.
|
||||
* @returns The normalized NDJSON transcript, one frame per line.
|
||||
*/
|
||||
export function normalizeStdout(rawStdout: string, ctx: NormalizeContext): string {
|
||||
const lines = rawStdout.split('\n').filter(line => line.trim().length > 0)
|
||||
// Map each distinct JSON-RPC id (request/response correlate by id) to a stable
|
||||
// sequence number, in first-seen order, so id churn doesn't perturb the golden.
|
||||
const idSeq = new Map<string, number>()
|
||||
const stableId = (id: unknown): number => {
|
||||
const key = JSON.stringify(id)
|
||||
let n = idSeq.get(key)
|
||||
if (n === undefined) { n = idSeq.size + 1; idSeq.set(key, n) }
|
||||
return n
|
||||
}
|
||||
const frames = lines.map((line) => {
|
||||
const frame = JSON.parse(line) as Record<string, unknown>
|
||||
if ('id' in frame && frame.id !== undefined && frame.id !== null) {
|
||||
frame.id = stableId(frame.id)
|
||||
}
|
||||
return scrubValue(frame, ctx) as Record<string, unknown>
|
||||
})
|
||||
return frames.map(f => JSON.stringify(f)).join('\n') + '\n'
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize a session JSONL log into a stable golden: the header line's
|
||||
* volatile fields (`createdAt`, `id`, `cwd`) and every event's `time` are
|
||||
* zeroed/scrubbed, all volatile strings scrubbed, and `seq` is LEFT INTACT
|
||||
* (deterministic by contract). Output is JSONL in the same shape as the input —
|
||||
* one compact record per line.
|
||||
*
|
||||
* @param rawLog The raw session `.jsonl` content.
|
||||
* @param ctx The run's volatile values to scrub.
|
||||
* @returns The normalized JSONL log, one record per line.
|
||||
*/
|
||||
export function normalizeSessionLog(rawLog: string, ctx: NormalizeContext): string {
|
||||
const lines = rawLog.split('\n').filter(line => line.trim().length > 0)
|
||||
const records = lines.map((line) => {
|
||||
const record = JSON.parse(line) as Record<string, unknown>
|
||||
// Header line: { type: 'session', createdAt, id, cwd, … }.
|
||||
if (record.type === 'session') {
|
||||
if ('createdAt' in record) record.createdAt = 0
|
||||
} else if ('time' in record) {
|
||||
// Event line: zero the epoch-ms timestamp; keep seq (deterministic).
|
||||
record.time = 0
|
||||
// A hook/result carries the hook's wall-clock runtime (`data.durationMs`),
|
||||
// which is run-to-run noise like `time` — zero it so the golden reflects
|
||||
// the hook's decision/exit, not how long the shell took.
|
||||
if (record.type === 'hook/result' && record.data !== null && typeof record.data === 'object') {
|
||||
const data = record.data as Record<string, unknown>
|
||||
if ('durationMs' in data) data.durationMs = 0
|
||||
}
|
||||
}
|
||||
return scrubValue(record, ctx) as Record<string, unknown>
|
||||
})
|
||||
return records.map(r => JSON.stringify(r)).join('\n') + '\n'
|
||||
}
|
||||
|
||||
/**
|
||||
* Replace request-header CONTENT in a session JSONL with stable tokens,
|
||||
* keeping its structure: a `request/header` event's `data.header.system` →
|
||||
* `{{system}}`, `data.header.tools` → `{{tools}}`, and
|
||||
* `data.header.messagePrefix` → one `{{messagePrefix}}` token per message
|
||||
* (the session prefix is model-visible bulk — an AGENTS digest, a skills
|
||||
* catalog — so its COUNT stays a structural fact while its text never lands
|
||||
* in a fixture); a
|
||||
* `request/header-delta` event keeps every structural fact — the system
|
||||
* delta's `keepStart`/`keepEnd` line positions and inserted-line COUNT (one
|
||||
* `{{system}}` token per inserted line), the tools delta's
|
||||
* added/removed/changed tool NAMES, the prefix replacement's message COUNT —
|
||||
* and tokenizes only the bulk (prompt
|
||||
* text; each added/changed schema's fields other than `name` → `{{tools}}`;
|
||||
* each replacement prefix message → `{{messagePrefix}}`),
|
||||
* so two different deltas still compare different.
|
||||
* Absent fields stay absent — WHETHER a header carried a system prompt,
|
||||
* tools, or a prefix is behavior and stays visible; `config` and `reason`
|
||||
* are small and
|
||||
* stable, so they stay verbatim (a model swap churns every fixture by design
|
||||
* — it invalidates the recorded responses; a prompt/schema edit churns none —
|
||||
* replay never reads this content, see dsh-llm-replay).
|
||||
*
|
||||
* Only lines with something to scrub are re-serialized; every other line
|
||||
* passes through byte-for-byte, so the transform is idempotent and applying
|
||||
* it to an already-scrubbed fixture is a no-op — the on-disk-fixtures guard
|
||||
* in ./suite.ts relies on exactly that.
|
||||
*
|
||||
* @param rawLog The raw session `.jsonl` content.
|
||||
* @returns The JSONL with header content tokenized, other lines byte-identical.
|
||||
*/
|
||||
export function scrubRequestHeaders(rawLog: string): string {
|
||||
const lines = rawLog.split('\n')
|
||||
const out = lines.map((line) => {
|
||||
if (line.trim().length === 0) return line
|
||||
const record = JSON.parse(line) as Record<string, unknown>
|
||||
const data = record.data as Record<string, unknown> | null | undefined
|
||||
if (data === null || typeof data !== 'object') return line
|
||||
if (record.type === 'request/header') {
|
||||
const header = data.header as Record<string, unknown> | null | undefined
|
||||
if (header === null || typeof header !== 'object') return line
|
||||
if (!('system' in header) && !('tools' in header) && !('messagePrefix' in header)) return line
|
||||
if ('system' in header) header.system = SYSTEM
|
||||
if ('tools' in header) header.tools = TOOLS
|
||||
if (Array.isArray(header.messagePrefix)) header.messagePrefix = header.messagePrefix.map(() => MESSAGE_PREFIX)
|
||||
return JSON.stringify(record)
|
||||
}
|
||||
if (record.type === 'request/header-delta') {
|
||||
let touched = false
|
||||
const system = data.system as Record<string, unknown> | null | undefined
|
||||
if (system !== null && typeof system === 'object' && Array.isArray(system.insert)) {
|
||||
system.insert = system.insert.map(() => SYSTEM)
|
||||
touched = true
|
||||
}
|
||||
const tools = data.tools as Record<string, unknown> | null | undefined
|
||||
if (tools !== null && typeof tools === 'object') {
|
||||
if (Array.isArray(tools.added)) { tools.added = tools.added.map(scrubToolSchema); touched = true }
|
||||
if (Array.isArray(tools.changed)) { tools.changed = tools.changed.map(scrubToolSchema); touched = true }
|
||||
}
|
||||
if (Array.isArray(data.messagePrefix)) {
|
||||
data.messagePrefix = data.messagePrefix.map(() => MESSAGE_PREFIX)
|
||||
touched = true
|
||||
}
|
||||
return touched ? JSON.stringify(record) : line
|
||||
}
|
||||
return line
|
||||
})
|
||||
return out.join('\n')
|
||||
}
|
||||
|
||||
/** Tokenize one tool schema's bulk (description, parameters, anything else), keeping its identifying `name`. */
|
||||
function scrubToolSchema(tool: unknown): unknown {
|
||||
if (tool === null || typeof tool !== 'object' || Array.isArray(tool)) return tool
|
||||
const out: Record<string, unknown> = {}
|
||||
for (const [k, v] of Object.entries(tool)) out[k] = k === 'name' ? v : TOOLS
|
||||
return out
|
||||
}
|
||||
393
packages/support/acp-snapshot/src/suite.ts
Normal file
393
packages/support/acp-snapshot/src/suite.ts
Normal file
@@ -0,0 +1,393 @@
|
||||
/**
|
||||
* The ACP snapshot suite factory (REPLAY by default, keyless). A suite is a
|
||||
* scenario table plus a snapshots directory: each scenario under
|
||||
* `<snapshotsDir>/<name>/` ships an `input.json` (the client stdin script) and
|
||||
* a `session.jsonl` fixture; replay boots the real agent subprocess
|
||||
* (./harness.ts), drives it, and diffs the normalized stdout transcript
|
||||
* against the committed `stdout.golden.jsonl`. For model scenarios it ALSO
|
||||
* checks the re-persisted session log — against the `session.jsonl` fixture
|
||||
* itself, not a separate golden: the fixture doubles as the replay source
|
||||
* (recorded scenarios) and the expected produced log (both sides normalized
|
||||
* before comparing).
|
||||
*
|
||||
* Request-header content (the composed system prompt + tool schemas riding on
|
||||
* `request/header` events) is pinned by exactly ONE scenario per suite — the
|
||||
* one with `pinsHeader` — and scrubbed to `{{system}}`/`{{tools}}` tokens in
|
||||
* every other fixture and compare, so a prompt or tool-schema edit churns one
|
||||
* committed line instead of every fixture. A per-run uniformity guard keeps
|
||||
* the single pin sound: every live header must equal the pinned one, and no
|
||||
* header-delta may appear outside the pinning scenario (see the
|
||||
* pinned-header RFC,
|
||||
* docs/rfc/implemented/testing/2026-07-06-pin-request-header-content-in-one-scenario.md).
|
||||
*
|
||||
* `pnpm run test:snapshot:record` (DSH_SNAPSHOT=record + -u) re-records the
|
||||
* `session.jsonl` fixtures against the real API and refreshes the stdout golden
|
||||
* in one pass; the caller resolves that env into {@link SnapshotSuiteOptions}
|
||||
* (env reading stays at the suite edge, not in this library).
|
||||
*
|
||||
* @module @deepseek-ai/dsh-acp-snapshot/suite
|
||||
*/
|
||||
|
||||
import { readFile, readdir, writeFile } from 'node:fs/promises'
|
||||
import { existsSync } from 'node:fs'
|
||||
import { join } from 'node:path'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { type AgentUnderTest, type HarvestedLog, type InputScript, runScenario } from './harness.ts'
|
||||
import { type NormalizeContext, normalizeSessionLog, normalizeStdout, scrubRequestHeaders } from './normalize.ts'
|
||||
|
||||
/** A snapshot scenario and how its fixtures are produced. */
|
||||
export interface Scenario {
|
||||
name: string
|
||||
/** Whether the scenario drives at least one model turn (so a JSONL golden applies). */
|
||||
hasModelTurn: boolean
|
||||
/**
|
||||
* Whether the run persists a comparable session log to diff against the
|
||||
* `session.jsonl` fixture. Defaults to {@link hasModelTurn} (a model turn
|
||||
* always produces a log worth comparing). Set it independently for a scenario
|
||||
* that produces a non-trivial log WITHOUT a model turn — e.g. a prompt blocked
|
||||
* by a `UserPromptSubmit` hook, which opens a `rejected` turn carrying `hook/*`
|
||||
* events but never calls the model.
|
||||
*/
|
||||
comparesLog?: boolean
|
||||
/**
|
||||
* Whether `test:snapshot:record` regenerates this scenario's `session.jsonl`
|
||||
* from the LIVE API. `recorded` scenarios are model-driven and reproducible;
|
||||
* `authored` scenarios (fixtures hand-written or hand-harvested — e.g. a
|
||||
* provider error or a cancel the live API can't be coaxed into
|
||||
* deterministically, a deterministic hook scenario, or a scripted repetition
|
||||
* a live model won't reproduce) are NEVER re-recorded.
|
||||
*/
|
||||
recorded: boolean
|
||||
/**
|
||||
* Whether replay is driven by a hand-written `replay.override.json` sidecar
|
||||
* (a `ReplayEntry[]` that REPLACES the script derived from `session.jsonl`)
|
||||
* — the throw/hang cases chunks cannot express. The fixture guard requires
|
||||
* the sidecar exactly when this is set: the harness forwards the file purely
|
||||
* on existence, so an unregistered stray sidecar would silently replace the
|
||||
* derived script — the guard fails loud on either mismatch. Defaults to
|
||||
* false (replay derives from the fixture's `assistant/chunk` events).
|
||||
*/
|
||||
overridden?: boolean
|
||||
/**
|
||||
* How many SUBAGENT child sessions this scenario records beyond the top-level
|
||||
* one (0 for a single-session scenario). Each child rides in a sibling fixture
|
||||
* `session.<n>.jsonl` (1-based); replay forwards them to `dsh-llm-replay` so
|
||||
* each child session replays from its own script, and record mode writes the
|
||||
* harvested child logs back to those files. Defaults to 0.
|
||||
*/
|
||||
childSessions?: number
|
||||
/**
|
||||
* Whether THIS scenario's fixtures keep the full request-header content (the
|
||||
* composed system prompt and tool schema list on `request/header` /
|
||||
* `request/header-delta` events) and compare it verbatim. Exactly one
|
||||
* scenario per suite pins it; every other scenario stores and compares that
|
||||
* content as `{{system}}`/`{{tools}}` tokens ({@link scrubRequestHeaders}),
|
||||
* so a system prompt or tool-schema change shows up as ONE committed-fixture
|
||||
* diff, not one per scenario. One pin suffices because header composition is
|
||||
* suite-uniform (parent, spawn child, and fork child all compose the same
|
||||
* prompt-modulo-cwd and the same tools) — and that premise is ASSERTED, not
|
||||
* assumed: every non-pinning run's live headers must equal the pinned
|
||||
* fixture's (normalized), so a session-dependent header (say, a restricted
|
||||
* subagent toolset) fails loud until it gets its own pinning scenario.
|
||||
* Defaults to false.
|
||||
*/
|
||||
pinsHeader?: boolean
|
||||
/**
|
||||
* Whether this scenario intentionally composes a different live request
|
||||
* header from the suite's pinned baseline while still storing scrubbed header
|
||||
* content in its fixture. Use only for cwd/project-sensitive context fixtures
|
||||
* whose header difference is the behavior under test; the fixture guard still
|
||||
* rejects committed full header content unless {@link pinsHeader} is true.
|
||||
* Defaults to false.
|
||||
*/
|
||||
variesHeader?: boolean
|
||||
}
|
||||
|
||||
/** One suite's inputs: the agent to boot, where its fixtures live, and its scenario table. */
|
||||
export interface SnapshotSuiteOptions {
|
||||
/** The agent composition every scenario boots. */
|
||||
agent: AgentUnderTest
|
||||
/** Absolute path of the suite's `snapshots/` directory (one subdir per scenario). */
|
||||
snapshotsDir: string
|
||||
/** The scenario table; exactly one entry must set `pinsHeader`. */
|
||||
scenarios: Scenario[]
|
||||
/**
|
||||
* `replay` (keyless, the default tier) or `record` (live API; re-records the
|
||||
* `recorded` scenarios' fixtures and refreshes the vitest goldens under
|
||||
* `--update`). The caller derives this from `$DSH_SNAPSHOT` — env reading
|
||||
* stays outside this library.
|
||||
*/
|
||||
mode: 'replay' | 'record'
|
||||
}
|
||||
|
||||
/**
|
||||
* The sibling child-fixture paths for a scenario (`session.1.jsonl` …).
|
||||
*
|
||||
* @param dir The scenario's snapshots directory (`<snapshotsDir>/<name>`).
|
||||
* @param childSessions How many subagent child sessions the scenario records.
|
||||
* @returns One path per child, 1-based, in fixture order.
|
||||
*/
|
||||
export function childFixturePaths(dir: string, childSessions: number): string[] {
|
||||
return Array.from({ length: childSessions }, (_, i) => join(dir, `session.${i + 1}.jsonl`))
|
||||
}
|
||||
|
||||
/**
|
||||
* Derive the {@link NormalizeContext} for a `session.jsonl` fixture from its own
|
||||
* header line (`{ type: 'session', id, cwd }`). A committed fixture carries the
|
||||
* session id and cwd of the run that harvested it — different from the live
|
||||
* replay run — so normalizing it against the live run's ctx would leave those
|
||||
* recorded values unscrubbed. Reading them from the header scrubs the fixture's
|
||||
* own id/cwd to the same `{{sessionId}}`/`{{cwd}}` tokens the replay output gets.
|
||||
* An authored fixture whose header is already normalized (`id:'{{sessionId}}'`,
|
||||
* `cwd:'{{cwd}}'`) yields those tokens as the volatile values, so scrubbing them
|
||||
* is an idempotent no-op. A header with no `cwd` falls back to a sentinel that
|
||||
* cannot occur in a log (NOT `''`, which `String.split` would match on every
|
||||
* character boundary and corrupt the output).
|
||||
*
|
||||
* @param fixture The committed `session.jsonl` content.
|
||||
* @returns The fixture's own volatile values, ready for {@link normalizeSessionLog}.
|
||||
*/
|
||||
export function fixtureContext(fixture: string): NormalizeContext {
|
||||
const firstLine = fixture.split('\n').find(line => line.trim().length > 0) ?? '{}'
|
||||
const header = JSON.parse(firstLine) as { id?: unknown; cwd?: unknown }
|
||||
return {
|
||||
sessionIds: typeof header.id === 'string' ? [header.id] : [],
|
||||
cwd: typeof header.cwd === 'string' ? header.cwd : '\0no-cwd\0',
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The `data.header` payload of every `request/header` event in a session
|
||||
* JSONL, in log order, with the log's volatile values scrubbed first
|
||||
* ({@link normalizeSessionLog}) so headers harvested from different runs —
|
||||
* each embedding its own temp cwd in the composed prompt — compare on equal
|
||||
* footing.
|
||||
*
|
||||
* @param rawLog The session `.jsonl` content to extract headers from.
|
||||
* @param ctx The volatile values of the run that produced it.
|
||||
* @returns The normalized `data.header` payloads, in log order.
|
||||
*/
|
||||
export function normalizedHeaders(rawLog: string, ctx: NormalizeContext): unknown[] {
|
||||
return normalizeSessionLog(rawLog, ctx)
|
||||
.split('\n')
|
||||
.filter(line => line.trim().length > 0)
|
||||
.map(line => JSON.parse(line) as { type?: unknown; data?: { header?: unknown } })
|
||||
.filter(record => record.type === 'request/header')
|
||||
.map(record => record.data?.header)
|
||||
}
|
||||
|
||||
/**
|
||||
* Count the `request/header-delta` events in a session JSONL.
|
||||
*
|
||||
* @param rawLog The session `.jsonl` content.
|
||||
* @returns How many `request/header-delta` events the log carries.
|
||||
*/
|
||||
export function headerDeltaCount(rawLog: string): number {
|
||||
return rawLog.split('\n')
|
||||
.filter(line => line.trim().length > 0)
|
||||
.filter(line => (JSON.parse(line) as { type?: unknown }).type === 'request/header-delta')
|
||||
.length
|
||||
}
|
||||
|
||||
/**
|
||||
* Register the suite: one `describe` per scenario (the golden/log compares and
|
||||
* the header-uniformity guard) plus the fixture guard block (no orphan
|
||||
* scenario dirs, required files present, exactly one pin, non-pinning fixtures
|
||||
* header-scrubbed). Must run at vitest collection time — it calls
|
||||
* `describe`/`it`. Throws immediately if no scenario pins the header (the
|
||||
* uniformity guard would have nothing to compare against).
|
||||
*
|
||||
* @param options The agent, snapshots directory, scenario table, and mode.
|
||||
*/
|
||||
export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
|
||||
const { agent, snapshotsDir, scenarios, mode } = options
|
||||
const RECORDING = mode === 'record'
|
||||
|
||||
/** The suite's single header-pinning scenario. Guarded here (and by a meta-test) so the pin cannot silently vanish. */
|
||||
const pinningScenario = scenarios.find(s => s.pinsHeader === true)
|
||||
if (pinningScenario === undefined) throw new Error('acp-snapshot: no scenario pins the request-header content')
|
||||
|
||||
for (const scenario of scenarios) {
|
||||
describe(`snapshot: ${scenario.name}`, () => {
|
||||
// In RECORD mode, only re-run the `recorded` (live-API) scenarios; the
|
||||
// `authored` ones (sidecar-driven errors/cancel) are never re-recorded.
|
||||
it.skipIf(RECORDING && !scenario.recorded)('matches the goldens', async () => {
|
||||
const dir = join(snapshotsDir, scenario.name)
|
||||
const input = JSON.parse(await readFile(join(dir, 'input.json'), 'utf8')) as InputScript
|
||||
const overrideFile = join(dir, 'replay.override.json')
|
||||
const workspaceDir = join(dir, 'workspace')
|
||||
const childSessions = scenario.childSessions ?? 0
|
||||
const result = await runScenario(input, {
|
||||
agent,
|
||||
mode,
|
||||
fixtureFile: join(dir, 'session.jsonl'),
|
||||
...existsSync(overrideFile) ? { overrideFile } : {},
|
||||
// In REPLAY, forward the recorded child fixtures so each subagent session
|
||||
// replays from its own script. In RECORD they are harvested, not read.
|
||||
...!RECORDING && childSessions > 0 ? { childFiles: childFixturePaths(dir, childSessions) } : {},
|
||||
...existsSync(workspaceDir) ? { workspaceDir } : {},
|
||||
})
|
||||
|
||||
// Scrub every volatile id the run produced: the ACP server-issued session
|
||||
// id plus every harvested log's recorded id (a subagent child id never
|
||||
// surfaces over ACP, but it appears in the child's own log header). The
|
||||
// normalizer's UUID catch-all covers any we don't enumerate.
|
||||
const ctx: NormalizeContext = {
|
||||
sessionIds: [
|
||||
...result.sessionId !== undefined ? [result.sessionId] : [],
|
||||
...result.sessionLogs.map(l => l.id),
|
||||
],
|
||||
cwd: result.cwd,
|
||||
}
|
||||
|
||||
// RECORD mode (recorded model scenarios only): persist the freshly-harvested
|
||||
// logs back to their fixtures — the primary to session.jsonl, each child to
|
||||
// session.<n>.jsonl in harvest order. `--update` refreshes the Vitest
|
||||
// goldens but NOT these fixtures, so write them here. A non-pinning
|
||||
// scenario's fixtures are written header-scrubbed, so a re-record can
|
||||
// never smuggle the full prompt/schema content back into every fixture.
|
||||
const scrub = scenario.pinsHeader === true
|
||||
? (log: string): string => log
|
||||
: scrubRequestHeaders
|
||||
if (RECORDING && scenario.recorded && scenario.hasModelTurn) {
|
||||
expect(result.sessionLogs.length, 'record produced no session log to harvest').toBeGreaterThan(0)
|
||||
expect(result.sessionLogs.length, `expected ${childSessions + 1} session logs (parent + children)`)
|
||||
.toBe(childSessions + 1)
|
||||
await writeFile(join(dir, 'session.jsonl'), scrub((result.sessionLogs[0] as HarvestedLog).content))
|
||||
for (let i = 1; i < result.sessionLogs.length; i++) {
|
||||
await writeFile(join(dir, `session.${i}.jsonl`), scrub((result.sessionLogs[i] as HarvestedLog).content))
|
||||
}
|
||||
}
|
||||
|
||||
await expect(normalizeStdout(result.rawStdout, ctx))
|
||||
.toMatchFileSnapshot(join(dir, 'stdout.golden.jsonl'))
|
||||
|
||||
// A model turn always produces a log worth comparing; a hook scenario can
|
||||
// produce one without a model turn (a `rejected` turn carrying `hook/*`).
|
||||
const comparesLog = scenario.comparesLog ?? scenario.hasModelTurn
|
||||
if (comparesLog) {
|
||||
// The harvested logs (primary-first) must match their committed fixtures
|
||||
// 1:1. Each side passes through normalizeSessionLog, scrubbed against ITS
|
||||
// OWN volatile values — the live run's via `ctx`, the committed fixture's
|
||||
// via its own header (a committed file cannot share the live run's ids).
|
||||
// Unless this scenario pins the header, both sides ALSO pass through
|
||||
// scrubRequestHeaders: the live log carries the real prompt/schemas, the
|
||||
// fixture carries the `{{system}}`/`{{tools}}` tokens, and the scrub is
|
||||
// idempotent — so the compare checks the header's presence, position,
|
||||
// reason, and config, but not its bulk content (pinned once, in the
|
||||
// `pinsHeader` scenario).
|
||||
expect(result.sessionLogs.length, 'this scenario must persist a session log').toBe(childSessions + 1)
|
||||
const fixtureFiles = ['session.jsonl', ...Array.from({ length: childSessions }, (_, i) => `session.${i + 1}.jsonl`)]
|
||||
for (let i = 0; i < fixtureFiles.length; i++) {
|
||||
const harvested = scrub((result.sessionLogs[i] as HarvestedLog).content)
|
||||
const fixture = scrub(await readFile(join(dir, fixtureFiles[i] as string), 'utf8'))
|
||||
expect(normalizeSessionLog(harvested, ctx), `${fixtureFiles[i]} mismatch`)
|
||||
.toEqual(normalizeSessionLog(fixture, fixtureContext(fixture)))
|
||||
}
|
||||
}
|
||||
|
||||
// Header-uniformity guard: the single pin is sound only while every
|
||||
// session in the suite composes the SAME header and keeps it for the
|
||||
// whole run. Assert both halves live. (1) Every request/header the run
|
||||
// produced (parent, spawn child, fork child, initial or resume) must
|
||||
// equal the pinned fixture's header after each side is normalized
|
||||
// against its own volatile values. (2) No request/header-delta may
|
||||
// appear at all — a mid-run header change diverges from the pin by
|
||||
// construction, and its content would be invisible under the scrub. If
|
||||
// either fails, either the header changed (update the pin: re-record or
|
||||
// hand-edit the pinning scenario's fixture) or composition became
|
||||
// session-dependent by design (give the divergent shape its own
|
||||
// pinning scenario).
|
||||
if (scenario.pinsHeader !== true && scenario.variesHeader !== true) {
|
||||
const pinnedFixture = await readFile(join(snapshotsDir, pinningScenario.name, 'session.jsonl'), 'utf8')
|
||||
const pinned = normalizedHeaders(pinnedFixture, fixtureContext(pinnedFixture))
|
||||
expect(pinned.length, `the pinning fixture (${pinningScenario.name}) must carry exactly one request/header`)
|
||||
.toBe(1)
|
||||
for (const log of result.sessionLogs) {
|
||||
expect(headerDeltaCount(log.content), `session ${log.id}: a request/header-delta in a non-pinning scenario`)
|
||||
.toBe(0)
|
||||
const headers = normalizedHeaders(log.content, ctx)
|
||||
for (const [k, header] of headers.entries()) {
|
||||
expect(header, `session ${log.id}: request/header #${k + 1} diverged from the pinned (${pinningScenario.name}) header`)
|
||||
.toEqual(pinned[0])
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
describe('snapshot fixtures', () => {
|
||||
it('every scenario directory is registered (no orphans)', async () => {
|
||||
// toMatchFileSnapshot does not prune orphaned golden/fixture files, so a
|
||||
// renamed/removed scenario could leave a stale dir that nothing exercises.
|
||||
// Fail loud on any snapshots/<dir> not present in the scenario table.
|
||||
const entries = await readdir(snapshotsDir, { withFileTypes: true })
|
||||
const onDisk = entries.filter(e => e.isDirectory()).map(e => e.name).sort()
|
||||
const registered = scenarios.map(s => s.name).sort()
|
||||
expect(onDisk).toEqual(registered)
|
||||
})
|
||||
|
||||
it('every registered scenario has its required fixture files', () => {
|
||||
// Every scenario has an input script and an stdout golden. EVERY scenario
|
||||
// also needs `session.jsonl`: the suite boots `llm-replay` with that path
|
||||
// as the replay source for ALL scenarios (the factory passes
|
||||
// `fixtureFile: <dir>/session.jsonl` unconditionally), and `loadReplayScript`
|
||||
// throws "fixture not found" when it is absent and no override replaces it.
|
||||
// A no-model scenario ships a header-only `session.jsonl` (it derives to an
|
||||
// empty script — no model call is made); a model scenario's fixture also
|
||||
// doubles as the expected-log artifact the run is diffed against. The
|
||||
// `replay.override.json` sidecar is matched BOTH ways against the table's
|
||||
// `overridden` flag: required when set, forbidden when not — the harness
|
||||
// forwards the file purely on existence, so an unregistered stray sidecar
|
||||
// would silently replace the derived script.
|
||||
for (const { name, overridden, childSessions } of scenarios) {
|
||||
const dir = join(snapshotsDir, name)
|
||||
expect(existsSync(join(dir, 'input.json')), `${name}/input.json`).toBe(true)
|
||||
expect(existsSync(join(dir, 'stdout.golden.jsonl')), `${name}/stdout.golden.jsonl`).toBe(true)
|
||||
expect(existsSync(join(dir, 'session.jsonl')), `${name}/session.jsonl`).toBe(true)
|
||||
expect(existsSync(join(dir, 'replay.override.json')), `${name}/replay.override.json presence must match \`overridden\``)
|
||||
.toBe(overridden === true)
|
||||
// A nested-agent scenario ships one child fixture per recorded subagent
|
||||
// session (`session.1.jsonl` …), the replay source for that child session.
|
||||
for (const childFixture of childFixturePaths(dir, childSessions ?? 0)) {
|
||||
expect(existsSync(childFixture), childFixture).toBe(true)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
it('exactly one scenario pins the request-header content', () => {
|
||||
// Zero pins would drop the prompt/schema surface from the suite entirely;
|
||||
// two would split it. One pin per suite is the design (pinned-header RFC);
|
||||
// WHICH scenario pins is the scenario table's reviewable choice.
|
||||
expect(scenarios.filter(s => s.pinsHeader === true).map(s => s.name)).toEqual([pinningScenario.name])
|
||||
})
|
||||
|
||||
it('committed fixtures carry request-header content ONLY in the pinning scenario', async () => {
|
||||
// The whole point of the pin: a system-prompt or tool-schema change must
|
||||
// churn exactly one committed line. A non-pinning fixture that carries the
|
||||
// full header (a hand-recorded file, or a header line hand-edited out of
|
||||
// its canonical JSON form) silently reopens the suite-wide churn, so fail
|
||||
// loud here: every non-pinning session*.jsonl must be a fixed point of
|
||||
// scrubRequestHeaders (apply the scrub to fix a violation), and the
|
||||
// pinning scenario's fixtures must NOT be (their content IS the pin).
|
||||
for (const scenario of scenarios) {
|
||||
const dir = join(snapshotsDir, scenario.name)
|
||||
const files = [
|
||||
'session.jsonl',
|
||||
...Array.from({ length: scenario.childSessions ?? 0 }, (_, i) => `session.${i + 1}.jsonl`),
|
||||
]
|
||||
for (const file of files) {
|
||||
const fixture = await readFile(join(dir, file), 'utf8')
|
||||
if (scenario.pinsHeader === true) {
|
||||
expect(scrubRequestHeaders(fixture), `${scenario.name}/${file} must PIN the full header content`)
|
||||
.not.toEqual(fixture)
|
||||
} else {
|
||||
expect(scrubRequestHeaders(fixture), `${scenario.name}/${file} carries unscrubbed header content`)
|
||||
.toEqual(fixture)
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
Reference in New Issue
Block a user