Merge branch 'code-runtime-worker' into code-mode-tools
Brings in the refreshed base (master merged through the stack after #203 and #205 landed), including the acp-snapshot extraction (#204), and re-ports this PR's snapshot-suite extensions onto the extracted package: - dsh-acp-snapshot's Scenario gains headerClass and configPath; the suite factory pins the request header PER CLASS (construction rejects a missing or duplicated class pin), forwards a scenario's configPath to the harness (RunOptions.configPath overrides AgentUnderTest.configPath), and a new fixtures meta-test asserts every pinning fixture carries exactly one request/header and no deltas. - The acp-agent example's thin scenario table re-registers code-mode-turn and both-mode-turn with their overlay configs and per-class pins; the committed fixtures replay unchanged. - The package's synthetic suites cover the new surface (explicit headerClass on one suite, the default on the other, a configPath override through the fake bin, and the two construction throws).
This commit is contained in:
Tianyi Cui
@@ -57,8 +57,8 @@ interface Spawned {
|
||||
}
|
||||
|
||||
// TODO(acp-test-harness): this subprocess/client boot glue is duplicated with
|
||||
// hooks.e2e.ts and partly with snapshot-harness.ts. Extract one shared ACP test
|
||||
// launcher before the TSX/env/permission-stub details drift again.
|
||||
// hooks.e2e.ts and partly with dsh-acp-snapshot's harness. Migrate both e2e
|
||||
// files onto that launcher before the TSX/env/permission-stub details drift.
|
||||
function spawnAcpAgent(cwd: string, env: NodeJS.ProcessEnv = process.env): Spawned {
|
||||
const child = spawn(
|
||||
process.execPath,
|
||||
|
||||
@@ -1,106 +1,31 @@
|
||||
import { readFile, readdir, writeFile } from 'node:fs/promises'
|
||||
import { existsSync } from 'node:fs'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import { dirname, join } from 'node:path'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { type HarvestedLog, type InputScript, runScenario } from './snapshot-harness.ts'
|
||||
import { type NormalizeContext, normalizeSessionLog, normalizeStdout, scrubRequestHeaders } from './snapshot-normalize.ts'
|
||||
import { defineAcpSnapshotSuite, type Scenario } from '@deepseek-ai/dsh-acp-snapshot'
|
||||
|
||||
/**
|
||||
* ACP snapshot tests (REPLAY by default, keyless). Each scenario under
|
||||
* `snapshots/<name>/` ships an `input.json` (the client stdin script) and a
|
||||
* `session.jsonl` fixture; replay boots the real acp-agent subprocess, drives
|
||||
* it, and diffs the normalized stdout transcript against the committed
|
||||
* `stdout.golden.jsonl`. For model scenarios it ALSO checks the re-persisted
|
||||
* session log — against the `session.jsonl` fixture itself, not a separate
|
||||
* golden: the fixture doubles as the replay source (recorded scenarios) and the
|
||||
* expected produced log (both sides normalized before comparing).
|
||||
*
|
||||
* Request-header content (the composed system prompt + tool schemas riding on
|
||||
* `request/header` events) is pinned by exactly ONE scenario — the one with
|
||||
* `pinsHeader` — and scrubbed to `{{system}}`/`{{tools}}` tokens in every
|
||||
* other fixture and compare, so a prompt or tool-schema edit churns one
|
||||
* committed line instead of every fixture. A per-run uniformity guard keeps
|
||||
* the single pin sound: every live header must equal the pinned one, and no
|
||||
* header-delta may appear outside the pinning scenario (see the
|
||||
* pinned-header RFC,
|
||||
* docs/rfc/implemented/testing/2026-07-06-pin-request-header-content-in-one-scenario.md).
|
||||
*
|
||||
* `pnpm run test:snapshot:record` (DSH_SNAPSHOT=record + -u) re-records the
|
||||
* `session.jsonl` fixtures against the real API and refreshes the stdout golden
|
||||
* in one pass.
|
||||
* The acp-agent example's snapshot suite: the scenario table for
|
||||
* `dsh-acp-snapshot`'s suite factory, which owns every compare/guard mechanic
|
||||
* (golden + re-persisted-log diffs, record write-back, the pinned-header
|
||||
* uniformity guard, the fixture guards). Fixtures live under `snapshots/<name>/`;
|
||||
* `pnpm run test:snapshot:record` re-records the `recorded` scenarios against
|
||||
* the real API. See the package README (packages/support/acp-snapshot) and the
|
||||
* snapshot RFC, docs/rfc/implemented/testing/2026-06-19-acp-snapshot-tests.md.
|
||||
*/
|
||||
|
||||
const SNAPSHOTS_DIR = join(dirname(fileURLToPath(import.meta.url)), 'snapshots')
|
||||
const RECORDING = process.env.DSH_SNAPSHOT === 'record'
|
||||
|
||||
/** A snapshot scenario and how its fixtures are produced. */
|
||||
interface Scenario {
|
||||
name: string
|
||||
/** Whether the scenario drives at least one model turn (so a JSONL golden applies). */
|
||||
hasModelTurn: boolean
|
||||
/**
|
||||
* Whether the run persists a comparable session log to diff against the
|
||||
* `session.jsonl` fixture. Defaults to {@link hasModelTurn} (a model turn
|
||||
* always produces a log worth comparing). Set it independently for a scenario
|
||||
* that produces a non-trivial log WITHOUT a model turn — e.g. a prompt blocked
|
||||
* by a `UserPromptSubmit` hook, which opens a `rejected` turn carrying `hook/*`
|
||||
* events but never calls the model.
|
||||
*/
|
||||
comparesLog?: boolean
|
||||
/**
|
||||
* Whether `test:snapshot:record` regenerates this scenario's `session.jsonl`
|
||||
* from the LIVE API. `recorded` scenarios are model-driven and reproducible;
|
||||
* `authored` scenarios (a hand-written `replay.override.json` sidecar drives
|
||||
* replay — e.g. a provider error or a cancel, which the live API can't be
|
||||
* coaxed into deterministically — or a deterministic hook scenario whose
|
||||
* derived empty script needs no sidecar) are NEVER re-recorded.
|
||||
*/
|
||||
recorded: boolean
|
||||
/**
|
||||
* How many SUBAGENT child sessions this scenario records beyond the top-level
|
||||
* one (0 for a single-session scenario). Each child rides in a sibling fixture
|
||||
* `session.<n>.jsonl` (1-based); replay forwards them to `dsh-llm-replay` so
|
||||
* each child session replays from its own script, and record mode writes the
|
||||
* harvested child logs back to those files. Defaults to 0.
|
||||
*/
|
||||
childSessions?: number
|
||||
/**
|
||||
* Whether THIS scenario's fixtures keep the full request-header content (the
|
||||
* composed system prompt and tool schema list on `request/header` /
|
||||
* `request/header-delta` events) and compare it verbatim. Exactly one
|
||||
* scenario pins it PER HEADER CLASS ({@link headerClass}); every other
|
||||
* scenario of that class stores and compares that content as
|
||||
* `{{system}}`/`{{tools}}` tokens ({@link scrubRequestHeaders}), so a system
|
||||
* prompt or tool-schema change shows up as ONE committed-fixture diff per
|
||||
* class, not one per scenario. One pin per class suffices because header
|
||||
* composition is class-uniform (parent, spawn child, and fork child all
|
||||
* compose the same prompt-modulo-cwd and the same tools) — and that premise
|
||||
* is ASSERTED, not assumed: every non-pinning run's live headers must equal
|
||||
* its class's pinned fixture's (normalized), so a session-dependent header
|
||||
* (say, a restricted subagent toolset) fails loud until it gets its own
|
||||
* pinning scenario.
|
||||
* Defaults to false.
|
||||
*/
|
||||
pinsHeader?: boolean
|
||||
/**
|
||||
* Which header-composition class this scenario belongs to. Scenarios that
|
||||
* boot the same config compose the same header; each class has exactly one
|
||||
* {@link pinsHeader} scenario, and the uniformity guard compares every
|
||||
* other member against ITS class's pin. Defaults to `'default'` (the
|
||||
* example's stock `cordis.yml`); the Code Mode scenarios — booting overlay
|
||||
* configs whose tool list and prompt sections differ by construction —
|
||||
* carry their own classes.
|
||||
*/
|
||||
headerClass?: string
|
||||
/**
|
||||
* Alternate live-config basename under `examples/acp-agent/` for this
|
||||
* scenario's boot (the replay swap derives `*cordis.snapshot.yml` from it).
|
||||
* Defaults to `cordis.yml`.
|
||||
*/
|
||||
configBase?: string
|
||||
// The dsh-acp-agent bin (the demo:acp entry), this example's cordis.yml, and
|
||||
// the repo-root tsconfig (four levels up from examples/acp-agent/tests) — all
|
||||
// ABSOLUTE: the subprocess cwd is a temp dir outside the repo.
|
||||
const AGENT = {
|
||||
binScript: fileURLToPath(new URL('../../../packages/ui/acp-agent/src/bin.ts', import.meta.url)),
|
||||
configPath: fileURLToPath(new URL('../cordis.yml', import.meta.url)),
|
||||
tsconfigPath: fileURLToPath(new URL('../../../tsconfig.json', import.meta.url)),
|
||||
}
|
||||
|
||||
// The Code Mode overlay configs (include-patched variants of cordis.yml; the
|
||||
// replay swap resolves each one's sibling `*cordis.snapshot.yml`).
|
||||
const CODE_MODE_CONFIG = fileURLToPath(new URL('../code-mode.cordis.yml', import.meta.url))
|
||||
const BOTH_MODE_CONFIG = fileURLToPath(new URL('../both-mode.cordis.yml', import.meta.url))
|
||||
|
||||
const SCENARIOS: Scenario[] = [
|
||||
{ name: 'handshake', hasModelTurn: false, recorded: false },
|
||||
{ name: 'reject-extra-dirs', hasModelTurn: false, recorded: false },
|
||||
@@ -169,283 +94,13 @@ const SCENARIOS: Scenario[] = [
|
||||
// tool calls land as tool/code-dispatch events. Each mode boots its own
|
||||
// overlay config, composes a different header by construction, and
|
||||
// therefore pins its own class.
|
||||
{ name: 'code-mode-turn', hasModelTurn: true, recorded: true, pinsHeader: true, headerClass: 'code', configBase: 'code-mode.cordis.yml' },
|
||||
{ name: 'both-mode-turn', hasModelTurn: true, recorded: true, pinsHeader: true, headerClass: 'both', configBase: 'both-mode.cordis.yml' },
|
||||
{ name: 'code-mode-turn', hasModelTurn: true, recorded: true, pinsHeader: true, headerClass: 'code', configPath: CODE_MODE_CONFIG },
|
||||
{ name: 'both-mode-turn', hasModelTurn: true, recorded: true, pinsHeader: true, headerClass: 'both', configPath: BOTH_MODE_CONFIG },
|
||||
]
|
||||
|
||||
/** Each header class's single pinning scenario. Guarded here (and by a meta-test) so a pin cannot silently vanish. */
|
||||
const pinningByClass = new Map<string, Scenario>()
|
||||
for (const scenario of SCENARIOS) {
|
||||
if (scenario.pinsHeader !== true) continue
|
||||
const cls = scenario.headerClass ?? 'default'
|
||||
const existing = pinningByClass.get(cls)
|
||||
if (existing) throw new Error(`acp.snapshot: header class "${cls}" pinned by both ${existing.name} and ${scenario.name}`)
|
||||
pinningByClass.set(cls, scenario)
|
||||
}
|
||||
for (const scenario of SCENARIOS) {
|
||||
const cls = scenario.headerClass ?? 'default'
|
||||
if (!pinningByClass.has(cls)) throw new Error(`acp.snapshot: no scenario pins the request-header content of class "${cls}" (needed by ${scenario.name})`)
|
||||
}
|
||||
|
||||
/** The sibling child-fixture paths for a scenario (`session.1.jsonl` …). */
|
||||
function childFixturePaths(dir: string, childSessions: number): string[] {
|
||||
return Array.from({ length: childSessions }, (_, i) => join(dir, `session.${i + 1}.jsonl`))
|
||||
}
|
||||
|
||||
/**
|
||||
* Derive the {@link NormalizeContext} for a `session.jsonl` fixture from its own
|
||||
* header line (`{ type: 'session', id, cwd }`). A committed fixture carries the
|
||||
* session id and cwd of the run that harvested it — different from the live
|
||||
* replay run — so normalizing it against the live run's ctx would leave those
|
||||
* recorded values unscrubbed. Reading them from the header scrubs the fixture's
|
||||
* own id/cwd to the same `{{sessionId}}`/`{{cwd}}` tokens the replay output gets.
|
||||
* An authored fixture whose header is already normalized (`id:'{{sessionId}}'`,
|
||||
* `cwd:'{{cwd}}'`) yields those tokens as the volatile values, so scrubbing them
|
||||
* is an idempotent no-op. A header with no `cwd` falls back to a sentinel that
|
||||
* cannot occur in a log (NOT `''`, which `String.split` would match on every
|
||||
* character boundary and corrupt the output).
|
||||
*/
|
||||
function fixtureContext(fixture: string): NormalizeContext {
|
||||
const firstLine = fixture.split('\n').find(line => line.trim().length > 0) ?? '{}'
|
||||
const header = JSON.parse(firstLine) as { id?: unknown; cwd?: unknown }
|
||||
return {
|
||||
sessionIds: typeof header.id === 'string' ? [header.id] : [],
|
||||
cwd: typeof header.cwd === 'string' ? header.cwd : '\0no-cwd\0',
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The `data.header` payload of every `request/header` event in a session
|
||||
* JSONL, in log order, with the log's volatile values scrubbed first
|
||||
* ({@link normalizeSessionLog}) so headers harvested from different runs —
|
||||
* each embedding its own temp cwd in the composed prompt — compare on equal
|
||||
* footing.
|
||||
*/
|
||||
function normalizedHeaders(rawLog: string, ctx: NormalizeContext): unknown[] {
|
||||
return normalizeSessionLog(rawLog, ctx)
|
||||
.split('\n')
|
||||
.filter(line => line.trim().length > 0)
|
||||
.map(line => JSON.parse(line) as { type?: unknown; data?: { header?: unknown } })
|
||||
.filter(record => record.type === 'request/header')
|
||||
.map(record => record.data?.header)
|
||||
}
|
||||
|
||||
/** Count the `request/header-delta` events in a session JSONL. */
|
||||
function headerDeltaCount(rawLog: string): number {
|
||||
return rawLog.split('\n')
|
||||
.filter(line => line.trim().length > 0)
|
||||
.filter(line => (JSON.parse(line) as { type?: unknown }).type === 'request/header-delta')
|
||||
.length
|
||||
}
|
||||
|
||||
for (const scenario of SCENARIOS) {
|
||||
describe(`snapshot: ${scenario.name}`, () => {
|
||||
// In RECORD mode, only re-run the `recorded` (live-API) scenarios; the
|
||||
// `authored` ones (sidecar-driven errors/cancel) are never re-recorded.
|
||||
it.skipIf(RECORDING && !scenario.recorded)('matches the goldens', async () => {
|
||||
const dir = join(SNAPSHOTS_DIR, scenario.name)
|
||||
const input = JSON.parse(await readFile(join(dir, 'input.json'), 'utf8')) as InputScript
|
||||
const overrideFile = join(dir, 'replay.override.json')
|
||||
const workspaceDir = join(dir, 'workspace')
|
||||
const childSessions = scenario.childSessions ?? 0
|
||||
const result = await runScenario(input, {
|
||||
mode: RECORDING ? 'record' : 'replay',
|
||||
fixtureFile: join(dir, 'session.jsonl'),
|
||||
...existsSync(overrideFile) ? { overrideFile } : {},
|
||||
// In REPLAY, forward the recorded child fixtures so each subagent session
|
||||
// replays from its own script. In RECORD they are harvested, not read.
|
||||
...!RECORDING && childSessions > 0 ? { childFiles: childFixturePaths(dir, childSessions) } : {},
|
||||
...existsSync(workspaceDir) ? { workspaceDir } : {},
|
||||
// A scenario booting an overlay tree passes its live config; the bin's
|
||||
// replay swap derives the sibling `*cordis.snapshot.yml` from it.
|
||||
...scenario.configBase !== undefined
|
||||
? { configPath: join(SNAPSHOTS_DIR, '..', '..', scenario.configBase) }
|
||||
: {},
|
||||
})
|
||||
|
||||
// Scrub every volatile id the run produced: the ACP server-issued session
|
||||
// id plus every harvested log's recorded id (a subagent child id never
|
||||
// surfaces over ACP, but it appears in the child's own log header). The
|
||||
// normalizer's UUID catch-all covers any we don't enumerate.
|
||||
const ctx: NormalizeContext = {
|
||||
sessionIds: [
|
||||
...result.sessionId !== undefined ? [result.sessionId] : [],
|
||||
...result.sessionLogs.map(l => l.id),
|
||||
],
|
||||
cwd: result.cwd,
|
||||
}
|
||||
|
||||
// RECORD mode (recorded model scenarios only): persist the freshly-harvested
|
||||
// logs back to their fixtures — the primary to session.jsonl, each child to
|
||||
// session.<n>.jsonl in harvest order. `--update` refreshes the Vitest
|
||||
// goldens but NOT these fixtures, so write them here. A non-pinning
|
||||
// scenario's fixtures are written header-scrubbed, so a re-record can
|
||||
// never smuggle the full prompt/schema content back into every fixture.
|
||||
const scrub = scenario.pinsHeader === true
|
||||
? (log: string): string => log
|
||||
: scrubRequestHeaders
|
||||
if (RECORDING && scenario.recorded && scenario.hasModelTurn) {
|
||||
expect(result.sessionLogs.length, 'record produced no session log to harvest').toBeGreaterThan(0)
|
||||
expect(result.sessionLogs.length, `expected ${childSessions + 1} session logs (parent + children)`)
|
||||
.toBe(childSessions + 1)
|
||||
await writeFile(join(dir, 'session.jsonl'), scrub((result.sessionLogs[0] as HarvestedLog).content))
|
||||
for (let i = 1; i < result.sessionLogs.length; i++) {
|
||||
await writeFile(join(dir, `session.${i}.jsonl`), scrub((result.sessionLogs[i] as HarvestedLog).content))
|
||||
}
|
||||
}
|
||||
|
||||
await expect(normalizeStdout(result.rawStdout, ctx))
|
||||
.toMatchFileSnapshot(join(dir, 'stdout.golden.jsonl'))
|
||||
|
||||
// A model turn always produces a log worth comparing; a hook scenario can
|
||||
// produce one without a model turn (a `rejected` turn carrying `hook/*`).
|
||||
const comparesLog = scenario.comparesLog ?? scenario.hasModelTurn
|
||||
if (comparesLog) {
|
||||
// The harvested logs (primary-first) must match their committed fixtures
|
||||
// 1:1. Each side passes through normalizeSessionLog, scrubbed against ITS
|
||||
// OWN volatile values — the live run's via `ctx`, the committed fixture's
|
||||
// via its own header (a committed file cannot share the live run's ids).
|
||||
// Unless this scenario pins the header, both sides ALSO pass through
|
||||
// scrubRequestHeaders: the live log carries the real prompt/schemas, the
|
||||
// fixture carries the `{{system}}`/`{{tools}}` tokens, and the scrub is
|
||||
// idempotent — so the compare checks the header's presence, position,
|
||||
// reason, and config, but not its bulk content (pinned once, in the
|
||||
// `pinsHeader` scenario).
|
||||
expect(result.sessionLogs.length, 'this scenario must persist a session log').toBe(childSessions + 1)
|
||||
const fixtureFiles = ['session.jsonl', ...Array.from({ length: childSessions }, (_, i) => `session.${i + 1}.jsonl`)]
|
||||
for (let i = 0; i < fixtureFiles.length; i++) {
|
||||
const harvested = scrub((result.sessionLogs[i] as HarvestedLog).content)
|
||||
const fixture = scrub(await readFile(join(dir, fixtureFiles[i] as string), 'utf8'))
|
||||
expect(normalizeSessionLog(harvested, ctx), `${fixtureFiles[i]} mismatch`)
|
||||
.toEqual(normalizeSessionLog(fixture, fixtureContext(fixture)))
|
||||
}
|
||||
}
|
||||
|
||||
// Header-uniformity guard: a class's single pin is sound only while
|
||||
// every session in that class composes the SAME header and keeps it for
|
||||
// the whole run. Assert both halves live. (1) Every request/header the
|
||||
// run produced (parent, spawn child, fork child, initial or resume)
|
||||
// must equal the CLASS's pinned fixture's header after each side is
|
||||
// normalized against its own volatile values. (2) No
|
||||
// request/header-delta may appear at all — a mid-run header change
|
||||
// diverges from the pin by construction, and its content would be
|
||||
// invisible under the scrub. If either fails, either the header changed
|
||||
// (update the pin: re-record or hand-edit the pinning scenario's
|
||||
// fixture) or composition became session-dependent by design (give the
|
||||
// divergent shape its own pinning scenario and class).
|
||||
if (scenario.pinsHeader !== true) {
|
||||
const pinningScenario = pinningByClass.get(scenario.headerClass ?? 'default')!
|
||||
const pinnedFixture = await readFile(join(SNAPSHOTS_DIR, pinningScenario.name, 'session.jsonl'), 'utf8')
|
||||
const pinned = normalizedHeaders(pinnedFixture, fixtureContext(pinnedFixture))
|
||||
expect(pinned.length, `the pinning fixture (${pinningScenario.name}) must carry exactly one request/header`)
|
||||
.toBe(1)
|
||||
for (const log of result.sessionLogs) {
|
||||
expect(headerDeltaCount(log.content), `session ${log.id}: a request/header-delta in a non-pinning scenario`)
|
||||
.toBe(0)
|
||||
const headers = normalizedHeaders(log.content, ctx)
|
||||
for (const [k, header] of headers.entries()) {
|
||||
expect(header, `session ${log.id}: request/header #${k + 1} diverged from the pinned (${pinningScenario.name}) header`)
|
||||
.toEqual(pinned[0])
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
describe('snapshot fixtures', () => {
|
||||
it('every scenario directory is registered (no orphans)', async () => {
|
||||
// toMatchFileSnapshot does not prune orphaned golden/fixture files, so a
|
||||
// renamed/removed scenario could leave a stale dir that nothing exercises.
|
||||
// Fail loud on any snapshots/<dir> not present in SCENARIOS.
|
||||
const entries = await readdir(SNAPSHOTS_DIR, { withFileTypes: true })
|
||||
const onDisk = entries.filter(e => e.isDirectory()).map(e => e.name).sort()
|
||||
const registered = SCENARIOS.map(s => s.name).sort()
|
||||
expect(onDisk).toEqual(registered)
|
||||
})
|
||||
|
||||
it('every registered scenario has its required fixture files', async () => {
|
||||
// Every scenario has an input script and an stdout golden. EVERY scenario
|
||||
// also needs `session.jsonl`: the harness boots `llm-replay` with that path
|
||||
// as the replay source for ALL scenarios (acp.snapshot.ts passes
|
||||
// `fixtureFile: <dir>/session.jsonl` unconditionally), and `loadReplayScript`
|
||||
// throws "fixture not found" when it is absent and no override replaces it.
|
||||
// A no-model scenario ships a header-only `session.jsonl` (it derives to an
|
||||
// empty script — no model call is made); a model scenario's fixture also
|
||||
// doubles as the expected-log artifact the run is diffed against. An authored
|
||||
// (non-`recorded`) model scenario additionally ships a `replay.override.json`
|
||||
// sidecar for the throw/hang cases a derived script cannot express.
|
||||
for (const { name, hasModelTurn, recorded, childSessions } of SCENARIOS) {
|
||||
const dir = join(SNAPSHOTS_DIR, name)
|
||||
expect(existsSync(join(dir, 'input.json')), `${name}/input.json`).toBe(true)
|
||||
expect(existsSync(join(dir, 'stdout.golden.jsonl')), `${name}/stdout.golden.jsonl`).toBe(true)
|
||||
expect(existsSync(join(dir, 'session.jsonl')), `${name}/session.jsonl`).toBe(true)
|
||||
if (hasModelTurn && !recorded) {
|
||||
expect(existsSync(join(dir, 'replay.override.json')), `${name}/replay.override.json`).toBe(true)
|
||||
}
|
||||
// A nested-agent scenario ships one child fixture per recorded subagent
|
||||
// session (`session.1.jsonl` …), the replay source for that child session.
|
||||
for (const childFixture of childFixturePaths(dir, childSessions ?? 0)) {
|
||||
expect(existsSync(childFixture), childFixture).toBe(true)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
it('exactly one scenario pins the request-header content of each header class', () => {
|
||||
// Zero pins would drop a class's prompt/schema surface from the suite
|
||||
// entirely; two would split it. One pin per class is the design
|
||||
// (pinned-header RFC; the Code Mode classes compose different headers by
|
||||
// construction, so each carries its own pin).
|
||||
const pins = new Map<string, string[]>()
|
||||
for (const scenario of SCENARIOS.filter(s => s.pinsHeader === true)) {
|
||||
const cls = scenario.headerClass ?? 'default'
|
||||
pins.set(cls, [...pins.get(cls) ?? [], scenario.name])
|
||||
}
|
||||
expect(Object.fromEntries(pins)).toEqual({
|
||||
'default': ['text-turn'],
|
||||
'code': ['code-mode-turn'],
|
||||
'both': ['both-mode-turn'],
|
||||
})
|
||||
})
|
||||
|
||||
it('every pinning fixture carries exactly one request/header and no deltas', async () => {
|
||||
// The live uniformity guard runs only in NON-pinning scenarios, so a
|
||||
// class made of just its pinning scenario (the Code Mode classes) would
|
||||
// otherwise accept a re-recorded pin with several headers or a mid-run
|
||||
// header-delta — shapes the pin design cannot represent. Assert the
|
||||
// committed pins directly.
|
||||
for (const scenario of pinningByClass.values()) {
|
||||
const fixture = await readFile(join(SNAPSHOTS_DIR, scenario.name, 'session.jsonl'), 'utf8')
|
||||
const headers = normalizedHeaders(fixture, fixtureContext(fixture))
|
||||
expect(headers.length, `${scenario.name}: a pinning fixture must carry exactly one request/header`).toBe(1)
|
||||
expect(headerDeltaCount(fixture), `${scenario.name}: a pinning fixture must carry no request/header-delta`).toBe(0)
|
||||
}
|
||||
})
|
||||
|
||||
it('committed fixtures carry request-header content ONLY in the pinning scenario', async () => {
|
||||
// The whole point of the pin: a system-prompt or tool-schema change must
|
||||
// churn exactly one committed line. A non-pinning fixture that carries the
|
||||
// full header (a hand-recorded file, or a header line hand-edited out of
|
||||
// its canonical JSON form) silently reopens the suite-wide churn, so fail
|
||||
// loud here: every non-pinning session*.jsonl must be a fixed point of
|
||||
// scrubRequestHeaders (apply the scrub to fix a violation), and the
|
||||
// pinning scenario's fixtures must NOT be (their content IS the pin).
|
||||
for (const scenario of SCENARIOS) {
|
||||
const dir = join(SNAPSHOTS_DIR, scenario.name)
|
||||
const files = [
|
||||
'session.jsonl',
|
||||
...Array.from({ length: scenario.childSessions ?? 0 }, (_, i) => `session.${i + 1}.jsonl`),
|
||||
]
|
||||
for (const file of files) {
|
||||
const fixture = await readFile(join(dir, file), 'utf8')
|
||||
if (scenario.pinsHeader === true) {
|
||||
expect(scrubRequestHeaders(fixture), `${scenario.name}/${file} must PIN the full header content`)
|
||||
.not.toEqual(fixture)
|
||||
} else {
|
||||
expect(scrubRequestHeaders(fixture), `${scenario.name}/${file} carries unscrubbed header content`)
|
||||
.toEqual(fixture)
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
defineAcpSnapshotSuite({
|
||||
agent: AGENT,
|
||||
snapshotsDir: join(dirname(fileURLToPath(import.meta.url)), 'snapshots'),
|
||||
scenarios: SCENARIOS,
|
||||
mode: process.env.DSH_SNAPSHOT === 'record' ? 'record' : 'replay',
|
||||
})
|
||||
|
||||
@@ -1,393 +0,0 @@
|
||||
/**
|
||||
* Shared harness for the ACP snapshot tests. A plain module (NOT a *.spec.ts /
|
||||
* *.snapshot.ts) so importing it never re-registers another file's tests.
|
||||
*
|
||||
* It boots the REAL examples/acp-agent subprocess via the cordis Loader (so the
|
||||
* export-shape bug class stays guarded — see docs/postmortem/0001), drives it
|
||||
* over real ACP JSON-RPC stdio with a deterministic input script, tees raw
|
||||
* stdout (for the golden + a purity check) into an SDK `ClientSideConnection`,
|
||||
* and — in record mode — harvests the persisted session JSONL after a graceful
|
||||
* shutdown flush. Two pure normalizers turn the captured stdout frames and the
|
||||
* session-log events into stable, snapshot-able text.
|
||||
*
|
||||
* See docs/rfc/implemented/testing/2026-06-19-acp-snapshot-tests.md.
|
||||
*/
|
||||
|
||||
import { spawn, type ChildProcessWithoutNullStreams } from 'node:child_process'
|
||||
import { cp, mkdtemp, readFile, readdir, rm } from 'node:fs/promises'
|
||||
import { existsSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join, delimiter } from 'node:path'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import { Readable, Writable } from 'node:stream'
|
||||
import {
|
||||
ClientSideConnection,
|
||||
ndJsonStream,
|
||||
PROTOCOL_VERSION,
|
||||
type Agent as AcpAgent,
|
||||
type Client,
|
||||
type RequestPermissionRequest,
|
||||
type RequestPermissionResponse,
|
||||
type SessionNotification,
|
||||
} from '@agentclientprotocol/sdk'
|
||||
|
||||
// The dsh-acp-agent bin (the demo:acp entry) and this example's cordis.yml.
|
||||
// The bin resolves its config-path arg from CWD and, under DSH_SNAPSHOT=replay,
|
||||
// swaps it for the sibling cordis.snapshot.yml. The child's cwd is a temp dir
|
||||
// OUTSIDE the repo, so pass the example config's ABSOLUTE path.
|
||||
const binScript = fileURLToPath(new URL('../../../packages/ui/acp-agent/src/bin.ts', import.meta.url))
|
||||
const configPath = fileURLToPath(new URL('../cordis.yml', import.meta.url))
|
||||
const tsxLoader = fileURLToPath(import.meta.resolve('tsx'))
|
||||
// The repo-root tsconfig: dev/test run UNBUILT and the `@deepseek-ai/dsh-*`
|
||||
// imports resolve through its `paths` map. The child's cwd is a temp dir
|
||||
// OUTSIDE the repo, so tsx's upward search would miss it — point tsx at the
|
||||
// repo tsconfig explicitly (same fix the e2e harness uses). Repo root is four
|
||||
// levels up from this file (examples/acp-agent/tests).
|
||||
const repoTsconfig = fileURLToPath(new URL('../../../tsconfig.json', import.meta.url))
|
||||
|
||||
/**
|
||||
* One step of a scenario's deterministic input script (`input.json`). The
|
||||
* harness interprets these in order. `newSession` captures the server-issued
|
||||
* (random) session id into a `{{sessionId}}` variable that later steps
|
||||
* reference, since a committed file cannot know the id in advance.
|
||||
*
|
||||
* `promptAndCancel` sends a prompt WITHOUT awaiting its response, waits until
|
||||
* the client observes the first streamed `agent_message_chunk` (so the emitted
|
||||
* frames deterministically precede the cancellation), then cancels the turn —
|
||||
* the only way to exercise a cancel deterministically (a plain `prompt` step
|
||||
* awaits the response, which a cancel/hang scenario would block on forever).
|
||||
*/
|
||||
type InputStep =
|
||||
| { op: 'initialize'; terminalOutput?: boolean }
|
||||
| { op: 'newSession' }
|
||||
| { op: 'newSessionExpectError'; additionalDirectories?: string[] }
|
||||
| { op: 'prompt'; text: string }
|
||||
| { op: 'promptExpectError'; text: string }
|
||||
| { op: 'promptAndCancel'; text: string }
|
||||
| { op: 'cancel' }
|
||||
|
||||
/** A scenario's `input.json`: an ordered list of input steps. */
|
||||
export interface InputScript {
|
||||
steps: InputStep[]
|
||||
}
|
||||
|
||||
/** One harvested session log plus the identifying facts off its header line. */
|
||||
export interface HarvestedLog {
|
||||
/** The recorded session id (header `id`). */
|
||||
id: string
|
||||
/** Session creation time (header `createdAt`) — the child-ordering key. */
|
||||
createdAt: number
|
||||
/** The parent session id, if this log is a subagent child (header `parentSession`). */
|
||||
parentSession?: string
|
||||
/** The full `.jsonl` file content. */
|
||||
content: string
|
||||
}
|
||||
|
||||
/** The result of running a scenario: raw stdout + the harvested session log(s). */
|
||||
export interface RunResult {
|
||||
/** Raw stdout bytes (decoded utf8), every newline-delimited JSON-RPC frame. */
|
||||
rawStdout: string
|
||||
/** stderr (for diagnostics on failure). */
|
||||
stderr: string
|
||||
/** The session id the server issued (undefined if no session was created). */
|
||||
sessionId?: string
|
||||
/** The temp cwd the session ran in (the bash workspace). */
|
||||
cwd: string
|
||||
/**
|
||||
* Every persisted session log harvested after the run, ordered primary-first:
|
||||
* the top-level (parent) session — the one with no `parentSession` — then each
|
||||
* subagent child by ascending `createdAt`. A single-session scenario harvests
|
||||
* exactly one; a nested-agent scenario harvests the parent plus one per child.
|
||||
*/
|
||||
sessionLogs: HarvestedLog[]
|
||||
}
|
||||
|
||||
interface RunOptions {
|
||||
/** `replay` (default, keyless) or `record` (real API, harvests the log). */
|
||||
mode: 'replay' | 'record'
|
||||
/** The recorded session JSONL fixture path (replay reads it; record writes near it). */
|
||||
fixtureFile: string
|
||||
/** Optional sidecar override path (replay). */
|
||||
overrideFile?: string
|
||||
/**
|
||||
* Recorded SUBAGENT child-session fixture paths (replay). A nested-agent
|
||||
* scenario ships one per child (`session.1.jsonl`, …); the harness forwards
|
||||
* them to `dsh-llm-replay` via `$DSH_SNAPSHOT_CHILD_FILES` so each child
|
||||
* session replays from its own recorded script. Empty for single-session
|
||||
* scenarios. Ignored in record mode (children are harvested, not replayed).
|
||||
*/
|
||||
childFiles?: string[]
|
||||
/**
|
||||
* Optional `<scenario>/workspace/` directory whose contents are copied into
|
||||
* the temp cwd BEFORE the run — the standard way to seed files the agent
|
||||
* operates on (a file to read, edit, or grep). Absent for scenarios that
|
||||
* start from an empty workspace.
|
||||
*/
|
||||
workspaceDir?: string
|
||||
/**
|
||||
* Alternate LIVE config path for the boot (absolute). Defaults to the
|
||||
* example's `cordis.yml`. A scenario needing a differently-composed tree
|
||||
* (the Code Mode scenarios) ships an overlay whose basename still ends in
|
||||
* `cordis.yml`, so the bin's replay swap resolves the sibling
|
||||
* `*cordis.snapshot.yml` the same way it does for the default.
|
||||
*/
|
||||
configPath?: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Run a scenario end-to-end against a freshly-spawned subprocess. Owns the
|
||||
* child and its temp dirs; always tears them down. Returns the captured stdout
|
||||
* and (record mode) the harvested session-log path.
|
||||
*/
|
||||
export async function runScenario(input: InputScript, opts: RunOptions): Promise<RunResult> {
|
||||
const cwd = await mkdtemp(join(tmpdir(), 'acp-snap-cwd-'))
|
||||
const sessionsRoot = await mkdtemp(join(tmpdir(), 'acp-snap-sessions-'))
|
||||
// Everything past the temp-dir creation runs under a try/finally that always
|
||||
// removes both dirs — so a failure in workspace seeding, spawn, or any step
|
||||
// never leaks them (the "e2e tests own their resources" rule).
|
||||
let child: ChildProcessWithoutNullStreams | undefined
|
||||
let sessionId: string | undefined
|
||||
let sessionLogs: HarvestedLog[] = []
|
||||
const rawBuffers: Buffer[] = []
|
||||
const stderrChunks: string[] = []
|
||||
try {
|
||||
// Seed the workspace if the scenario ships one (a file the agent reads/edits).
|
||||
// Copied into the temp cwd so the agent's bash tools see it; the goldens
|
||||
// normalize the cwd, so the seeded paths stay stable across runs.
|
||||
if (opts.workspaceDir !== undefined && existsSync(opts.workspaceDir)) {
|
||||
await cp(opts.workspaceDir, cwd, { recursive: true })
|
||||
}
|
||||
const env: NodeJS.ProcessEnv = {
|
||||
...process.env,
|
||||
TSX_TSCONFIG_PATH: repoTsconfig,
|
||||
DSH_SNAPSHOT: opts.mode,
|
||||
DSH_SNAPSHOT_FILE: opts.fixtureFile,
|
||||
DSH_SNAPSHOT_SESSIONS_ROOT: sessionsRoot,
|
||||
...opts.overrideFile !== undefined ? { DSH_SNAPSHOT_OVERRIDE: opts.overrideFile } : {},
|
||||
...opts.childFiles !== undefined && opts.childFiles.length > 0
|
||||
? { DSH_SNAPSHOT_CHILD_FILES: opts.childFiles.join(delimiter) }
|
||||
: {},
|
||||
}
|
||||
|
||||
child = spawn(
|
||||
process.execPath,
|
||||
['--import', tsxLoader, binScript, opts.configPath ?? configPath],
|
||||
{ cwd, env, stdio: ['pipe', 'pipe', 'pipe'] },
|
||||
)
|
||||
|
||||
child.stderr.setEncoding('utf8')
|
||||
child.stderr.on('data', (c: string) => stderrChunks.push(c))
|
||||
|
||||
// Tee raw stdout: accumulate the bytes for the golden + purity check, and ALSO
|
||||
// feed the same bytes to the SDK client through a passthrough. Buffer the raw
|
||||
// bytes (not per-chunk utf8 strings) and decode once at the end, so a
|
||||
// multibyte sequence split across two 'data' events can't corrupt the golden.
|
||||
const passthrough = new Readable({ read() {} })
|
||||
child.stdout.on('data', (buf: Buffer) => {
|
||||
rawBuffers.push(buf)
|
||||
passthrough.push(buf)
|
||||
})
|
||||
child.stdout.on('end', () => passthrough.push(null))
|
||||
|
||||
const stream = ndJsonStream(
|
||||
Writable.toWeb(child.stdin) as WritableStream<Uint8Array>,
|
||||
Readable.toWeb(passthrough) as ReadableStream<Uint8Array>,
|
||||
)
|
||||
// Watcher so a step can block until the client OBSERVES a particular
|
||||
// session/update — used by promptAndCancel to pin frame order (send cancel
|
||||
// only after the streamed agent_message_chunk has arrived, so those frames
|
||||
// deterministically precede the cancelled prompt response).
|
||||
const updateWaiters: { match: (u: SessionNotification['update']) => boolean; resolve: () => void }[] = []
|
||||
const waitForUpdate = (match: (u: SessionNotification['update']) => boolean): Promise<void> =>
|
||||
new Promise<void>(resolve => updateWaiters.push({ match, resolve }))
|
||||
|
||||
const makeClient = (_agent: AcpAgent): Client => ({
|
||||
sessionUpdate(params: SessionNotification): Promise<void> {
|
||||
for (let i = updateWaiters.length - 1; i >= 0; i--) {
|
||||
const waiter = updateWaiters[i]
|
||||
if (waiter !== undefined && waiter.match(params.update)) {
|
||||
updateWaiters.splice(i, 1)
|
||||
waiter.resolve()
|
||||
}
|
||||
}
|
||||
return Promise.resolve()
|
||||
},
|
||||
requestPermission(_params: RequestPermissionRequest): Promise<RequestPermissionResponse> {
|
||||
return Promise.resolve({ outcome: { outcome: 'cancelled' } })
|
||||
},
|
||||
})
|
||||
const client = new ClientSideConnection(makeClient, stream)
|
||||
|
||||
for (const step of input.steps) {
|
||||
await runStep(client, step, cwd, waitForUpdate, () => sessionId, (id) => { sessionId = id })
|
||||
}
|
||||
// Done driving: close stdin so the server disposes gracefully (flushing
|
||||
// persistence) and exits. Then await exit so the harvested log is complete.
|
||||
child.stdin.end()
|
||||
await waitForExit(child)
|
||||
// Harvest EVERY persisted log (parent + any subagent children) while the
|
||||
// temp dirs still exist, ordered primary-first.
|
||||
sessionLogs = await harvestSessionLogs(sessionsRoot)
|
||||
} finally {
|
||||
// Failure-safe teardown: kill a still-running child and drop the temp dirs
|
||||
// even if seeding/spawn/a step/harvest threw, so a flaky run never leaks a
|
||||
// process or dir. `child` is undefined only if spawn itself threw.
|
||||
if (child !== undefined && child.exitCode === null && child.signalCode === null) {
|
||||
child.kill('SIGKILL')
|
||||
await waitForExit(child)
|
||||
}
|
||||
await rm(cwd, { recursive: true, force: true })
|
||||
await rm(sessionsRoot, { recursive: true, force: true })
|
||||
}
|
||||
|
||||
return {
|
||||
rawStdout: Buffer.concat(rawBuffers).toString('utf8'),
|
||||
stderr: stderrChunks.join(''),
|
||||
cwd,
|
||||
...sessionId !== undefined ? { sessionId } : {},
|
||||
sessionLogs,
|
||||
}
|
||||
}
|
||||
|
||||
/** Drive one input step over the client connection. */
|
||||
async function runStep(
|
||||
client: ClientSideConnection,
|
||||
step: InputStep,
|
||||
cwd: string,
|
||||
waitForUpdate: (match: (u: SessionNotification['update']) => boolean) => Promise<void>,
|
||||
getSessionId: () => string | undefined,
|
||||
setSessionId: (id: string) => void,
|
||||
): Promise<void> {
|
||||
switch (step.op) {
|
||||
case 'initialize':
|
||||
await client.initialize({
|
||||
protocolVersion: PROTOCOL_VERSION,
|
||||
clientCapabilities: step.terminalOutput === true ? { _meta: { terminal_output: true } } : {},
|
||||
})
|
||||
return
|
||||
case 'newSession': {
|
||||
const { sessionId } = await client.newSession({ cwd, mcpServers: [] })
|
||||
setSessionId(sessionId)
|
||||
return
|
||||
}
|
||||
case 'newSessionExpectError': {
|
||||
// The bridge rejects a session/new that widens the workspace scope
|
||||
// (non-empty additionalDirectories / mcpServers — unimplemented). The SDK
|
||||
// surfaces that as a rejected RPC; swallow it so the run completes and the
|
||||
// error frame is captured in the transcript.
|
||||
await client.newSession({
|
||||
cwd,
|
||||
mcpServers: [],
|
||||
...step.additionalDirectories !== undefined ? { additionalDirectories: step.additionalDirectories } : {},
|
||||
}).then(
|
||||
() => { throw new Error('snapshot-harness: expected session/new to be rejected but it succeeded') },
|
||||
() => { /* expected: the bridge rejected the unsupported workspace scope */ },
|
||||
)
|
||||
return
|
||||
}
|
||||
case 'prompt': {
|
||||
const sessionId = getSessionId()
|
||||
if (sessionId === undefined) throw new Error('snapshot-harness: prompt before newSession')
|
||||
await client.prompt({ sessionId, prompt: [{ type: 'text', text: step.text }] })
|
||||
return
|
||||
}
|
||||
case 'promptExpectError': {
|
||||
const sessionId = getSessionId()
|
||||
if (sessionId === undefined) throw new Error('snapshot-harness: promptExpectError before newSession')
|
||||
// The model fails this turn (a recorded provider error), so the bridge
|
||||
// answers the prompt with a JSON-RPC error and the SDK rejects. That
|
||||
// rejection IS the expected editor experience — swallow it so the run
|
||||
// completes and the stdout transcript (the error frame) is captured.
|
||||
await client.prompt({ sessionId, prompt: [{ type: 'text', text: step.text }] })
|
||||
.then(() => { throw new Error('snapshot-harness: expected the prompt to fail but it succeeded') },
|
||||
() => { /* expected: the turn failed and the bridge returned an error */ })
|
||||
return
|
||||
}
|
||||
case 'promptAndCancel': {
|
||||
const sessionId = getSessionId()
|
||||
if (sessionId === undefined) throw new Error('snapshot-harness: promptAndCancel before newSession')
|
||||
// Dispatch the prompt WITHOUT awaiting (a hang fixture never resolves on
|
||||
// its own). To pin frame order deterministically, wait until the client
|
||||
// has OBSERVED the hang's streamed agent_message_chunk before cancelling —
|
||||
// so those update frames always precede the cancelled prompt response in
|
||||
// the transcript (without this, the late chunk and the response race; see
|
||||
// the Codex review of commit 5). Then cancel and await the prompt, which
|
||||
// the bridge settles as `cancelled` once the abort propagates.
|
||||
const promptDone = client.prompt({ sessionId, prompt: [{ type: 'text', text: step.text }] })
|
||||
await waitForUpdate(u => u.sessionUpdate === 'agent_message_chunk')
|
||||
await client.cancel({ sessionId })
|
||||
await promptDone
|
||||
return
|
||||
}
|
||||
case 'cancel': {
|
||||
const sessionId = getSessionId()
|
||||
if (sessionId === undefined) throw new Error('snapshot-harness: cancel before newSession')
|
||||
await client.cancel({ sessionId })
|
||||
return
|
||||
}
|
||||
default:
|
||||
throw new Error(`snapshot-harness: unknown input op ${JSON.stringify(step)}`)
|
||||
}
|
||||
}
|
||||
|
||||
/** Resolve once the child process exits (any code/signal). */
|
||||
function waitForExit(child: ChildProcessWithoutNullStreams): Promise<void> {
|
||||
if (child.exitCode !== null || child.signalCode !== null) return Promise.resolve()
|
||||
return new Promise<void>(resolve => child.once('exit', () => { resolve() }))
|
||||
}
|
||||
|
||||
/**
|
||||
* Harvest EVERY persisted `.jsonl` session log under a sessions root, parse each
|
||||
* header line, and return them ordered primary-first: the top-level session (no
|
||||
* `parentSession`) leads, then each subagent child by ascending `createdAt`.
|
||||
*
|
||||
* The JSONL backend lays sessions out as `<root>/<cwd-bucket>/<encoded-id>.jsonl`
|
||||
* (one bucket per cwd), so a parent and its same-cwd in-process child land in
|
||||
* the SAME bucket — collecting all files across all buckets catches both (the
|
||||
* old first-match short-circuit silently dropped the child). Returns `[]` if no
|
||||
* log was produced (a no-session scenario).
|
||||
*/
|
||||
async function harvestSessionLogs(root: string): Promise<HarvestedLog[]> {
|
||||
let cwdDirs: string[]
|
||||
try {
|
||||
cwdDirs = await readdir(root)
|
||||
} catch {
|
||||
return []
|
||||
}
|
||||
const logs: HarvestedLog[] = []
|
||||
for (const dir of cwdDirs) {
|
||||
const sub = join(root, dir)
|
||||
let files: string[]
|
||||
try {
|
||||
files = await readdir(sub)
|
||||
} catch {
|
||||
continue
|
||||
}
|
||||
for (const f of files) {
|
||||
if (!f.endsWith('.jsonl')) continue
|
||||
const content = await readFile(join(sub, f), 'utf8')
|
||||
const firstLine = content.split('\n').find(line => line.trim().length > 0) ?? '{}'
|
||||
const header = JSON.parse(firstLine) as { id?: unknown; createdAt?: unknown; parentSession?: unknown }
|
||||
logs.push({
|
||||
id: typeof header.id === 'string' ? header.id : '',
|
||||
createdAt: typeof header.createdAt === 'number' ? header.createdAt : 0,
|
||||
...typeof header.parentSession === 'string' ? { parentSession: header.parentSession } : {},
|
||||
content,
|
||||
})
|
||||
}
|
||||
}
|
||||
// Primary (no parentSession) first, then children by ascending createdAt. A
|
||||
// scenario has exactly one top-level session. In the synchronous cut sibling
|
||||
// children are created strictly sequentially, so their createdAt values are
|
||||
// strictly ordered; the recordedId tiebreak only keeps a degenerate
|
||||
// same-millisecond collision (unreachable here) deterministic. This harvest
|
||||
// order must match the replay load order in dsh-llm-replay's loadSessionScripts
|
||||
// so session.<n>.jsonl maps to the same child on record and replay — replay
|
||||
// re-sorts childFiles by the same key, so the two stay consistent.
|
||||
logs.sort((a, b) => {
|
||||
const ap = a.parentSession === undefined ? 0 : 1
|
||||
const bp = b.parentSession === undefined ? 0 : 1
|
||||
return ap - bp || a.createdAt - b.createdAt || a.id.localeCompare(b.id)
|
||||
})
|
||||
return logs
|
||||
}
|
||||
@@ -1,185 +0,0 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { type NormalizeContext, normalizeSessionLog, normalizeStdout, scrubRequestHeaders } from '../tests/snapshot-normalize.ts'
|
||||
|
||||
/**
|
||||
* Unit tests for the pure snapshot normalizers. Live as a *.spec.ts (runs in
|
||||
* the default unit gate) and import the harness-side normalizers directly.
|
||||
*/
|
||||
|
||||
const ctx: NormalizeContext = {
|
||||
sessionIds: ['11111111-2222-3333-4444-555555555555'],
|
||||
cwd: '/tmp/acp-snap-cwd-abc123',
|
||||
}
|
||||
|
||||
describe('normalizeStdout', () => {
|
||||
it('rewrites JSON-RPC ids to a stable first-seen sequence', () => {
|
||||
const raw = [
|
||||
JSON.stringify({ jsonrpc: '2.0', id: 42, method: 'initialize' }),
|
||||
JSON.stringify({ jsonrpc: '2.0', id: 42, result: {} }),
|
||||
JSON.stringify({ jsonrpc: '2.0', id: 99, method: 'session/new' }),
|
||||
].join('\n')
|
||||
const out = normalizeStdout(raw, ctx)
|
||||
expect(out).toContain('"id":1')
|
||||
expect(out).toContain('"id":2')
|
||||
expect(out).not.toContain('42')
|
||||
expect(out).not.toContain('99')
|
||||
})
|
||||
|
||||
it('scrubs the cwd and session id anywhere they appear', () => {
|
||||
const raw = JSON.stringify({
|
||||
jsonrpc: '2.0', method: 'session/update',
|
||||
params: { sessionId: ctx.sessionIds[0], cwd: ctx.cwd, note: `at ${ctx.cwd}/x` },
|
||||
})
|
||||
const out = normalizeStdout(raw, ctx)
|
||||
expect(out).toContain('{{sessionId}}')
|
||||
expect(out).toContain('{{cwd}}')
|
||||
expect(out).not.toContain(ctx.cwd)
|
||||
expect(out).not.toContain(ctx.sessionIds[0] as string)
|
||||
})
|
||||
|
||||
it('scrubs a stray UUID not in the known list', () => {
|
||||
const raw = JSON.stringify({ jsonrpc: '2.0', method: 'x', params: { id: 'aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee' } })
|
||||
expect(normalizeStdout(raw, ctx)).toContain('{{sessionId}}')
|
||||
})
|
||||
|
||||
it('leaves notification frames without an id untouched in id-space', () => {
|
||||
const raw = JSON.stringify({ jsonrpc: '2.0', method: 'session/update', params: {} })
|
||||
const out = normalizeStdout(raw, ctx)
|
||||
expect(out).not.toContain('"id"')
|
||||
})
|
||||
|
||||
it('throws on a non-JSON stdout line (the purity check)', () => {
|
||||
const raw = `${JSON.stringify({ jsonrpc: '2.0', id: 1 })}\noops a log leaked\n`
|
||||
expect(() => normalizeStdout(raw, ctx)).toThrow()
|
||||
})
|
||||
|
||||
it('ignores blank lines', () => {
|
||||
const raw = `\n${JSON.stringify({ jsonrpc: '2.0', id: 1, method: 'm' })}\n\n`
|
||||
expect(() => normalizeStdout(raw, ctx)).not.toThrow()
|
||||
})
|
||||
})
|
||||
|
||||
describe('normalizeSessionLog', () => {
|
||||
const header = (over: object) => JSON.stringify({ type: 'session', version: 0, id: 's', createdAt: 123, ...over })
|
||||
const event = (over: object) => JSON.stringify({ type: 'turn/start', seq: 1, time: 999, data: { turn: 1 }, ...over })
|
||||
|
||||
it('zeroes the header createdAt', () => {
|
||||
const out = normalizeSessionLog(`${header({})}\n`, ctx)
|
||||
expect(out).toContain('"createdAt":0')
|
||||
expect(out).not.toContain('123')
|
||||
})
|
||||
|
||||
it('zeroes each event time but keeps seq', () => {
|
||||
const out = normalizeSessionLog(`${header({})}\n${event({ seq: 7, time: 999 })}\n`, ctx)
|
||||
expect(out).toContain('"time":0')
|
||||
expect(out).toContain('"seq":7') // seq is deterministic — NOT scrubbed
|
||||
expect(out).not.toContain('999')
|
||||
})
|
||||
|
||||
it('scrubs cwd and session id deep inside event data', () => {
|
||||
const ev = JSON.stringify({
|
||||
type: 'tool/result', seq: 2, time: 5,
|
||||
data: { content: [{ type: 'text', text: `wrote ${ctx.cwd}/proof.txt` }] },
|
||||
})
|
||||
const out = normalizeSessionLog(`${header({ cwd: ctx.cwd })}\n${ev}\n`, ctx)
|
||||
expect(out).toContain('{{cwd}}')
|
||||
expect(out).not.toContain(ctx.cwd)
|
||||
})
|
||||
|
||||
it('scrubs the session id in the header', () => {
|
||||
const out = normalizeSessionLog(`${header({ id: ctx.sessionIds[0] })}\n`, ctx)
|
||||
expect(out).toContain('{{sessionId}}')
|
||||
})
|
||||
|
||||
it('zeroes a hook/result durationMs (run-to-run noise) but keeps its decision', () => {
|
||||
const ev = JSON.stringify({
|
||||
type: 'hook/result', seq: 2, time: 5,
|
||||
data: { turn: 1, point: 'UserPromptSubmit', handlerId: 'h', decision: 'block', exitCode: 2, durationMs: 37 },
|
||||
})
|
||||
const out = normalizeSessionLog(`${header({})}\n${ev}\n`, ctx)
|
||||
expect(out).toContain('"durationMs":0')
|
||||
expect(out).not.toContain('37')
|
||||
expect(out).toContain('"decision":"block"') // the decision is the behavior — kept
|
||||
})
|
||||
|
||||
it('leaves a non-hook event durationMs untouched (only hook/result is scrubbed)', () => {
|
||||
const ev = JSON.stringify({ type: 'tool/result', seq: 2, time: 5, data: { durationMs: 88 } })
|
||||
const out = normalizeSessionLog(`${header({})}\n${ev}\n`, ctx)
|
||||
expect(out).toContain('"durationMs":88')
|
||||
})
|
||||
})
|
||||
|
||||
describe('scrubRequestHeaders', () => {
|
||||
const headerLine = JSON.stringify({ type: 'session', version: 0, id: 's', createdAt: 1, cwd: '/w' })
|
||||
const headerEvent = (header: object) =>
|
||||
JSON.stringify({ type: 'request/header', seq: 3, time: 9, data: { header, reason: 'initial' } })
|
||||
|
||||
it('replaces header system and tools with tokens, keeping config and reason', () => {
|
||||
const ev = headerEvent({
|
||||
config: { model: 'm' },
|
||||
system: 'You are an agent.\nBe brief.',
|
||||
tools: [{ name: 'read', description: 'Read a file.', parameters: { type: 'object' } }],
|
||||
})
|
||||
const out = scrubRequestHeaders(`${headerLine}\n${ev}\n`)
|
||||
expect(out).toContain('"system":"{{system}}"')
|
||||
expect(out).toContain('"tools":"{{tools}}"')
|
||||
expect(out).toContain('"config":{"model":"m"}')
|
||||
expect(out).toContain('"reason":"initial"')
|
||||
expect(out).not.toContain('You are an agent')
|
||||
expect(out).not.toContain('Read a file')
|
||||
})
|
||||
|
||||
it('keeps an absent system/tools absent (presence is behavior)', () => {
|
||||
const out = scrubRequestHeaders(`${headerLine}\n${headerEvent({ config: { model: 'm' } })}\n`)
|
||||
expect(out).not.toContain('{{system}}')
|
||||
expect(out).not.toContain('{{tools}}')
|
||||
})
|
||||
|
||||
it('scrubs a header-delta system payload but keeps its line positions and arity', () => {
|
||||
const delta = JSON.stringify({
|
||||
type: 'request/header-delta', seq: 8, time: 9,
|
||||
data: { system: { keepStart: 1, keepEnd: 4, insert: ['leaked prompt line', 'second line'] }, config: { model: 'm2' } },
|
||||
})
|
||||
const out = scrubRequestHeaders(`${headerLine}\n${delta}\n`)
|
||||
// One token PER inserted line: the edit's position AND extent survive.
|
||||
expect(out).toContain('"insert":["{{system}}","{{system}}"]')
|
||||
expect(out).toContain('"keepStart":1')
|
||||
expect(out).toContain('"keepEnd":4')
|
||||
expect(out).toContain('"config":{"model":"m2"}')
|
||||
expect(out).not.toContain('leaked prompt line')
|
||||
expect(out).not.toContain('{{tools}}') // no tools delta → none invented
|
||||
})
|
||||
|
||||
it('scrubs a header-delta tools payload but keeps the added/removed/changed names', () => {
|
||||
const delta = JSON.stringify({
|
||||
type: 'request/header-delta', seq: 8, time: 9,
|
||||
data: {
|
||||
tools: {
|
||||
added: [{ name: 'grep', description: 'Search files.', parameters: { type: 'object' } }],
|
||||
removed: ['bash_kill'],
|
||||
changed: [{ name: 'read', description: 'Read v2.', parameters: { type: 'object' } }],
|
||||
},
|
||||
},
|
||||
})
|
||||
const out = scrubRequestHeaders(`${headerLine}\n${delta}\n`)
|
||||
// WHICH tools changed is behavior and survives; their bulk does not.
|
||||
expect(out).toContain('"added":[{"name":"grep","description":"{{tools}}","parameters":"{{tools}}"}]')
|
||||
expect(out).toContain('"removed":["bash_kill"]')
|
||||
expect(out).toContain('"changed":[{"name":"read","description":"{{tools}}","parameters":"{{tools}}"}]')
|
||||
expect(out).not.toContain('Search files')
|
||||
expect(out).not.toContain('Read v2')
|
||||
})
|
||||
|
||||
it('passes every other line through byte-for-byte and is idempotent', () => {
|
||||
const other = JSON.stringify({ type: 'assistant/chunk', seq: 4, time: 9, data: { turn: 1, step: 1, chunk: { type: 'text-delta', index: 0, text: 'hi' } } })
|
||||
const delta = JSON.stringify({
|
||||
type: 'request/header-delta', seq: 8, time: 9,
|
||||
data: { system: { keepStart: 0, keepEnd: 0, insert: ['x'] }, tools: { added: [{ name: 't', description: 'd', parameters: {} }], removed: [], changed: [] } },
|
||||
})
|
||||
const raw = `${headerLine}\n${headerEvent({ config: { model: 'm' }, system: 's', tools: [] })}\n${delta}\n${other}\n`
|
||||
const once = scrubRequestHeaders(raw)
|
||||
expect(once.split('\n')[0]).toBe(headerLine)
|
||||
expect(once.split('\n')[3]).toBe(other)
|
||||
expect(scrubRequestHeaders(once)).toBe(once)
|
||||
})
|
||||
})
|
||||
@@ -1,185 +0,0 @@
|
||||
/**
|
||||
* Pure normalizers for the ACP snapshot goldens. They replace the
|
||||
* non-deterministic values in the two captured surfaces — the stdout JSON-RPC
|
||||
* transcript and the persisted session JSONL — with stable tokens, so a golden
|
||||
* compare reflects behavior, not run-to-run noise. Kept dependency-free and
|
||||
* side-effect-free so they unit-test trivially.
|
||||
*
|
||||
* Scrubbed: `randomUUID()` session ids → `{{sessionId}}`; the temp `mkdtemp`
|
||||
* cwd → `{{cwd}}` (it appears in terminal-card `_meta` and the log header);
|
||||
* JSON-RPC request `id` → a stable per-transcript sequence; the log's per-event
|
||||
* `time` (epoch ms) and header `createdAt` → 0; a `hook/result` event's
|
||||
* `durationMs` (wall-clock hook runtime) → 0. NOT scrubbed: the log's `seq`
|
||||
* (deterministic — `seq = log.length`, part of the event-log contract).
|
||||
*
|
||||
* A separate, composable normalizer — {@link scrubRequestHeaders} — replaces
|
||||
* the bulky request-header CONTENT (the composed system prompt and the tool
|
||||
* schema list) with `{{system}}`/`{{tools}}` tokens. It is deliberately NOT
|
||||
* folded into {@link normalizeSessionLog}: the one header-pinning scenario
|
||||
* compares that content verbatim, every other scenario composes the scrub in
|
||||
* (the `pinsHeader` flag in acp.snapshot.ts; see the pinned-header RFC,
|
||||
* docs/rfc/implemented/testing/2026-07-06-pin-request-header-content-in-one-scenario.md).
|
||||
*
|
||||
* See docs/rfc/implemented/testing/2026-06-19-acp-snapshot-tests.md.
|
||||
*/
|
||||
|
||||
const SESSION_ID = '{{sessionId}}'
|
||||
const CWD = '{{cwd}}'
|
||||
const SYSTEM = '{{system}}'
|
||||
const TOOLS = '{{tools}}'
|
||||
|
||||
/** A UUID v4 string, the shape `randomUUID()` produces for session ids. */
|
||||
const UUID_RE = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/gi
|
||||
|
||||
/** Inputs the normalizers need to recognize a run's volatile values. */
|
||||
export interface NormalizeContext {
|
||||
/** The session id(s) the run issued — replaced with `{{sessionId}}`. */
|
||||
sessionIds: string[]
|
||||
/** The temp cwd the run used — replaced with `{{cwd}}`. */
|
||||
cwd: string
|
||||
}
|
||||
|
||||
/** Replace cwd, session ids, and any stray UUID with stable tokens in a string. */
|
||||
function scrubString(value: string, ctx: NormalizeContext): string {
|
||||
let out = value
|
||||
// cwd first (longest, most specific), then explicit session ids, then any
|
||||
// residual UUID (covers ids that appear in places we didn't enumerate).
|
||||
out = out.split(ctx.cwd).join(CWD)
|
||||
for (const id of ctx.sessionIds) out = out.split(id).join(SESSION_ID)
|
||||
out = out.replace(UUID_RE, SESSION_ID)
|
||||
return out
|
||||
}
|
||||
|
||||
/** Recursively scrub a parsed JSON value (strings replaced; structure kept). */
|
||||
function scrubValue(value: unknown, ctx: NormalizeContext): unknown {
|
||||
if (typeof value === 'string') return scrubString(value, ctx)
|
||||
if (Array.isArray(value)) return value.map(v => scrubValue(v, ctx))
|
||||
if (value !== null && typeof value === 'object') {
|
||||
const out: Record<string, unknown> = {}
|
||||
for (const [k, v] of Object.entries(value)) out[k] = scrubValue(v, ctx)
|
||||
return out
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize a raw stdout transcript (newline-delimited JSON-RPC frames) into a
|
||||
* stable golden in the SAME shape as the wire: one compact JSON frame per line
|
||||
* (NDJSON), with the JSON-RPC `id` rewritten to a per-transcript sequence
|
||||
* (1, 2, 3, …) and all volatile strings scrubbed. Throws if any non-empty line
|
||||
* is not valid JSON — that doubles as the stdout-purity check (no logger leaked
|
||||
* onto the protocol).
|
||||
*/
|
||||
export function normalizeStdout(rawStdout: string, ctx: NormalizeContext): string {
|
||||
const lines = rawStdout.split('\n').filter(line => line.trim().length > 0)
|
||||
// Map each distinct JSON-RPC id (request/response correlate by id) to a stable
|
||||
// sequence number, in first-seen order, so id churn doesn't perturb the golden.
|
||||
const idSeq = new Map<string, number>()
|
||||
const stableId = (id: unknown): number => {
|
||||
const key = JSON.stringify(id)
|
||||
let n = idSeq.get(key)
|
||||
if (n === undefined) { n = idSeq.size + 1; idSeq.set(key, n) }
|
||||
return n
|
||||
}
|
||||
const frames = lines.map((line) => {
|
||||
const frame = JSON.parse(line) as Record<string, unknown>
|
||||
if ('id' in frame && frame.id !== undefined && frame.id !== null) {
|
||||
frame.id = stableId(frame.id)
|
||||
}
|
||||
return scrubValue(frame, ctx) as Record<string, unknown>
|
||||
})
|
||||
return frames.map(f => JSON.stringify(f)).join('\n') + '\n'
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize a session JSONL log into a stable golden: the header line's
|
||||
* volatile fields (`createdAt`, `id`, `cwd`) and every event's `time` are
|
||||
* zeroed/scrubbed, all volatile strings scrubbed, and `seq` is LEFT INTACT
|
||||
* (deterministic by contract). Output is JSONL in the same shape as the input —
|
||||
* one compact record per line.
|
||||
*/
|
||||
export function normalizeSessionLog(rawLog: string, ctx: NormalizeContext): string {
|
||||
const lines = rawLog.split('\n').filter(line => line.trim().length > 0)
|
||||
const records = lines.map((line) => {
|
||||
const record = JSON.parse(line) as Record<string, unknown>
|
||||
// Header line: { type: 'session', createdAt, id, cwd, … }.
|
||||
if (record.type === 'session') {
|
||||
if ('createdAt' in record) record.createdAt = 0
|
||||
} else if ('time' in record) {
|
||||
// Event line: zero the epoch-ms timestamp; keep seq (deterministic).
|
||||
record.time = 0
|
||||
// A hook/result carries the hook's wall-clock runtime (`data.durationMs`),
|
||||
// which is run-to-run noise like `time` — zero it so the golden reflects
|
||||
// the hook's decision/exit, not how long the shell took.
|
||||
if (record.type === 'hook/result' && record.data !== null && typeof record.data === 'object') {
|
||||
const data = record.data as Record<string, unknown>
|
||||
if ('durationMs' in data) data.durationMs = 0
|
||||
}
|
||||
}
|
||||
return scrubValue(record, ctx) as Record<string, unknown>
|
||||
})
|
||||
return records.map(r => JSON.stringify(r)).join('\n') + '\n'
|
||||
}
|
||||
|
||||
/**
|
||||
* Replace request-header CONTENT in a session JSONL with stable tokens,
|
||||
* keeping its structure: a `request/header` event's `data.header.system` →
|
||||
* `{{system}}` and `data.header.tools` → `{{tools}}`; a
|
||||
* `request/header-delta` event keeps every structural fact — the system
|
||||
* delta's `keepStart`/`keepEnd` line positions and inserted-line COUNT (one
|
||||
* `{{system}}` token per inserted line), the tools delta's
|
||||
* added/removed/changed tool NAMES — and tokenizes only the bulk (prompt
|
||||
* text; each added/changed schema's fields other than `name` → `{{tools}}`),
|
||||
* so two different deltas still compare different.
|
||||
* Absent fields stay absent — WHETHER a header carried a system prompt or
|
||||
* tools is behavior and stays visible; `config` and `reason` are small and
|
||||
* stable, so they stay verbatim (a model swap churns every fixture by design
|
||||
* — it invalidates the recorded responses; a prompt/schema edit churns none —
|
||||
* replay never reads this content, see dsh-llm-replay).
|
||||
*
|
||||
* Only lines with something to scrub are re-serialized; every other line
|
||||
* passes through byte-for-byte, so the transform is idempotent and applying
|
||||
* it to an already-scrubbed fixture is a no-op — the on-disk-fixtures guard
|
||||
* in acp.snapshot.ts relies on exactly that.
|
||||
*/
|
||||
export function scrubRequestHeaders(rawLog: string): string {
|
||||
const lines = rawLog.split('\n')
|
||||
const out = lines.map((line) => {
|
||||
if (line.trim().length === 0) return line
|
||||
const record = JSON.parse(line) as Record<string, unknown>
|
||||
const data = record.data as Record<string, unknown> | null | undefined
|
||||
if (data === null || typeof data !== 'object') return line
|
||||
if (record.type === 'request/header') {
|
||||
const header = data.header as Record<string, unknown> | null | undefined
|
||||
if (header === null || typeof header !== 'object') return line
|
||||
if (!('system' in header) && !('tools' in header)) return line
|
||||
if ('system' in header) header.system = SYSTEM
|
||||
if ('tools' in header) header.tools = TOOLS
|
||||
return JSON.stringify(record)
|
||||
}
|
||||
if (record.type === 'request/header-delta') {
|
||||
let touched = false
|
||||
const system = data.system as Record<string, unknown> | null | undefined
|
||||
if (system !== null && typeof system === 'object' && Array.isArray(system.insert)) {
|
||||
system.insert = system.insert.map(() => SYSTEM)
|
||||
touched = true
|
||||
}
|
||||
const tools = data.tools as Record<string, unknown> | null | undefined
|
||||
if (tools !== null && typeof tools === 'object') {
|
||||
if (Array.isArray(tools.added)) { tools.added = tools.added.map(scrubToolSchema); touched = true }
|
||||
if (Array.isArray(tools.changed)) { tools.changed = tools.changed.map(scrubToolSchema); touched = true }
|
||||
}
|
||||
return touched ? JSON.stringify(record) : line
|
||||
}
|
||||
return line
|
||||
})
|
||||
return out.join('\n')
|
||||
}
|
||||
|
||||
/** Tokenize one tool schema's bulk (description, parameters, anything else), keeping its identifying `name`. */
|
||||
function scrubToolSchema(tool: unknown): unknown {
|
||||
if (tool === null || typeof tool !== 'object' || Array.isArray(tool)) return tool
|
||||
const out: Record<string, unknown> = {}
|
||||
for (const [k, v] of Object.entries(tool)) out[k] = k === 'name' ? v : TOOLS
|
||||
return out
|
||||
}
|
||||
Reference in New Issue
Block a user