Merge branch 'worktree-session-modes-rfc' (master: sandbox stack #169, skills #109, prompt snapshots #254)

The stack rebases onto a moved master through its base branch. Beyond
mechanical unions (both branches' demo scripts, example rows, service
roles, tool lists, acp deps, doc budgets — each side fit alone, the
union needs the higher ceilings), three semantic reconciliations:

- The ACP bridge now carries BOTH per-session surfaces: the sandbox
  stack's config options + approval answerer and this branch's session
  modes; session/new and session/load advertise modes AND configOptions
  side by side.
- The feature matrix supersedes the sandbox stance per the RFC's
  second-lander rule: session/set_mode and current_mode_update flip to
  shipped-by-dsh-mode, config-option rows stay as #169 wrote them, and
  §6 records both landed features under the picker-to-modes /
  knobs-to-config-options division.
- The snapshot pin grammar (#254: one header snapshot + declared deltas
  + a Markdown prompt golden) gains a symmetric declaration for what a
  delta cannot express: expectedHeaderSnapshots — a plan-mode flip
  resorts the canonical tool list, so its widening lands as a second
  full snapshot, now its own Markdown section. The pin-less-class and
  model-turn-only-pin amendments carry over; new fixtures cover the
  extended writer paths, and the plan-acp-agent scenarios re-recorded
  under the merged composition (the app now bundles the skill tool)
  with the suite's refresh mode wired through.
This commit is contained in:
kingwl
2026-07-12 20:08:00 +08:00
341 changed files with 27410 additions and 7518 deletions

View File

@@ -89,6 +89,8 @@ export type InputStep =
| { op: 'cancel' }
| { op: 'setMode'; modeId: string }
| { op: 'setModeExpectError'; modeId: string }
| { op: 'setConfigOption'; configId: string; value: string }
| { op: 'setConfigOptionExpectError'; configId: string; value: string }
/** A scenario's `input.json`: an ordered list of input steps. */
export interface InputScript {
@@ -234,6 +236,8 @@ export async function runScenario(input: InputScript, opts: RunOptions): Promise
DSH_SNAPSHOT: opts.mode,
DSH_SNAPSHOT_FILE: opts.fixtureFile,
DSH_SNAPSHOT_SESSIONS_ROOT: sessionsRoot,
DSH_HOME: join(cwd, '.dsh'),
DSH_AGENTS_HOME: join(cwd, '.agents'),
...opts.overrideFile !== undefined ? { DSH_SNAPSHOT_OVERRIDE: opts.overrideFile } : {},
...opts.childFiles !== undefined && opts.childFiles.length > 0
? { DSH_SNAPSHOT_CHILD_FILES: opts.childFiles.join(delimiter) }
@@ -461,6 +465,24 @@ async function runStep(
)
return
}
case 'setConfigOption': {
const sessionId = getSessionId()
if (sessionId === undefined) throw new Error('snapshot-harness: setConfigOption before newSession')
await client.setSessionConfigOption({ sessionId, configId: step.configId, value: step.value })
return
}
case 'setConfigOptionExpectError': {
const sessionId = getSessionId()
if (sessionId === undefined) throw new Error('snapshot-harness: setConfigOptionExpectError before newSession')
// The bridge rejects an unknown id / out-of-vocabulary value; the SDK
// surfaces that as a rejected RPC — swallow it so the run completes and
// the error frame is captured in the transcript.
await client.setSessionConfigOption({ sessionId, configId: step.configId, value: step.value }).then(
() => { throw new Error('snapshot-harness: expected set_config_option to be rejected but it succeeded') },
() => { /* expected: the bridge rejected the id or value */ },
)
return
}
default:
throw new Error(`snapshot-harness: unknown input op ${JSON.stringify(step)}`)
}

View File

@@ -3,7 +3,7 @@
* tier (`pnpm run test:snapshot`). Three layers, composable per example:
* the subprocess scenario harness ({@link runScenario}), the pure golden
* normalizers ({@link normalizeStdout} / {@link normalizeSessionLog} /
* {@link scrubRequestHeaders}), and the suite factory
* {@link scrubRequestHeaders} / {@link scrubSystemPrompts}), and the suite factory
* ({@link defineAcpSnapshotSuite}) that registers a scenario table as a full
* describe/it tree. An example's `*.snapshot.ts` supplies only its
* {@link AgentUnderTest} paths, its snapshots directory, and its
@@ -30,6 +30,7 @@ export {
normalizeSessionLog,
normalizeStdout,
scrubRequestHeaders,
scrubSystemPrompts,
type NormalizeContext,
} from './normalize.ts'
export {

View File

@@ -12,14 +12,14 @@
* `durationMs` (wall-clock hook runtime) → 0. NOT scrubbed: the log's `seq`
* (deterministic — `seq = log.length`, part of the event-log contract).
*
* A separate, composable normalizer — {@link scrubRequestHeaders} — replaces
* the bulky request-header CONTENT (the composed system prompt, the tool
* schema list, and the session prefix) with
* `{{system}}`/`{{tools}}`/`{{messagePrefix}}` tokens. It is deliberately NOT
* folded into {@link normalizeSessionLog}: each suite's one header-pinning
* scenario compares that content verbatim, every other scenario composes the
* scrub in (the `pinsHeader` flag on the scenario table, consumed by the suite
* factory in ./suite.ts; see the pinned-header RFC,
* Separate, composable normalizers keep bulky request-header content out of
* session fixtures. {@link scrubSystemPrompts} replaces the composed system
* prompt in EVERY fixture; {@link scrubRequestHeaders} additionally replaces
* tool schemas and the session prefix outside each suite's header-pinning
* scenario. They are deliberately NOT folded into
* {@link normalizeSessionLog}: the suite factory composes the right scrub for
* each scenario and snapshots the pin's actual prompt as Markdown (see the
* pinned-header RFC,
* docs/rfc/implemented/testing/2026-07-06-pin-request-header-content-in-one-scenario.md).
*
* See docs/rfc/implemented/testing/2026-06-19-acp-snapshot-tests.md.
@@ -135,37 +135,37 @@ export function normalizeSessionLog(rawLog: string, ctx: NormalizeContext): stri
}
/**
* Replace request-header CONTENT in a session JSONL with stable tokens,
* keeping its structure: a `request/header` event's `data.header.system` →
* `{{system}}`, `data.header.tools` → `{{tools}}`, and
* `data.header.messagePrefix` → one `{{messagePrefix}}` token per message
* (the session prefix is model-visible bulk — an AGENTS digest, a skills
* catalog — so its COUNT stays a structural fact while its text never lands
* in a fixture); a
* `request/header-delta` event keeps every structural fact — the system
* delta's `keepStart`/`keepEnd` line positions and inserted-line COUNT (one
* `{{system}}` token per inserted line), the tools delta's
* added/removed/changed tool NAMES, the prefix replacement's message COUNT —
* and tokenizes only the bulk (prompt
* text; each added/changed schema's fields other than `name` → `{{tools}}`;
* each replacement prefix message → `{{messagePrefix}}`),
* so two different deltas still compare different.
* Absent fields stay absent — WHETHER a header carried a system prompt,
* tools, or a prefix is behavior and stays visible; `config` and `reason`
* are small and
* stable, so they stay verbatim (a model swap churns every fixture by design
* — it invalidates the recorded responses; a prompt/schema edit churns none —
* replay never reads this content, see dsh-llm-replay).
*
* Only lines with something to scrub are re-serialized; every other line
* passes through byte-for-byte, so the transform is idempotent and applying
* it to an already-scrubbed fixture is a no-op — the on-disk-fixtures guard
* in ./suite.ts relies on exactly that.
* Replace system-prompt content in request headers and header deltas with
* `{{system}}` tokens while retaining field presence and delta structure.
* Other header content stays verbatim, so a header-pinning fixture can keep
* its complete tool schemas while every JSONL fixture omits the prompt text.
* Lines without a system payload pass through byte-for-byte; the transform is
* idempotent.
*
* @param rawLog The raw session `.jsonl` content.
* @returns The JSONL with header content tokenized, other lines byte-identical.
* @returns The JSONL with system-prompt content tokenized.
*/
export function scrubSystemPrompts(rawLog: string): string {
return scrubHeaderContent(rawLog, false)
}
/**
* Replace all bulky request-header content in a session JSONL with stable
* tokens. This includes the system-prompt fields handled by
* {@link scrubSystemPrompts}, tool schemas, and session-prefix messages. It
* keeps system-delta line positions and arity, tool-delta names, prefix
* message counts, field presence, config, and reason. Lines without content
* to scrub pass through byte-for-byte, and the transform is idempotent.
*
* @param rawLog The raw session `.jsonl` content.
* @returns The JSONL with all header bulk tokenized, other lines byte-identical.
*/
export function scrubRequestHeaders(rawLog: string): string {
return scrubHeaderContent(rawLog, true)
}
/** Transform header content, optionally including tool schemas and the session prefix. */
function scrubHeaderContent(rawLog: string, scrubToolsAndPrefix: boolean): string {
const lines = rawLog.split('\n')
const out = lines.map((line) => {
if (line.trim().length === 0) return line
@@ -175,11 +175,14 @@ export function scrubRequestHeaders(rawLog: string): string {
if (record.type === 'request/header') {
const header = data.header as Record<string, unknown> | null | undefined
if (header === null || typeof header !== 'object') return line
if (!('system' in header) && !('tools' in header) && !('messagePrefix' in header)) return line
if ('system' in header) header.system = SYSTEM
if ('tools' in header) header.tools = TOOLS
if (Array.isArray(header.messagePrefix)) header.messagePrefix = header.messagePrefix.map(() => MESSAGE_PREFIX)
return JSON.stringify(record)
let touched = false
if ('system' in header) { header.system = SYSTEM; touched = true }
if (scrubToolsAndPrefix && 'tools' in header) { header.tools = TOOLS; touched = true }
if (scrubToolsAndPrefix && Array.isArray(header.messagePrefix)) {
header.messagePrefix = header.messagePrefix.map(() => MESSAGE_PREFIX)
touched = true
}
return touched ? JSON.stringify(record) : line
}
if (record.type === 'request/header-delta') {
let touched = false
@@ -189,11 +192,11 @@ export function scrubRequestHeaders(rawLog: string): string {
touched = true
}
const tools = data.tools as Record<string, unknown> | null | undefined
if (tools !== null && typeof tools === 'object') {
if (scrubToolsAndPrefix && tools !== null && typeof tools === 'object') {
if (Array.isArray(tools.added)) { tools.added = tools.added.map(scrubToolSchema); touched = true }
if (Array.isArray(tools.changed)) { tools.changed = tools.changed.map(scrubToolSchema); touched = true }
}
if (Array.isArray(data.messagePrefix)) {
if (scrubToolsAndPrefix && Array.isArray(data.messagePrefix)) {
data.messagePrefix = data.messagePrefix.map(() => MESSAGE_PREFIX)
touched = true
}

View File

@@ -10,21 +10,23 @@
* (recorded scenarios) and the expected produced log (both sides normalized
* before comparing).
*
* Request-header content (the composed system prompt + tool schemas riding on
* `request/header` events) is pinned by exactly ONE scenario per HEADER CLASS
* — scenarios that boot the same config compose the same header; each class's
* `pinsHeader` scenario commits it verbatim — and scrubbed to
* `{{system}}`/`{{tools}}` tokens in every other fixture and compare, so a
* prompt or tool-schema edit churns one committed line per class instead of
* every fixture. A per-run uniformity guard keeps each pin sound: every live
* header must equal its class's pinned one, and no header-delta may appear
* outside a pinning scenario (see the pinned-header RFC,
* Request-header content is pinned by exactly ONE scenario per HEADER CLASS —
* scenarios that boot the same config compose the same header. Every JSONL
* fixture scrubs the system prompt to `{{system}}`; each class's pinning
* scenario stores the readable prompt in `system-prompt.golden.md` and keeps its full
* tool schemas in `session.jsonl`, while every other fixture also scrubs tools
* to `{{tools}}`. A per-run uniformity guard compares both artifacts against
* every live header and forbids unrepresented header deltas (see the
* pinned-header RFC,
* docs/rfc/implemented/testing/2026-07-06-pin-request-header-content-in-one-scenario.md).
*
* `pnpm run test:snapshot:record` (DSH_SNAPSHOT=record + -u) re-records the
* `session.jsonl` fixtures against the real API and refreshes the stdout golden
* in one pass; the caller resolves that env into {@link SnapshotSuiteOptions}
* (env reading stays at the suite edge, not in this library).
* in one pass. `pnpm run test:snapshot:refresh` (DSH_SNAPSHOT=refresh) instead
* replays the committed model scripts keylessly and writes the current stdout
* + persisted-log goldens back without calling a live LLM. The caller resolves
* that env into {@link SnapshotSuiteOptions} (env reading stays at the suite
* edge, not in this library).
*
* @module @deepseek-ai/dsh-acp-snapshot/suite
*/
@@ -34,7 +36,16 @@ import { existsSync } from 'node:fs'
import { join } from 'node:path'
import { describe, expect, it } from 'vitest'
import { type AgentUnderTest, type HarvestedLog, type InputScript, runScenario } from './harness.ts'
import { type NormalizeContext, normalizeSessionLog, normalizeStdout, scrubRequestHeaders } from './normalize.ts'
import {
type NormalizeContext,
normalizeSessionLog,
normalizeStdout,
scrubRequestHeaders,
scrubSystemPrompts,
} from './normalize.ts'
/** The readable system-prompt snapshot beside each header-pinning fixture. */
const SYSTEM_PROMPT_SNAPSHOT = 'system-prompt.golden.md'
/** A snapshot scenario and how its fixtures are produced. */
export interface Scenario {
@@ -78,14 +89,13 @@ export interface Scenario {
*/
childSessions?: number
/**
* Whether THIS scenario's fixtures keep the full request-header content (the
* composed system prompt and tool schema list on `request/header` /
* `request/header-delta` events) and compare it verbatim. Exactly one
* scenario per HEADER CLASS ({@link headerClass}) pins it; every other
* scenario of that class stores and compares that content as
* `{{system}}`/`{{tools}}` tokens ({@link scrubRequestHeaders}),
* so a system prompt or tool-schema change shows up as ONE committed-fixture
* diff per class, not one per scenario. One pin per class suffices because
* Whether THIS scenario pins its header class's model-facing request-header
* content. Its actual composed prompt is maintained as a readable
* `system-prompt.golden.md`; its JSONL keeps full tool schemas but stores the prompt
* as `{{system}}`. Every other scenario of the class stores tools as
* `{{tools}}` too ({@link scrubRequestHeaders}). A prompt or tool-schema
* change therefore shows up in one focused artifact per class, not every
* session fixture. One pin per class suffices because
* header composition is class-uniform (parent, spawn child, and fork child
* all compose the same prompt-modulo-cwd and the same tools) — and that
* premise is ASSERTED, not assumed: every non-pinning run's live headers
@@ -95,6 +105,24 @@ export interface Scenario {
* Defaults to false.
*/
pinsHeader?: boolean
/**
* How many `request/header-delta` events this PINNING scenario's fixture
* legitimately carries (default 0). A recorded mid-run header change — a
* config-option switch rewriting a prompt section — is part of the pinned
* surface, with readable prompt text in Markdown; any OTHER count
* still fails, so fixture rot stays caught. Meaningless off the pin (the
* live uniformity guard keeps non-pinning scenarios delta-free).
*/
expectedHeaderDeltas?: number
/**
* How many `request/header` SNAPSHOTS the pinning fixture legitimately
* carries (default 1). A mid-run header change the delta encoding cannot
* express — a session-mode flip resorts the canonical tool list, and a pure
* reordering has no delta form — lands as a further full snapshot; the
* Markdown prompt golden then carries each snapshot's prompt as its own
* section. Meaningless off the pin (non-pinning scenarios stay single-shape).
*/
expectedHeaderSnapshots?: number
/**
* Which header-composition class this scenario belongs to. Scenarios that
* boot the same config compose the same header; each class has exactly one
@@ -121,15 +149,16 @@ export interface SnapshotSuiteOptions {
agent: AgentUnderTest
/** Absolute path of the suite's `snapshots/` directory (one subdir per scenario). */
snapshotsDir: string
/** The scenario table; exactly one entry must set `pinsHeader`. */
/** The scenario table; exactly one entry per header class must set `pinsHeader`. */
scenarios: Scenario[]
/**
* `replay` (keyless, the default tier) or `record` (live API; re-records the
* `recorded` scenarios' fixtures and refreshes the vitest goldens under
* `--update`). The caller derives this from `$DSH_SNAPSHOT` — env reading
* stays outside this library.
* `replay` (keyless, the default tier), `record` (live API; re-records the
* `recorded` scenarios' fixtures and refreshes the Vitest goldens under
* `--update`), or `refresh` (keyless replay that rewrites stdout goldens and
* comparable session fixtures from the replay run). The caller derives this
* from `$DSH_SNAPSHOT` — env reading stays outside this library.
*/
mode: 'replay' | 'record'
mode: 'replay' | 'record' | 'refresh'
}
/**
@@ -188,6 +217,94 @@ export function normalizedHeaders(rawLog: string, ctx: NormalizeContext): unknow
.map(record => record.data?.header)
}
/**
* The normalized string-valued system prompts carried by request headers in a
* session JSONL, in log order. Headers without a string prompt are omitted so
* callers can assert one prompt per header explicitly.
*
* @param rawLog The session `.jsonl` content to inspect.
* @param ctx The volatile values of the run that produced it.
* @returns The normalized system prompts, in header order.
*/
export function normalizedSystemPrompts(rawLog: string, ctx: NormalizeContext): string[] {
return normalizedHeaders(rawLog, ctx).flatMap((header) => {
if (header === null || typeof header !== 'object') return []
const system = (header as { system?: unknown }).system
return typeof system === 'string' ? [system] : []
})
}
/** One normalized system-prompt edit carried by a `request/header-delta`. */
export interface SystemPromptDeltaSnapshot {
/** How many leading lines remain from the prior prompt. */
keepStart: number
/** How many trailing lines remain from the prior prompt. */
keepEnd: number
/** The normalized replacement lines inserted between the retained ranges. */
insert: string[]
}
/**
* Extract normalized system-prompt edits from request-header deltas in log
* order. Deltas without a well-formed system edit are omitted; their non-prompt
* structure remains pinned in JSONL.
*
* @param rawLog The session `.jsonl` content to inspect.
* @param ctx The volatile values of the run that produced it.
* @returns The normalized system-prompt edits, in event order.
*/
export function normalizedSystemPromptDeltas(rawLog: string, ctx: NormalizeContext): SystemPromptDeltaSnapshot[] {
return normalizeSessionLog(rawLog, ctx)
.split('\n')
.filter(line => line.trim().length > 0)
.map(line => JSON.parse(line) as { type?: unknown; data?: { system?: unknown } })
.filter(record => record.type === 'request/header-delta')
.flatMap((record) => {
const system = record.data?.system
if (system === null || typeof system !== 'object') return []
const { keepStart, keepEnd, insert } = system as { keepStart?: unknown; keepEnd?: unknown; insert?: unknown }
if (typeof keepStart !== 'number' || typeof keepEnd !== 'number' || !Array.isArray(insert)) return []
if (!insert.every(line => typeof line === 'string')) return []
return [{ keepStart, keepEnd, insert: insert }]
})
}
/**
* Render a normalized prompt as a repository-friendly Markdown snapshot.
* Prompt text is unchanged except that a missing terminal newline is added so
* the committed file follows the repository newline contract.
*
* @param prompt The normalized system prompt.
* @param deltas Normalized prompt edits to append as readable sections.
* @param furtherSnapshots Later full header snapshots' prompts (transitions
* the delta encoding cannot express), each rendered as its own section
* before the delta sections.
* @returns Markdown snapshot text ending in a newline.
*/
export function formatSystemPromptSnapshot(
prompt: string,
deltas: readonly SystemPromptDeltaSnapshot[] = [],
furtherSnapshots: readonly string[] = [],
): string {
let snapshot = prompt.endsWith('\n') ? prompt : `${prompt}\n`
for (const [index, further] of furtherSnapshots.entries()) {
snapshot += `\n<!-- request/header ${index + 2} (snapshot) -->\n\n`
snapshot += further.endsWith('\n') ? further : `${further}\n`
}
for (const [index, delta] of deltas.entries()) {
snapshot += `\n<!-- request/header-delta ${index + 1}: keepStart=${delta.keepStart}, keepEnd=${delta.keepEnd} -->\n\n`
const insert = delta.insert.join('\n')
snapshot += insert.endsWith('\n') ? insert : `${insert}\n`
}
return snapshot
}
/** Return the initial-prompt portion of a possibly delta-bearing snapshot. */
function initialSystemPromptSnapshot(snapshot: string): string {
const marker = snapshot.indexOf('\n<!-- request/header')
return marker < 0 ? snapshot : snapshot.slice(0, marker)
}
/**
* Count the `request/header-delta` events in a session JSONL.
*
@@ -201,11 +318,92 @@ export function headerDeltaCount(rawLog: string): number {
.length
}
/** A literal string replacement used to carry an existing fixture's volatile value into a refreshed log. */
export interface FixtureReplacement {
/** The fresh replay-run value to replace. */
from: string
/** The existing fixture value to keep. */
to: string
}
function parseJsonlRecords(text: string): Record<string, unknown>[] {
return text.split('\n')
.filter(line => line.trim().length > 0)
.map(line => JSON.parse(line) as Record<string, unknown>)
}
/**
* Build the cross-log id/cwd replacements used by refresh write-back.
*
* @param logs The freshly harvested logs, in fixture order.
* @param fixtures The existing fixture contents, in matching order.
* @returns Literal replacements from fresh volatile values to the fixture's old values.
*/
export function refreshFixtureReplacements(logs: HarvestedLog[], fixtures: string[]): FixtureReplacement[] {
const replacements: FixtureReplacement[] = []
for (let i = 0; i < logs.length; i++) {
const fresh = parseJsonlRecords((logs[i] as HarvestedLog).content)[0]
const existing = parseJsonlRecords(fixtures[i] ?? '')[0]
for (const field of ['id', 'cwd'] as const) {
const from = fresh?.[field]
const to = existing?.[field]
if (typeof from === 'string' && typeof to === 'string' && from.length > 0 && from !== to) {
replacements.push({ from, to })
}
}
}
return replacements
}
function preserveFixtureVolatiles(record: Record<string, unknown>, existing: Record<string, unknown> | undefined): void {
if (existing === undefined || existing.type !== record.type) return
if (record.type === 'session') {
for (const field of ['id', 'createdAt', 'cwd', 'parentSession'] as const) {
if (field in record && field in existing) record[field] = existing[field]
}
return
}
if ('time' in record && 'time' in existing) record.time = existing.time
if (record.type !== 'hook/result') return
const data = record.data
const existingData = existing.data
if (
data !== null && typeof data === 'object'
&& existingData !== null && typeof existingData === 'object'
&& 'durationMs' in data && 'durationMs' in existingData
) {
(data as Record<string, unknown>).durationMs = (existingData as Record<string, unknown>).durationMs
}
}
/**
* Rewrite a fresh replay-produced log so repeated refreshes do not churn
* volatile fixture fields. Meaningful event payloads come from `fresh`; the
* existing fixture lends session ids, cwd, creation times, event times, and
* hook durations where the record shape still matches.
*
* @param fresh The newly harvested session JSONL.
* @param existing The committed fixture JSONL being refreshed.
* @param replacements Cross-log literal replacements from {@link refreshFixtureReplacements}.
* @returns The stabilized JSONL content to write back.
*/
export function stabilizeRefreshLog(fresh: string, existing: string, replacements: FixtureReplacement[]): string {
let stable = fresh
for (const { from, to } of replacements) stable = stable.split(from).join(to)
const existingRecords = parseJsonlRecords(existing)
const records = parseJsonlRecords(stable)
for (let i = 0; i < records.length; i++) {
preserveFixtureVolatiles(records[i] as Record<string, unknown>, existingRecords[i])
}
return records.map(record => JSON.stringify(record)).join('\n') + '\n'
}
/**
* Register the suite: one `describe` per scenario (the golden/log compares and
* the header-uniformity guard) plus the fixture guard block (no orphan
* scenario dirs, required files present, exactly one pin per header class,
* pinning fixtures well-formed, non-pinning fixtures header-scrubbed). Must
* pinning fixtures well-formed, every JSONL prompt-scrubbed, non-pinning
* fixtures fully header-scrubbed). Must
* run at vitest collection time — it calls `describe`/`it`. Throws
* immediately if any header class lacks a pinning scenario or carries two
* (the uniformity guard needs exactly one comparison anchor per class).
@@ -215,6 +413,8 @@ export function headerDeltaCount(rawLog: string): number {
export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
const { agent, snapshotsDir, scenarios, mode } = options
const RECORDING = mode === 'record'
const REFRESHING = mode === 'refresh'
const childMode: 'replay' | 'record' = RECORDING ? 'record' : 'replay'
/** The class a scenario's header composition belongs to (see {@link Scenario.headerClass}). */
const classOf = (scenario: Scenario): string => scenario.headerClass ?? 'default'
@@ -243,15 +443,18 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
describe(`snapshot: ${scenario.name}`, () => {
// In RECORD mode, only re-run the `recorded` (live-API) scenarios; the
// `authored` ones (sidecar-driven errors/cancel) are never re-recorded.
// REFRESH mode is replay-backed and deterministic, so it runs every
// scenario and rewrites the comparable fixtures from that replay run.
it.skipIf(RECORDING && !scenario.recorded)('matches the goldens', async () => {
const dir = join(snapshotsDir, scenario.name)
const input = JSON.parse(await readFile(join(dir, 'input.json'), 'utf8')) as InputScript
const overrideFile = join(dir, 'replay.override.json')
const workspaceDir = join(dir, 'workspace')
const childSessions = scenario.childSessions ?? 0
const comparesLog = scenario.comparesLog ?? scenario.hasModelTurn
const result = await runScenario(input, {
agent,
mode,
mode: childMode,
fixtureFile: join(dir, 'session.jsonl'),
...existsSync(overrideFile) ? { overrideFile } : {},
// In REPLAY, forward the recorded child fixtures so each subagent session
@@ -276,43 +479,79 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
}
// RECORD mode (recorded model scenarios only): persist the freshly-harvested
// logs back to their fixtures — the primary to session.jsonl, each child to
// session.<n>.jsonl in harvest order. `--update` refreshes the Vitest
// goldens but NOT these fixtures, so write them here. A non-pinning
// scenario's fixtures are written header-scrubbed, so a re-record can
// never smuggle the full prompt/schema content back into every fixture.
// live logs back to their fixtures. REFRESH mode does the same from a
// keyless replay run for every comparable log, including authored
// scenarios that live record deliberately skips. The primary goes to
// session.jsonl, each child to session.<n>.jsonl in harvest order. A
// Every fixture is written with its system prompt scrubbed. A pinning
// scenario keeps the remaining header content (notably tool schemas);
// every other scenario scrubs that bulk too. Record/refresh therefore
// cannot smuggle prompt text back into JSONL or duplicate schemas.
const scrub = scenario.pinsHeader === true
? (log: string): string => log
? scrubSystemPrompts
: scrubRequestHeaders
if (RECORDING && scenario.recorded && scenario.hasModelTurn) {
expect(result.sessionLogs.length, 'record produced no session log to harvest').toBeGreaterThan(0)
const fixtureFiles = ['session.jsonl', ...Array.from({ length: childSessions }, (_, i) => `session.${i + 1}.jsonl`)]
const existingFixtures = REFRESHING
? await Promise.all(fixtureFiles.map(file => readFile(join(dir, file), 'utf8')))
: []
const replacements = REFRESHING ? refreshFixtureReplacements(result.sessionLogs, existingFixtures) : []
const writesSessionFixtures = (RECORDING && scenario.recorded && scenario.hasModelTurn)
|| (REFRESHING && comparesLog)
if (writesSessionFixtures) {
expect(result.sessionLogs.length, `${mode} produced no session log to harvest`).toBeGreaterThan(0)
expect(result.sessionLogs.length, `expected ${childSessions + 1} session logs (parent + children)`)
.toBe(childSessions + 1)
await writeFile(join(dir, 'session.jsonl'), scrub((result.sessionLogs[0] as HarvestedLog).content))
const primary = (result.sessionLogs[0] as HarvestedLog).content
await writeFile(join(dir, 'session.jsonl'), scrub(
REFRESHING ? stabilizeRefreshLog(primary, existingFixtures[0] as string, replacements) : primary,
))
for (let i = 1; i < result.sessionLogs.length; i++) {
await writeFile(join(dir, `session.${i}.jsonl`), scrub((result.sessionLogs[i] as HarvestedLog).content))
const child = (result.sessionLogs[i] as HarvestedLog).content
await writeFile(join(dir, `session.${i}.jsonl`), scrub(
REFRESHING ? stabilizeRefreshLog(child, existingFixtures[i] as string, replacements) : child,
))
}
if (scenario.pinsHeader === true) {
const primary = result.sessionLogs[0] as HarvestedLog
const primaryPrompts = normalizedSystemPrompts(primary.content, ctx)
expect(primaryPrompts.length, `${mode} must produce the declared header snapshot count`)
.toBe(scenario.expectedHeaderSnapshots ?? 1)
// Child sessions have no transition of their own: uniform with
// the initial prompt, exactly like a classmate.
const initialSnapshot = formatSystemPromptSnapshot(primaryPrompts[0] as string)
for (const log of result.sessionLogs.slice(1)) {
for (const prompt of normalizedSystemPrompts(log.content, ctx)) {
expect(formatSystemPromptSnapshot(prompt), 'a child session produced a divergent system prompt')
.toEqual(initialSnapshot)
}
}
const snapshot = formatSystemPromptSnapshot(
primaryPrompts[0] as string,
normalizedSystemPromptDeltas(primary.content, ctx),
primaryPrompts.slice(1),
)
await writeFile(join(dir, SYSTEM_PROMPT_SNAPSHOT), snapshot)
}
}
await expect(normalizeStdout(result.rawStdout, ctx))
.toMatchFileSnapshot(join(dir, 'stdout.golden.jsonl'))
const stdout = normalizeStdout(result.rawStdout, ctx)
if (REFRESHING) {
await writeFile(join(dir, 'stdout.golden.jsonl'), stdout)
}
await expect(stdout).toMatchFileSnapshot(join(dir, 'stdout.golden.jsonl'))
// A model turn always produces a log worth comparing; a hook scenario can
// produce one without a model turn (a `rejected` turn carrying `hook/*`).
const comparesLog = scenario.comparesLog ?? scenario.hasModelTurn
if (comparesLog) {
// The harvested logs (primary-first) must match their committed fixtures
// 1:1. Each side passes through normalizeSessionLog, scrubbed against ITS
// OWN volatile values — the live run's via `ctx`, the committed fixture's
// via its own header (a committed file cannot share the live run's ids).
// Unless this scenario pins the header, both sides ALSO pass through
// scrubRequestHeaders: the live log carries the real prompt/schemas, the
// fixture carries the `{{system}}`/`{{tools}}` tokens, and the scrub is
// idempotent — so the compare checks the header's presence, position,
// reason, and config, but not its bulk content (pinned once, in the
// `pinsHeader` scenario).
// Both sides pass through the scenario's idempotent scrub: every live
// prompt becomes the fixture's `{{system}}`; non-pinning scenarios
// additionally tokenize tools/prefix. The dedicated header guard below
// compares those omitted values against their class's pin artifacts.
expect(result.sessionLogs.length, 'this scenario must persist a session log').toBe(childSessions + 1)
const fixtureFiles = ['session.jsonl', ...Array.from({ length: childSessions }, (_, i) => `session.${i + 1}.jsonl`)]
for (let i = 0; i < fixtureFiles.length; i++) {
const harvested = scrub((result.sessionLogs[i] as HarvestedLog).content)
const fixture = scrub(await readFile(join(dir, fixtureFiles[i] as string), 'utf8'))
@@ -321,35 +560,50 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
}
}
// Header-uniformity guard: a class's single pin is sound only while
// every session in that class composes the SAME header and keeps it
// for the whole run. Assert both halves live. (1) Every
// request/header the run produced (parent, spawn child, fork child,
// initial or resume) must equal the CLASS's pinned fixture's header
// after each side is normalized against its own volatile values.
// (2) No request/header-delta may appear at all — a mid-run header
// change diverges from the pin by construction, and its content
// would be invisible under the scrub. If either fails, either the
// header changed (update the pin: re-record or hand-edit the pinning
// scenario's fixture) or composition became session-dependent by
// design (give the divergent shape its own pinning scenario and
// class).
// Header-uniformity guard: every live header in a class must equal
// the class pin, split across its JSONL header (system token + real
// tools) and readable Markdown prompt. A pinning scenario may carry
// its declared header deltas (prompt edits live in the Markdown
// golden) and its declared TRANSITION SNAPSHOTS (a change the delta
// encoding cannot express — a session-mode flip resorts the canonical
// tool list — lands as a further full snapshot; the Markdown carries
// each snapshot's prompt). Classmates anchor to the pin's FIRST
// header; a class with no pin (protocol-only scenarios, no model
// turn) has no header content to anchor.
const classPin = pinningByClass.get(classOf(scenario))
if (scenario.pinsHeader !== true && classPin !== undefined) {
const pinningScenario = classPin
const pinnedFixture = await readFile(join(snapshotsDir, pinningScenario.name, 'session.jsonl'), 'utf8')
if (classPin !== undefined) {
const pinningDir = join(snapshotsDir, classPin.name)
const pinnedFixture = await readFile(join(pinningDir, 'session.jsonl'), 'utf8')
const pinned = normalizedHeaders(pinnedFixture, fixtureContext(pinnedFixture))
// The classmates' anchor is the pin's FIRST header; a pin may carry
// further transition headers of its own (legal only there).
expect(pinned.length, `the pinning fixture (${pinningScenario.name}) must carry at least one request/header`)
.toBeGreaterThanOrEqual(1)
for (const log of result.sessionLogs) {
expect(headerDeltaCount(log.content), `session ${log.id}: a request/header-delta in a non-pinning scenario`)
.toBe(0)
const headers = normalizedHeaders(log.content, ctx)
const promptSnapshot = await readFile(join(pinningDir, SYSTEM_PROMPT_SNAPSHOT), 'utf8')
const initialPromptSnapshot = initialSystemPromptSnapshot(promptSnapshot)
expect(pinned.length, `the pinning fixture (${classPin.name}) must carry exactly its declared request/header snapshots`)
.toBe(classPin.expectedHeaderSnapshots ?? 1)
for (const [logIndex, log] of result.sessionLogs.entries()) {
const isPinPrimary = scenario.pinsHeader === true && logIndex === 0
const expectedDeltas = isPinPrimary ? scenario.expectedHeaderDeltas ?? 0 : 0
expect(headerDeltaCount(log.content), `session ${log.id}: request/header-delta count`)
.toBe(expectedDeltas)
const headers = normalizedHeaders(scrubSystemPrompts(log.content), ctx)
const prompts = normalizedSystemPrompts(log.content, ctx)
expect(prompts.length, `session ${log.id}: every request/header must carry a string system prompt`)
.toBe(headers.length)
for (const [k, header] of headers.entries()) {
expect(header, `session ${log.id}: request/header #${k + 1} diverged from the pinned (${pinningScenario.name}) header`)
.toEqual(pinned[0])
const anchor = isPinPrimary ? pinned[k] : pinned[0]
expect(header, `session ${log.id}: request/header #${k + 1} diverged from the pinned (${classPin.name}) header`)
.toEqual(anchor)
if (!isPinPrimary) {
expect(formatSystemPromptSnapshot(prompts[k] as string), `session ${log.id}: initial system prompt #${k + 1} diverged from ${classPin.name}/${SYSTEM_PROMPT_SNAPSHOT}`)
.toEqual(initialPromptSnapshot)
}
}
if (isPinPrimary) {
expect(formatSystemPromptSnapshot(
prompts[0] as string,
normalizedSystemPromptDeltas(log.content, ctx),
prompts.slice(1),
), `session ${log.id}: system-prompt content diverged from ${classPin.name}/${SYSTEM_PROMPT_SNAPSHOT}`)
.toEqual(promptSnapshot)
}
}
}
@@ -381,13 +635,15 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
// `overridden` flag: required when set, forbidden when not — the harness
// forwards the file purely on existence, so an unregistered stray sidecar
// would silently replace the derived script.
for (const { name, overridden, childSessions } of scenarios) {
for (const { name, overridden, childSessions, pinsHeader } of scenarios) {
const dir = join(snapshotsDir, name)
expect(existsSync(join(dir, 'input.json')), `${name}/input.json`).toBe(true)
expect(existsSync(join(dir, 'stdout.golden.jsonl')), `${name}/stdout.golden.jsonl`).toBe(true)
expect(existsSync(join(dir, 'session.jsonl')), `${name}/session.jsonl`).toBe(true)
expect(existsSync(join(dir, 'replay.override.json')), `${name}/replay.override.json presence must match \`overridden\``)
.toBe(overridden === true)
expect(existsSync(join(dir, SYSTEM_PROMPT_SNAPSHOT)), `${name}/${SYSTEM_PROMPT_SNAPSHOT} presence must match \`pinsHeader\``)
.toBe(pinsHeader === true)
// A nested-agent scenario ships one child fixture per recorded subagent
// session (`session.1.jsonl` …), the replay source for that child session.
for (const childFixture of childFixturePaths(dir, childSessions ?? 0)) {
@@ -413,28 +669,33 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
}
})
it('every pinning fixture carries at least one request/header', async () => {
// The live uniformity guard runs only in NON-pinning scenarios, so the
// committed pins are asserted directly. A pin carries the class's full
// header content — INCLUDING mid-run transitions (a session-mode flip
// logs a second snapshot or a delta; those are legal only here, and the
// classmates' uniformity anchor is the pin's FIRST header) — so the
// shape requirement is presence, not uniqueness.
it('every pinning fixture carries its declared header snapshots, a readable prompt, and its declared deltas', async () => {
// The live uniformity guard runs only in NON-pinning scenarios, so a
// class made of just its pinning scenario would otherwise accept a
// re-recorded pin with undeclared headers or an undeclared mid-run
// header-delta — shapes the pin design cannot represent. Assert the
// committed pins directly. A scenario whose arc legitimately rewrites
// a prompt section declares the exact count via expectedHeaderDeltas;
// one whose transition is inexpressible as a delta (a session-mode flip
// resorts the canonical tool list, so it lands as a full fallback
// snapshot) declares its snapshot count via expectedHeaderSnapshots.
for (const scenario of pinningByClass.values()) {
const fixture = await readFile(join(snapshotsDir, scenario.name, 'session.jsonl'), 'utf8')
const headers = normalizedHeaders(fixture, fixtureContext(fixture))
expect(headers.length, `${scenario.name}: a pinning fixture must carry at least one request/header`).toBeGreaterThanOrEqual(1)
const promptSnapshot = await readFile(join(snapshotsDir, scenario.name, SYSTEM_PROMPT_SNAPSHOT), 'utf8')
expect(headers.length, `${scenario.name}: a pinning fixture must carry exactly its declared request/header snapshots`)
.toBe(scenario.expectedHeaderSnapshots ?? 1)
expect(promptSnapshot.length, `${scenario.name}/${SYSTEM_PROMPT_SNAPSHOT} must not be empty`).toBeGreaterThan(0)
expect(promptSnapshot.endsWith('\n'), `${scenario.name}/${SYSTEM_PROMPT_SNAPSHOT} must end in a newline`).toBe(true)
expect(headerDeltaCount(fixture), `${scenario.name}: a pinning fixture must carry exactly its declared request/header-deltas`)
.toBe(scenario.expectedHeaderDeltas ?? 0)
}
})
it('committed fixtures carry request-header content ONLY in the pinning scenario', async () => {
// The whole point of the pin: a system-prompt or tool-schema change must
// churn exactly one committed line. A non-pinning fixture that carries the
// full header (a hand-recorded file, or a header line hand-edited out of
// its canonical JSON form) silently reopens the suite-wide churn, so fail
// loud here: every non-pinning session*.jsonl must be a fixed point of
// scrubRequestHeaders (apply the scrub to fix a violation), and the
// pinning scenario's fixtures must NOT be (their content IS the pin).
it('every committed JSONL omits system prompts and only pinning fixtures keep other header bulk', async () => {
// System prompts always live in the readable Markdown artifact. Header
// pins keep tool schemas/prefixes in JSONL; every other fixture tokenizes
// all header bulk. Fixed-point checks make both storage rules fail loud.
for (const scenario of scenarios) {
const dir = join(snapshotsDir, scenario.name)
const files = [
@@ -443,8 +704,10 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
]
for (const file of files) {
const fixture = await readFile(join(dir, file), 'utf8')
expect(scrubSystemPrompts(fixture), `${scenario.name}/${file} carries an unscrubbed system prompt`)
.toEqual(fixture)
if (scenario.pinsHeader === true) {
expect(scrubRequestHeaders(fixture), `${scenario.name}/${file} must PIN the full header content`)
expect(scrubRequestHeaders(fixture), `${scenario.name}/${file} must pin the non-system header content`)
.not.toEqual(fixture)
} else {
expect(scrubRequestHeaders(fixture), `${scenario.name}/${file} carries unscrubbed header content`)