Merge remote-tracking branch 'origin/master' into codex/session-scoped-sandbox-roots

# Conflicts:
#	.agents/notes/implemented/feature/2026-07-14-cross-family-fs-sandbox.i18n.yaml
#	.agents/notes/implemented/feature/2026-07-14-cross-family-fs-sandbox.md
#	.agents/notes/implemented/feature/2026-07-14-cross-family-fs-sandbox.zh.md
#	examples/acp-agent/README.md
#	packages/examples/agent-spine-demo/package.json
#	packages/fs/fs-sandbox/src/index.ts
#	packages/fs/tool-fs-search/tests/tools.spec.ts
#	packages/support/acp-snapshot/README.md
#	packages/support/acp-snapshot/src/suite.ts
#	pnpm-lock.yaml
#	scripts/type-equiv.manifest.json
This commit is contained in:
Tianyi Cui
2026-07-22 21:37:30 +08:00
1133 changed files with 67897 additions and 10909 deletions

View File

@@ -4,10 +4,10 @@ The ACP snapshot suite kit: the shared machinery behind the keyless snapshot tie
Four layers, importable separately:
- **`launchAcpTestAgent` (launcher)** — boots an unbuilt ACP agent from a supplied cwd, pins tsx to the repo tsconfig, connects the SDK client over a raw-byte stdout tee, collects session updates and stderr, surfaces asynchronous spawn failures through its startup lifecycle, fails closed on unhandled permission requests, and owns graceful or signalled shutdown. Shutdown waits for process exit, inherited stdio closure, and ACP parser exhaustion before resolving or propagating a child error, so captures are complete and callers can remove owned paths after either outcome. Snapshot and ordinary e2e suites share this process boundary; a test supplies only agent paths, cwd, environment overrides, and any permission policy.
- **`runScenario` (harness)** — boots the real agent bin as a subprocess via tsx (unbuilt, Loader path), drives it over ACP JSON-RPC stdio from a deterministic `input.json` script, tees raw stdout for the expected-output and purity checks, and harvests every persisted session JSONL (parent + subagent children, primary-first) after a graceful stdin-EOF shutdown. Parameterized by `AgentUnderTest` (`binScript`, `configPath`, `tsconfigPath` — absolute paths) plus an optional parent for its generated session cwd; the default is the platform temp directory. Startup failures preserve captured agent stderr in the rejected diagnostic.
- **Normalizers** — pure functions turning the two captured surfaces into stable text: `normalizeStdout` (JSON-RPC ids → first-seen sequence; UUIDs/cwd → tokens; doubles as the stdout-purity check), `normalizeSessionLog` (times zeroed, `seq` kept), `scrubSystemPrompts` (prompt text → `{{system}}`), `scrubToolSchemas` (schema bulk → `{{tools}}`), and `scrubRequestHeaders` (all header bulk → `{{system}}`/`{{tools}}`/`{{messagePrefix}}` outside each pin, structure kept — [pinned-header Agent Note](../../../.agents/notes/implemented/testing/2026-07-06-pin-request-header-content-in-one-scenario.md)).
- **`defineAcpSnapshotSuite` (factory)** — registers the whole describe/it tree for a scenario table: per-scenario expected-output and re-persisted-log comparisons, record/refresh fixture write-back, rejection of structured `UNKNOWN_TOOL` results, the per-header-class pin (`system-prompt.expected.md` plus `tool-schemas.expected.json`) with its live uniformity guard, and the fixture guard block (no orphan scenario dirs, required files present, exactly one pin per class, every JSONL prompt/schema-scrubbed, non-pinning fixtures fully header-scrubbed). Each scenario directory's `session.jsonl` plus contiguous `session.<n>.jsonl` siblings are the ordered primary/child inventory; the scenario table does not duplicate their count. Must be called at vitest collection time.
- **`launchAcpTestAgent` (launcher)** — boots a source agent under tsx or a built `lib` agent under plain Node from a supplied cwd, connects the SDK client over a raw-byte stdout tee, collects session updates and stderr, surfaces asynchronous spawn failures through startup, fails closed on unhandled permission requests, and owns graceful or signalled shutdown. Shutdown waits for process exit, inherited stdio closure, and ACP parser exhaustion before resolving or propagating a child error, so captures are complete and callers can remove owned paths after either outcome. When Windows accepts forced termination but publishes its exit marker asynchronously, shutdown gives that marker a bounded grace before treating fallback refusal as a second failure. Snapshot and ordinary e2e suites share this process boundary; a test supplies only agent paths, cwd, environment overrides, and any permission policy.
- **`runScenario` (harness)** — drives ACP JSON-RPC stdio from a deterministic `input.json` script through the launcher, tees raw stdout for the expected-output and purity checks, and harvests every persisted raw JSONL session log (parent and subagent children, primary-first) after graceful stdin EOF. `AgentUnderTest` supplies absolute `binScript`, optional `libBinScript`, `configPath`, and `tsconfigPath` paths because the subprocess cwd is outside the repo; `workspaceParent` may move the generated child cwd from the platform temp directory when that grant is itself under test. Startup failures preserve captured agent stderr in the rejected diagnostic.
- **Normalizers** — pure functions turning the two captured surfaces into stable text: `normalizeStdout` (JSON-RPC ids → first-seen sequence; UUIDs/cwd → tokens; cwd-rooted separators selected as canonical `/` or host-native; `session_info_update.updatedAt` → `{{updatedAt}}`; doubles as the stdout-purity check), `normalizeSessionLog` (times zeroed, `seq` kept, the same cwd-path policy), `scrubSystemPrompts` (prompt text → `{{system}}`), `scrubToolSchemas` (schema bulk → `{{tools}}`), and `scrubRequestHeaders` (all header bulk → `{{system}}`/`{{tools}}`/`{{messagePrefix}}` outside each pin, structure kept — [pinned-header Agent Note](../../../.agents/notes/implemented/testing/2026-07-06-pin-request-header-content-in-one-scenario.md)).
- **`defineAcpSnapshotSuite` (factory)** — registers the whole describe/it tree for a scenario table: per-scenario expected-output and re-persisted-log comparisons, record/refresh fixture write-back, rejection of structured `UNKNOWN_TOOL` results, the per-header-class pin (`system-prompt.expected.md` plus `tool-schemas.expected.json`) with its live uniformity guard, and the fixture guard block (no orphan scenario dirs, required files present, exactly one pin per class, every JSONL prompt/schema-scrubbed, non-pinning fixtures fully header-scrubbed). Refresh preserves existing volatile fields by event position and gives a newly inserted `session/title` its preceding event's time, so feature-driven insertions do not churn the remainder of a fixture. Each scenario directory's `session.jsonl` plus contiguous `session.<n>.jsonl` siblings are the ordered primary/child inventory; the scenario table does not duplicate their count. Must be called at vitest collection time.
A consuming `*.snapshot.ts` is the scenario table plus one factory call:
@@ -38,6 +38,8 @@ defineAcpSnapshotSuite({
A scenario booting a differently-composed tree sets its own `configPath` (an overlay whose basename still ends in `cordis.yml`, so the bin's replay swap finds the sibling `*cordis.snapshot.yml`) and, when that composition changes the request header, its own `headerClass` with its own pinning scenario — the acp-agent example's Code Mode and filesystem scenarios are templates. `workspaceParent` moves the generated cwd outside the platform temp area when temporary-directory grants are themselves under test; the harness still owns and removes only the generated child. Each pinning directory stores the normalized full prompt sequence in generated `system-prompt.expected.md` and the corresponding full tool-schema sequence in generated `tool-schemas.expected.json`; `session.jsonl` stores `"system":"{{system}}","tools":"{{tools}}"` while retaining config, reason, and any model-visible prefix. A pin with legitimate mid-run header changes declares `expectedHeaderChanges`, which fixes the length of both sidecar sequences.
Every scenario compares `stdout.expected.jsonl` with cwd-rooted separators canonicalized to `/`. On Windows, `pinsNativeWindowsStdout` additionally compares the complete `stdout.expected.windows.jsonl` after the shared expected output and requires that sidecar exactly when enabled. A scenario whose driven behavior needs POSIX process semantics (e.g. cancelling a live bash call kills a detached process group) declares `posixOnly`, which skips its run test on Windows while the fixture guards keep covering its committed files everywhere.
The example also ships a `cordis.snapshot.yml` replay overlay next to its `cordis.yml` (the bin swaps them under `DSH_SNAPSHOT=replay` — [single-source replay config Agent Note](../../../.agents/notes/implemented/testing/2026-07-04-single-source-acp-replay-config.md)); replay fixtures are served by [`dsh-llm-replay`](../llm-replay/README.md), which this package points at via the `DSH_SNAPSHOT_*` env vars it sets on the child. `pnpm run test:snapshot:record` calls the live LLM and rewrites the recorded scenarios' model fixtures; `pnpm run test:snapshot:refresh` stays keyless, runs the replay overlay, and rewrites stdout, comparable session-log expected outputs, and each pin's prompt and tool-schema sidecars from the committed model scripts. Fixture roles, record/replay/refresh semantics, and scenario-table fields are documented on `Scenario` and in the [snapshot Agent Note](../../../.agents/notes/implemented/testing/2026-06-19-acp-snapshot-tests.md).
Constraints: `suite.ts` imports vitest, so the package entry is importable only inside a vitest run (the launcher, harness, and normalizers have no such dependency but ship from the same entry). ACP-specific by design — the launcher speaks the SDK's `ClientSideConnection`. Permission round-trips are scriptable: `InputScript.permissionAnswers` is a FIFO queue of option-kind selections (`allow_once`, `reject_once`, …) the client maps to the agent-issued `optionId` at answer time; an absent or exhausted queue answers `cancelled`, and a kind the request never offered rejects the run (the agent is answered `cancelled`, so a tolerant agent cannot absorb the scenario bug). Session config options are scriptable too: the `setConfigOption` step switches a knob over `session/set_config_option`, and `setConfigOptionExpectError` asserts the bridge rejects an unknown id or out-of-vocabulary value (the error frame stays in the transcript).
@@ -53,4 +55,4 @@ None; this package neither assembles nor sends a provider request.
## Known Limitations and Deferred Work
- **Session harvest requires raw JSONL mode** — `runScenario` collects persisted `.jsonl` logs, so snapshot configs set `persistenceCompression: 'none'`; compressed JSONL and SQLite compositions have no snapshot-harvest path.
- **The subprocess boots the unbuilt tsx/Loader path only** — the built-bin artifact is guarded by the separate `built-bin` e2e smokes, never by this tier.
- **Built mode requires current artifacts** — run `pnpm run build` before selecting `DSH_EXAMPLE_MODE=lib`; source mode remains the zero-build path.

View File

@@ -24,6 +24,8 @@ import { basename, dirname, join, delimiter } from 'node:path'
import {
ClientSideConnection,
PROTOCOL_VERSION,
type CreateElicitationRequest,
type CreateElicitationResponse,
type RequestPermissionRequest,
type RequestPermissionResponse,
type SessionNotification,
@@ -59,6 +61,8 @@ export type InputStep =
waitForToolCallUpdate?: string
}
| { op: 'cancel' }
| { op: 'setMode'; modeId: string }
| { op: 'setModeExpectError'; modeId: string }
| { op: 'setConfigOption'; configId: string; value: string }
| { op: 'setConfigOptionExpectError'; configId: string; value: string }
@@ -78,6 +82,16 @@ export interface InputScript {
* agent itself just sees `cancelled`, so it cannot absorb the bug).
*/
permissionAnswers?: PermissionAnswer[]
/**
* Ordered answers for the agent's `elicitation/create` round-trips (the
* ask_user_question / plan-review forms), consumed FIFO — the Nth request
* gets the Nth answer. Exhaustion (or no queue) answers `cancel`, the same
* fail-closed stub an elicitation-free scenario relies on. Unlike permission
* kinds, the scripted strings are not validated against the offered form —
* a stray `choice` reaches the agent verbatim, which reads it as a custom
* (non-consenting) answer, so a scenario bug fails safe in the transcript.
*/
elicitationAnswers?: ElicitationAnswer[]
}
/** One scripted answer to a permission request: which offered option kind to select. */
@@ -86,6 +100,16 @@ export interface PermissionAnswer {
kind: 'allow_once' | 'allow_always' | 'reject_once' | 'reject_always'
}
/** One scripted answer to an elicitation form (accept with choice/custom content, or cancel). */
export interface ElicitationAnswer {
/** Accept the form with the content below, or cancel it. */
action: 'accept' | 'cancel'
/** The selected option label (the form's `choice` field). */
choice?: string
/** Free-form text (the form's `custom` field). */
custom?: string
}
/** One harvested session log plus the identifying facts off its header line. */
export interface HarvestedLog {
/** The recorded session id (header `id`). */
@@ -161,11 +185,21 @@ export interface RunOptions {
configPath?: string
}
/** Derive one stable, fixed-length spill root owned by this scenario. */
function scenarioSpillRoot(fixtureFile: string): string {
/**
* Derive one stable, fixed-length spill root owned by this scenario.
* Windows uses a two-character-shorter root because drive resolution adds its drive prefix.
* @param fixtureFile - The scenario fixture whose parent directory provides the stable identity.
* @param platform - the host platform, injectable for unit coverage.
* @returns the root-relative snapshot spill directory.
*/
export function snapshotSpillRoot(
fixtureFile: string,
platform: NodeJS.Platform = process.platform,
): string {
const scenario = basename(dirname(fixtureFile))
const key = createHash('sha256').update(scenario).digest('hex').slice(0, 9)
return `/tmp/dsh-acp-snap-${key}`
const root = platform === 'win32' ? '/t' : '/tmp'
return `${root}/dsh-acp-snap-${key}`
}
/**
@@ -184,7 +218,7 @@ export async function runScenario(input: InputScript, opts: RunOptions): Promise
// before stdout normalization, so tmpdir() length differences churn expected outputs.
// Scenario ownership also matters: replay runs concurrently, and one teardown
// must never delete another scenario's in-flight full-output recovery file.
const spillRoot = scenarioSpillRoot(opts.fixtureFile)
const spillRoot = snapshotSpillRoot(opts.fixtureFile)
// Everything past the temp-dir creation is followed by failure-safe cleanup,
// so a failure in workspace seeding, spawn, or any step never leaks resources.
let launched: LaunchedAcpTestAgent | undefined
@@ -213,6 +247,8 @@ export async function runScenario(input: InputScript, opts: RunOptions): Promise
// Permission answers are consumed FIFO across the whole run; exhaustion
// falls back to `cancelled` so approval-free scenarios keep the plain stub.
const permissionQueue = [...input.permissionAnswers ?? []]
// Elicitation answers mirror the permission queue: FIFO, cancel on exhaustion.
const elicitationQueue = [...input.elicitationAnswers ?? []]
// A scenario bug detected inside a client callback (a scripted permission
// kind the agent never offered). It cannot fail the run from in there: a
// callback throw only becomes a JSON-RPC error RESPONSE to the agent, and
@@ -242,6 +278,17 @@ export async function runScenario(input: InputScript, opts: RunOptions): Promise
}
return Promise.resolve({ outcome: { outcome: 'selected', optionId: option.optionId } })
},
createElicitation(_params: CreateElicitationRequest): Promise<CreateElicitationResponse> {
const answer = elicitationQueue.shift()
if (answer === undefined || answer.action !== 'accept') return Promise.resolve({ action: 'cancel' })
return Promise.resolve({
action: 'accept',
content: {
...answer.choice !== undefined ? { choice: answer.choice } : {},
...answer.custom !== undefined ? { custom: answer.custom } : {},
},
})
},
})
const active = launched
await active.spawned
@@ -397,6 +444,24 @@ async function runStep(
await client.cancel({ sessionId })
return
}
case 'setMode': {
const sessionId = getSessionId()
if (sessionId === undefined) throw new Error('snapshot-harness: setMode before newSession')
await client.setSessionMode({ sessionId, modeId: step.modeId })
return
}
case 'setModeExpectError': {
const sessionId = getSessionId()
if (sessionId === undefined) throw new Error('snapshot-harness: setModeExpectError before newSession')
// The bridge rejects an unknown/uncomposed mode id with invalidParams;
// that rejection IS the expected wire behavior — swallow it so the run
// completes and the error frame is captured in the transcript.
await client.setSessionMode({ sessionId, modeId: step.modeId }).then(
() => { throw new Error('snapshot-harness: expected session/set_mode to be rejected but it succeeded') },
() => { /* expected: the bridge rejected the mode id */ },
)
return
}
case 'setConfigOption': {
const sessionId = getSessionId()
if (sessionId === undefined) throw new Error('snapshot-harness: setConfigOption before newSession')

View File

@@ -18,6 +18,7 @@
export {
runScenario,
type ElicitationAnswer,
type HarvestedLog,
type InputScript,
type InputStep,
@@ -37,10 +38,14 @@ export {
scrubRequestHeaders,
scrubSystemPrompts,
scrubToolSchemas,
type CwdPathMode,
type NormalizeContext,
type NormalizeOptions,
} from './normalize.ts'
export {
defineAcpSnapshotSuite,
refreshFixtureReplacements,
stabilizeRefreshLog,
type Scenario,
type SnapshotSuiteOptions,
} from './suite.ts'

View File

@@ -15,12 +15,16 @@ import {
ndJsonStream,
type Agent as AcpAgent,
type Client,
type CreateElicitationRequest,
type CreateElicitationResponse,
type RequestPermissionRequest,
type RequestPermissionResponse,
type SessionNotification,
} from '@agentclientprotocol/sdk'
import { resolveExampleLaunch } from '@deepseek-ai/dsh-loader-smoke'
const EXIT_MARKER_GRACE_MS = 250
/** The source/built agent entry, leaf config, and workspace tsconfig an ACP test boots. */
export interface AgentUnderTest {
/** The agent source bin entry (for example `packages/examples/acp-demo/src/bin.ts`). */
@@ -45,6 +49,8 @@ export interface AcpTestLaunchOptions {
env?: NodeJS.ProcessEnv
/** Permission handler; omitted requests fail closed as `cancelled`. */
requestPermission?: (params: RequestPermissionRequest) => Promise<RequestPermissionResponse>
/** Elicitation handler; omitted requests fail closed as `cancel`. */
createElicitation?: (params: CreateElicitationRequest) => Promise<CreateElicitationResponse>
}
/** A running ACP test process and its captured client-side surfaces. */
@@ -150,6 +156,8 @@ export function launchAcpTestAgent(options: AcpTestLaunchOptions): LaunchedAcpTe
}
const requestPermission = options.requestPermission
?? (() => Promise.resolve({ outcome: { outcome: 'cancelled' as const } }))
const createElicitation = options.createElicitation
?? (() => Promise.resolve({ action: 'cancel' as const }))
const makeClient = (_agent: AcpAgent): Client => ({
sessionUpdate(params: SessionNotification): Promise<void> {
return trackClientCallback(() => {
@@ -173,6 +181,7 @@ export function launchAcpTestAgent(options: AcpTestLaunchOptions): LaunchedAcpTe
})
},
requestPermission: params => trackClientCallback(() => requestPermission(params)),
unstable_createElicitation: params => trackClientCallback(() => createElicitation(params)),
})
const client = new ClientSideConnection(makeClient, stream)
// `exit` only reports the parent process's status. Descendants may retain
@@ -231,6 +240,15 @@ export function launchAcpTestAgent(options: AcpTestLaunchOptions): LaunchedAcpTe
return
}
const propagateFailureAfterDrain = async (): Promise<never> => {
await drained
closeUpdateStream()
throw failure
}
// Windows implements the supported signal names as forced termination. The exit markers
// may therefore arrive after the error wins the race above but before fallback begins.
if (!isRunning(child) || await exitMarkerWithinGrace(exited)) return propagateFailureAfterDrain()
// An `error` after spawn is not an exit edge: in particular, a failed
// signal can leave the subprocess live. Force termination, await the
// already-observed exit edge, and only then propagate the child error so
@@ -240,6 +258,10 @@ export function launchAcpTestAgent(options: AcpTestLaunchOptions): LaunchedAcpTe
child.once('error', observeFallbackError)
if (!child.kill('SIGKILL')) {
child.off('error', observeFallbackError)
// A successful earlier signal may win between the live check and this fallback call.
// In that case `kill()` correctly reports no process to signal; the original child error
// remains the shutdown result once inherited stdio and callbacks have drained.
if (!isRunning(child) || await exitMarkerWithinGrace(exited)) return propagateFailureAfterDrain()
closeUpdateStream()
throw new AggregateError(
[failure, new Error('Fallback SIGKILL was not accepted by the child process')],
@@ -258,9 +280,7 @@ export function launchAcpTestAgent(options: AcpTestLaunchOptions): LaunchedAcpTe
'ACP test agent failed and fallback termination was refused',
)
}
await drained
closeUpdateStream()
throw failure
return propagateFailureAfterDrain()
},
}
}
@@ -270,6 +290,17 @@ function waitForExit(child: ChildProcessWithoutNullStreams): Promise<void> {
return new Promise<void>(resolve => child.once('exit', () => { resolve() }))
}
/** Give an accepted Windows termination request a bounded window to publish its exit marker. */
function exitMarkerWithinGrace(exited: Promise<void>): Promise<boolean> {
return Promise.race([
exited.then(() => true),
new Promise<false>((resolve) => {
const timer = setTimeout(() => { resolve(false) }, EXIT_MARKER_GRACE_MS)
timer.unref()
}),
])
}
/** Whether the child still lacks either OS termination marker. */
function isRunning(child: ChildProcessWithoutNullStreams): boolean {
return child.exitCode === null && child.signalCode === null

View File

@@ -11,20 +11,35 @@ const CWD = '{{cwd}}'
const SYSTEM = '{{system}}'
const TOOLS = '{{tools}}'
const MESSAGE_PREFIX = '{{messagePrefix}}'
const UPDATED_AT = '{{updatedAt}}'
/** A cwd-rooted path after volatile cwd replacement, through its last separator-delimited segment. */
const CWD_ROOTED_PATH_RE = /\{\{cwd\}\}(?:[\\/][^\s<>"'`]+)+/g
const PATH_TAG_RE = /(<path>)([^<]*)(<\/path>)/g
const ADDITIONAL_INSTRUCTIONS_PATH_RE = /(Additional instructions from: )([^\r\n]+)/g
/** A UUID v4 string, the shape `randomUUID()` produces for session ids. */
const UUID_RE = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/gi
const LOCAL_SPILL_PATH_RE = new RegExp(
String.raw`\{\{cwd\}\}/\.spill/session-[0-9a-f]{12}/[0-9a-f]{12}-([A-Za-z0-9._~-]+?)`
String.raw`\{\{cwd\}\}[\\/]\.spill[\\/]session-[0-9a-f]{12}[\\/][0-9a-f]{12}-([A-Za-z0-9._~-]+?)`
+ String.raw`(?=\. Use read with offset/limit|[\s)]|$)`,
'g',
)
const SNAPSHOT_SPILL_PATH_RE = new RegExp(
String.raw`/tmp/(?:dsh-acp-snap-[0-9a-f]{9}|dsh-acp-snapshot-spill)/session-[0-9a-f]{12}/[0-9a-f]{12}-([A-Za-z0-9._~-]+?)`
String.raw`(?:[A-Za-z]:)?[\\/](?:tmp|t)[\\/](?:dsh-acp-snap-[0-9a-f]{9}|dsh-acp-snapshot-spill)[\\/]session-[0-9a-f]{12}[\\/][0-9a-f]{12}-([A-Za-z0-9._~-]+?)`
+ String.raw`(?=\. Use read with offset/limit|[\s)]|$)`,
'g',
)
/** Convert separators only inside generated path-bearing text markers. */
function canonicalizeEmbeddedPaths(value: string): string {
return value
.replace(PATH_TAG_RE, (_match, open: string, path: string, close: string) =>
`${open}${path.replaceAll('\\', '/')}${close}`)
.replace(ADDITIONAL_INSTRUCTIONS_PATH_RE, (_match, prefix: string, path: string) =>
`${prefix}${path.replaceAll('\\', '/')}`)
}
/** Inputs the normalizers need to recognize a run's volatile values. */
export interface NormalizeContext {
/** The session id(s) the run issued — replaced with `{{sessionId}}`. */
@@ -33,13 +48,28 @@ export interface NormalizeContext {
cwd: string
}
/** How cwd-rooted path separators are represented after the cwd is tokenized. */
export type CwdPathMode = 'canonical' | 'native'
/** Optional controls shared by stdout and session-log normalization. */
export interface NormalizeOptions {
/** Use `/` for shared goldens, or preserve captured separators for a platform-specific golden. */
cwdPathMode?: CwdPathMode
}
/** Replace cwd, session ids, and any stray UUID with stable tokens in a string. */
function scrubString(value: string, ctx: NormalizeContext): string {
function scrubString(value: string, ctx: NormalizeContext, cwdPathMode: CwdPathMode): string {
let out = value
// cwd first (longest, most specific), then explicit session ids, then any
// residual UUID (covers ids that appear in places we didn't enumerate).
out = out.split(ctx.cwd).join(CWD)
out = out.split(`/private${CWD}`).join(CWD)
if (cwdPathMode === 'canonical') {
// Restrict separator conversion to paths rooted at the cwd token. A global
// backslash rewrite would corrupt regexes, commands, and model-authored text.
out = out.replace(CWD_ROOTED_PATH_RE, path => path.replaceAll('\\', '/'))
out = canonicalizeEmbeddedPaths(out)
}
out = out.replace(LOCAL_SPILL_PATH_RE, (_match, name: string) => `{{spillLocator:${name}}}`)
out = out.replace(SNAPSHOT_SPILL_PATH_RE, (_match, name: string) => `{{spillLocator:${name}}}`)
for (const id of ctx.sessionIds) out = out.split(id).join(SESSION_ID)
@@ -48,12 +78,15 @@ function scrubString(value: string, ctx: NormalizeContext): string {
}
/** Recursively scrub a parsed JSON value (strings replaced; structure kept). */
function scrubValue(value: unknown, ctx: NormalizeContext): unknown {
if (typeof value === 'string') return scrubString(value, ctx)
if (Array.isArray(value)) return value.map(v => scrubValue(v, ctx))
function scrubValue(value: unknown, ctx: NormalizeContext, cwdPathMode: CwdPathMode, key?: string): unknown {
if (typeof value === 'string') {
const scrubbed = scrubString(value, ctx, cwdPathMode)
return cwdPathMode === 'canonical' && key === 'path' ? scrubbed.replaceAll('\\', '/') : scrubbed
}
if (Array.isArray(value)) return value.map(v => scrubValue(v, ctx, cwdPathMode))
if (value !== null && typeof value === 'object') {
const out: Record<string, unknown> = {}
for (const [k, v] of Object.entries(value)) out[k] = scrubValue(v, ctx)
for (const [k, v] of Object.entries(value)) out[k] = scrubValue(v, ctx, cwdPathMode, k)
return out
}
return value
@@ -67,9 +100,15 @@ function scrubValue(value: unknown, ctx: NormalizeContext): unknown {
*
* @param rawStdout The captured stdout bytes, decoded utf8.
* @param ctx The run's volatile values to scrub.
* @param options Separator output controls; shared canonical paths are the default.
* @returns The normalized NDJSON transcript, one frame per line.
*/
export function normalizeStdout(rawStdout: string, ctx: NormalizeContext): string {
export function normalizeStdout(
rawStdout: string,
ctx: NormalizeContext,
options: NormalizeOptions = {},
): string {
const cwdPathMode = options.cwdPathMode ?? 'canonical'
const lines = rawStdout.split('\n').filter(line => line.trim().length > 0)
// Map each distinct JSON-RPC id (request/response correlate by id) to a stable
// sequence number, in first-seen order, so id churn doesn't perturb the expected output.
@@ -85,7 +124,9 @@ export function normalizeStdout(rawStdout: string, ctx: NormalizeContext): strin
if ('id' in frame && frame.id !== undefined && frame.id !== null) {
frame.id = stableId(frame.id)
}
return scrubValue(frame, ctx) as Record<string, unknown>
const update = (frame.params as { update?: Record<string, unknown> } | undefined)?.update
if (update?.sessionUpdate === 'session_info_update') update.updatedAt = UPDATED_AT
return scrubValue(frame, ctx, cwdPathMode) as Record<string, unknown>
})
return frames.map(f => JSON.stringify(f)).join('\n') + '\n'
}
@@ -99,9 +140,15 @@ export function normalizeStdout(rawStdout: string, ctx: NormalizeContext): strin
*
* @param rawLog The raw session `.jsonl` content.
* @param ctx The run's volatile values to scrub.
* @param options Separator output controls; shared canonical paths are the default.
* @returns The normalized JSONL log, one record per line.
*/
export function normalizeSessionLog(rawLog: string, ctx: NormalizeContext): string {
export function normalizeSessionLog(
rawLog: string,
ctx: NormalizeContext,
options: NormalizeOptions = {},
): string {
const cwdPathMode = options.cwdPathMode ?? 'canonical'
const lines = rawLog.split('\n').filter(line => line.trim().length > 0)
const records = lines.map((line) => {
const record = JSON.parse(line) as Record<string, unknown>
@@ -119,7 +166,7 @@ export function normalizeSessionLog(rawLog: string, ctx: NormalizeContext): stri
if ('durationMs' in data) data.durationMs = 0
}
}
return scrubValue(record, ctx) as Record<string, unknown>
return scrubValue(record, ctx, cwdPathMode) as Record<string, unknown>
})
return records.map(r => JSON.stringify(r)).join('\n') + '\n'
}

View File

@@ -21,6 +21,7 @@ import { join } from 'node:path'
import { describe, expect, it } from 'vitest'
import { type AgentUnderTest, type HarvestedLog, type InputScript, runScenario } from './harness.ts'
import {
type CwdPathMode,
type NormalizeContext,
normalizeSessionLog,
normalizeStdout,
@@ -35,6 +36,9 @@ const SYSTEM_PROMPT_SNAPSHOT = 'system-prompt.expected.md'
/** The structured tool-schema snapshot beside each header-pinning fixture. */
const TOOL_SCHEMAS_SNAPSHOT = 'tool-schemas.expected.json'
/** The optional full Windows-native stdout transcript. */
const WINDOWS_STDOUT_SNAPSHOT = 'stdout.expected.windows.jsonl'
/** Stable session-log token standing in for the sidecar's initial schemas. */
const TOOLS_TOKEN = '{{tools}}'
@@ -106,6 +110,61 @@ export interface Scenario {
* test and the scenario needs an independent project location.
*/
workspaceParent?: string
/**
* Whether Windows additionally compares stdout with native separators against
* `stdout.expected.windows.jsonl`. The shared canonical stdout expected output is still
* compared on every platform, and the fixture guard requires this sidecar
* exactly when the option is set.
*/
pinsNativeWindowsStdout?: boolean
/**
* Whether the driven behavior needs POSIX process semantics the harness
* cannot exercise on Windows (e.g. cancelling a live bash tool call kills a
* detached process group). The scenario's run test is skipped on Windows;
* its fixtures stay guarded on every platform.
*/
posixOnly?: boolean
}
/**
* Whether a scenario's run test is skipped for this mode and host: record mode
* skips authored (non-`recorded`) scenarios, and {@link Scenario.posixOnly}
* scenarios skip on Windows.
*
* @param scenario The scenario whose run test is being registered.
* @param recording Whether the suite runs in record mode.
* @param platform The running Node platform, injectable for unit coverage.
* @returns True when the scenario's run test must not execute.
*/
export function scenarioSkipped(
scenario: Scenario,
recording: boolean,
platform: NodeJS.Platform = process.platform,
): boolean {
if (recording && !scenario.recorded) return true
return scenario.posixOnly === true && platform === 'win32'
}
/** One stdout expected output selected for a platform run. */
interface StdoutExpectedVariant {
file: string
cwdPathMode: CwdPathMode
}
/**
* Select the shared stdout expected output plus any platform-native assertion declared by a scenario.
*
* @param scenario The scenario whose stdout contract is being selected.
* @param platform The running Node platform, injectable for unit coverage.
* @returns The ordered expected-output variants: shared canonical first, then optional Windows native.
*/
export function stdoutExpectedVariants(
scenario: Scenario,
platform: NodeJS.Platform = process.platform,
): StdoutExpectedVariant[] {
const canonical: StdoutExpectedVariant = { file: 'stdout.expected.jsonl', cwdPathMode: 'canonical' }
if (platform !== 'win32' || scenario.pinsNativeWindowsStdout !== true) return [canonical]
return [canonical, { file: WINDOWS_STDOUT_SNAPSHOT, cwdPathMode: 'native' }]
}
/** One suite's inputs: the agent to boot, where its fixtures live, and its scenario table. */
@@ -426,8 +485,21 @@ export function stabilizeRefreshLog(fresh: string, existing: string, replacement
for (const { from, to } of replacements) stable = stable.split(from).join(to)
const existingRecords = parseJsonlRecords(existing)
const records = parseJsonlRecords(stable)
let existingIndex = 0
let previousEventTime: unknown
for (let i = 0; i < records.length; i++) {
preserveFixtureVolatiles(records[i] as Record<string, unknown>, existingRecords[i])
const record = records[i] as Record<string, unknown>
const existingRecord = existingRecords[existingIndex]
const insertedTitle = record.type === 'session/title' && existingRecord?.type !== 'session/title'
if (insertedTitle) {
/* v8 ignore next -- a title is turn-enclosed, so a preceding event time exists in every valid fixture. */
if (typeof previousEventTime !== 'number') throw new Error('acp-snapshot: inserted title has no preceding event time')
record.time = previousEventTime
} else {
preserveFixtureVolatiles(record, existingRecord)
existingIndex += 1
}
if (typeof record.time === 'number') previousEventTime = record.time
}
return records.map(record => JSON.stringify(record)).join('\n') + '\n'
}
@@ -472,8 +544,9 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
scenarioSuite('snapshot scenarios', () => {
for (const scenario of scenarios) {
// In RECORD mode, only re-run the `recorded` (live-API) scenarios; the `authored` ones
// (sidecar-driven errors/cancel) are never re-recorded.
it.skipIf(RECORDING && !scenario.recorded)(`snapshot: ${scenario.name} matches the expected outputs`, async ({ expect }) => {
// (sidecar-driven errors/cancel) are never re-recorded. `posixOnly` scenarios skip on
// Windows, where their process semantics cannot be driven.
it.skipIf(scenarioSkipped(scenario, RECORDING))(`snapshot: ${scenario.name} matches the expected outputs`, async ({ expect }) => {
const dir = join(snapshotsDir, scenario.name)
const input = JSON.parse(await readFile(join(dir, 'input.json'), 'utf8')) as InputScript
const overrideFile = join(dir, 'replay.override.json')
@@ -578,11 +651,13 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
}
}
const stdout = normalizeStdout(result.rawStdout, ctx)
if (REFRESHING) {
await writeFile(join(dir, 'stdout.expected.jsonl'), stdout)
for (const expected of stdoutExpectedVariants(scenario)) {
const stdout = normalizeStdout(result.rawStdout, ctx, { cwdPathMode: expected.cwdPathMode })
if (REFRESHING) {
await writeFile(join(dir, expected.file), stdout)
}
await expect(stdout, `${expected.file} mismatch`).toMatchFileSnapshot(join(dir, expected.file))
}
await expect(stdout).toMatchFileSnapshot(join(dir, 'stdout.expected.jsonl'))
// A model turn always produces a log worth comparing; a hook scenario can
// produce one without a model turn (a `rejected` turn carrying `hook/*`).
@@ -669,10 +744,14 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
it('every registered scenario has its required fixture files', async () => {
// Every scenario needs input, stdout, a primary session fixture, and matching optional sidecars.
for (const { name, overridden, pinsHeader } of scenarios) {
for (const { name, overridden, pinsHeader, pinsNativeWindowsStdout } of scenarios) {
const dir = join(snapshotsDir, name)
expect(existsSync(join(dir, 'input.json')), `${name}/input.json`).toBe(true)
expect(existsSync(join(dir, 'stdout.expected.jsonl')), `${name}/stdout.expected.jsonl`).toBe(true)
expect(
existsSync(join(dir, WINDOWS_STDOUT_SNAPSHOT)),
`${name}/${WINDOWS_STDOUT_SNAPSHOT} presence must match \`pinsNativeWindowsStdout\``,
).toBe(pinsNativeWindowsStdout === true)
expect(existsSync(join(dir, 'session.jsonl')), `${name}/session.jsonl`).toBe(true)
expect(existsSync(join(dir, 'replay.override.json')), `${name}/replay.override.json presence must match \`overridden\``)
.toBe(overridden === true)

View File

@@ -1,7 +1,19 @@
/**
* Scripted ACP agent for snapshot-kit tests. A fixture-adjacent `behavior.json` controls the
* subprocess reached through the real harness path; the bin reports observations over ACP and
* writes scripted logs before exiting on stdin EOF.
* Scripted fake ACP agent bin for `dsh-acp-snapshot`'s unit specs. Speaks
* newline-delimited JSON-RPC on stdio like the real `dsh-acp-agent` bin, but
* every behavior — how prompts settle, whether session/new rejects, which
* session logs get persisted, what filesystem noise to leave — comes from a
* `behavior.json` sitting NEXT to the `$DSH_SNAPSHOT_FILE` fixture, so a spec
* scripts a whole subprocess run from data. The specs launch it through the
* REAL `runScenario` spawn path (tsx loader, temp cwd, env plumbing), so the
* harness plumbing is exercised for real; only the agent behind the protocol
* is scripted.
*
* The specs (not the golden tier) own this bin: it asserts nothing, echoes
* observable facts into `session/update` text chunks (env probe, permission
* outcome, seeded-workspace listing) for the spec to read off `rawStdout`, and
* exits 0 on stdin EOF after writing the scripted logs — mirroring the real
* bin's dispose-flush-exit shape.
*/
import { mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
@@ -39,6 +51,10 @@ interface Behavior {
cancelToolCallUpdate?: boolean
/** Before responding to a prompt, send a `session/request_permission` request and echo its outcome as a chunk. */
permissionProbe?: boolean
/** Before responding to a prompt, send an `elicitation/create` request and echo its response as a chunk. */
elicitationProbe?: boolean
/** How `session/set_mode` settles: an empty response (echoing the modeId as a chunk) or a JSON-RPC error. */
setMode?: 'respond' | 'error'
/** Echo the `DSH_SNAPSHOT_*` env the harness set as a chunk (spec-side env-plumbing assertions). */
echoEnv?: boolean
/** Echo the sorted cwd listing as a chunk (spec-side workspace-seeding assertions). */
@@ -84,8 +100,8 @@ let sessionId = ''
let sessionCwd = ''
/** The parked prompt request id while `hang-until-cancel` waits for the cancel notification. */
let parkedPromptId: number | string | null = null
/** Resolvers for permission-probe responses, keyed by outbound request id. */
const pendingPermission = new Map<number, (outcome: unknown) => void>()
/** Resolvers for outbound probe responses (permission/elicitation), keyed by request id. */
const pendingOutbound = new Map<number, (result: unknown) => void>()
/** Per-run `session/set_config_option` state: config id → current value (first vocabulary entry until set). */
const currentConfig: Record<string, string> = {}
@@ -160,8 +176,8 @@ async function handlePrompt(id: number | string): Promise<void> {
}
if (behavior.permissionProbe === true) {
const requestId = nextOutboundId++
const outcome = await new Promise<unknown>((resolve) => {
pendingPermission.set(requestId, resolve)
const result = await new Promise<unknown>((resolve) => {
pendingOutbound.set(requestId, resolve)
send({
id: requestId,
method: 'session/request_permission',
@@ -175,7 +191,24 @@ async function handlePrompt(id: number | string): Promise<void> {
},
})
})
chunk(`permission:${JSON.stringify(outcome)}`)
chunk(`permission:${JSON.stringify((result as { outcome?: unknown } | undefined)?.outcome ?? null)}`)
}
if (behavior.elicitationProbe === true) {
const requestId = nextOutboundId++
const result = await new Promise<unknown>((resolve) => {
pendingOutbound.set(requestId, resolve)
send({
id: requestId,
method: 'elicitation/create',
params: {
sessionId,
mode: 'form',
message: 'Approve this plan and leave plan mode?',
requestedSchema: { type: 'object', title: 'Plan review', properties: { choice: { type: 'string' }, custom: { type: 'string' } }, required: [] },
},
})
})
chunk(`elicitation:${JSON.stringify(result ?? null)}`)
}
switch (behavior.prompt ?? 'respond') {
case 'respond':
@@ -195,10 +228,10 @@ function handleFrame(frame: Record<string, unknown>): void {
const method = frame.method as string | undefined
const params = (frame.params ?? {}) as Record<string, unknown>
// A response to one of OUR outbound requests (the permission probe).
if (method === undefined && id !== undefined && typeof id === 'number' && pendingPermission.has(id)) {
const resolve = pendingPermission.get(id) as (outcome: unknown) => void
pendingPermission.delete(id)
resolve((frame.result as { outcome?: unknown } | undefined)?.outcome ?? null)
if (method === undefined && id !== undefined && typeof id === 'number' && pendingOutbound.has(id)) {
const resolve = pendingOutbound.get(id) as (result: unknown) => void
pendingOutbound.delete(id)
resolve(frame.result)
return
}
switch (method) {
@@ -219,6 +252,14 @@ function handleFrame(frame: Record<string, unknown>): void {
case 'session/prompt':
void handlePrompt(id as number | string)
return
case 'session/set_mode':
if ((behavior.setMode ?? 'respond') === 'error') {
respondError(id as number | string, 'unknown mode')
return
}
chunk(`setMode:${String(params.modeId)}`)
respond(id as number | string, {})
return
case 'session/set_config_option': {
const vocabulary = behavior.configOptions
const configId = params.configId as string
@@ -300,7 +341,10 @@ function flushLogsAndExit(): void {
`setTimeout(() => process.stdout.write(${JSON.stringify(`${frame}\n`)}), 50)`,
`setTimeout(() => process.stderr.write(${JSON.stringify('late inherited stderr\n')}), 75)`,
].join(';')
spawn(process.execPath, ['-e', code], { stdio: ['ignore', 1, 2] }).unref()
spawn(process.execPath, ['-e', code], {
detached: true,
stdio: ['ignore', 'inherit', 'inherit'],
}).unref()
}
process.exit(0)
}

View File

@@ -0,0 +1,2 @@
{"type":"session","id":"abababab-cdcd-4efe-8ada-badabadabada","createdAt":800,"cwd":"/var/folders/2g/b32ct0qn1d728l_v6tdkjytr0000gn/T/acp-snap-cwd-KBQJbW","parentSession":"f6fa7fcf-dd9c-4b39-8815-b25ddcebfd88"}
{"type":"request/header","seq":0,"time":2,"data":{"header":{"config":{"model":"fake"},"system":"{{system}}","tools":"{{tools}}"},"reason":"initial"}}

View File

@@ -5,7 +5,7 @@ import { delimiter, join, relative, sep } from 'node:path'
import { fileURLToPath } from 'node:url'
import { afterAll, describe, expect, it, vi } from 'vitest'
import { PROTOCOL_VERSION } from '@agentclientprotocol/sdk'
import { runScenario, type AgentUnderTest, type InputStep } from '../src/harness.ts'
import { runScenario, snapshotSpillRoot, type AgentUnderTest, type InputStep } from '../src/harness.ts'
import { launchAcpTestAgent } from '../src/launcher.ts'
const fsControl = vi.hoisted(() => ({ cleanupFailure: undefined as Error | undefined }))
@@ -60,6 +60,15 @@ async function scenario(behavior: object): Promise<{ dir: string; fixtureFile: s
const boot: InputStep[] = [{ op: 'initialize' }, { op: 'newSession' }]
it('keeps scenario-owned snapshot spill root length stable across platforms', () => {
const fixtureFile = '/fixtures/scenario/session.jsonl'
const posix = snapshotSpillRoot(fixtureFile, 'linux')
const windows = snapshotSpillRoot(fixtureFile, 'win32')
expect(posix).toMatch(/^\/tmp\/dsh-acp-snap-[0-9a-f]{9}$/)
expect(windows).toMatch(/^\/t\/dsh-acp-snap-[0-9a-f]{9}$/)
expect(windows.length + 2).toBe(posix.length)
})
function environmentEcho(rawStdout: string): Record<string, unknown> {
const frames = rawStdout.trim().split('\n')
.map(line => JSON.parse(line) as { params?: { update?: { content?: { text?: unknown } } } })
@@ -86,8 +95,8 @@ describe('runScenario', () => {
expect(clientClosed).toBe(true)
})
it('centralizes ACP boot, captures, updates, fail-closed permissions, and shutdown', { timeout: 20_000 }, async () => {
const { dir, fixtureFile } = await scenario({ permissionProbe: true, echoEnv: true, stderrNote: 'launcher stderr' })
it('centralizes ACP boot, captures, updates, fail-closed interactions, and shutdown', { timeout: 20_000 }, async () => {
const { dir, fixtureFile } = await scenario({ permissionProbe: true, elicitationProbe: true, echoEnv: true, stderrNote: 'launcher stderr' })
const sessionsRoot = await mkdtemp(join(tmpdir(), 'acp-launcher-sessions-'))
tempDirs.push(sessionsRoot)
const launched = launchAcpTestAgent({
@@ -111,6 +120,7 @@ describe('runScenario', () => {
expect((await nextChunk).sessionUpdate).toBe('agent_message_chunk')
expect(launched.updates.some(update => update.sessionUpdate === 'agent_message_chunk')).toBe(true)
expect(launched.rawStdout()).toContain('permission:{\\"outcome\\":\\"cancelled\\"}')
expect(launched.rawStdout()).toContain('elicitation:{\\"action\\":\\"cancel\\"}')
expect(launched.stderr()).toContain('launcher stderr')
const unmatched = expect(launched.waitForUpdate(() => false)).rejects.toThrow(/update stream closed/)
await launched.close()
@@ -143,6 +153,9 @@ describe('runScenario', () => {
update.sessionUpdate === 'agent_message_chunk'
&& update.content.type === 'text'
&& update.content.text === 'late inherited stdout')
// Arm rejection handling before close may exhaust the stream; the later assertion still
// observes the original promise and turns a missing inherited frame into the test failure.
void lateUpdate.catch(() => undefined)
await launched.close()
@@ -180,6 +193,97 @@ describe('runScenario', () => {
}
})
it('preserves the child error when the requested signal sets an exit marker', async () => {
const { dir } = await scenario({})
const launched = launchAcpTestAgent({ agent: AGENT, cwd: dir })
await launched.spawned
const childFailure = Object.assign(new Error('signal failed as the child exited'), { code: 'EPERM' })
const originalKill = launched.child.kill.bind(launched.child)
const kill = vi.spyOn(launched.child, 'kill').mockImplementation((signal) => {
expect(signal).toBe('SIGTERM')
originalKill('SIGKILL')
Object.defineProperty(launched.child, 'signalCode', { configurable: true, enumerable: true, writable: true, value: 'SIGTERM' })
return true
})
try {
launched.child.emit('error', childFailure)
await expect(launched.close('SIGTERM')).rejects.toBe(childFailure)
expect(kill).toHaveBeenCalledOnce()
} finally {
kill.mockRestore()
if (launched.child.exitCode === null && launched.child.signalCode === null) originalKill('SIGKILL')
}
})
it('preserves the child error when the requested signal publishes its exit marker later', async () => {
const { dir } = await scenario({})
const launched = launchAcpTestAgent({ agent: AGENT, cwd: dir })
await launched.spawned
const childFailure = Object.assign(new Error('signal failed before the delayed exit marker'), { code: 'EPERM' })
const originalKill = launched.child.kill.bind(launched.child)
const kill = vi.spyOn(launched.child, 'kill').mockImplementation((signal) => {
expect(signal).toBe('SIGTERM')
setTimeout(() => { originalKill('SIGKILL') }, 10)
return true
})
try {
launched.child.emit('error', childFailure)
await expect(launched.close('SIGTERM')).rejects.toBe(childFailure)
expect(kill).toHaveBeenCalledOnce()
} finally {
kill.mockRestore()
if (launched.child.exitCode === null && launched.child.signalCode === null) originalKill('SIGKILL')
}
})
it('preserves the child error when fallback refusal races with an exit marker', async () => {
const { dir } = await scenario({})
const launched = launchAcpTestAgent({ agent: AGENT, cwd: dir })
await launched.spawned
const childFailure = Object.assign(new Error('signal failed while the child exited'), { code: 'EPERM' })
const originalKill = launched.child.kill.bind(launched.child)
const kill = vi.spyOn(launched.child, 'kill').mockImplementation((signal) => {
if (signal === 'SIGTERM') return true
originalKill('SIGKILL')
Object.defineProperty(launched.child, 'signalCode', { configurable: true, enumerable: true, writable: true, value: 'SIGKILL' })
return false
})
try {
launched.child.emit('error', childFailure)
await expect(launched.close('SIGTERM')).rejects.toBe(childFailure)
expect(kill).toHaveBeenNthCalledWith(1, 'SIGTERM')
expect(kill).toHaveBeenNthCalledWith(2, 'SIGKILL')
} finally {
kill.mockRestore()
if (launched.child.exitCode === null && launched.child.signalCode === null) originalKill('SIGKILL')
}
})
it('preserves the child error after accepted fallback termination drains', async () => {
const { dir } = await scenario({})
const launched = launchAcpTestAgent({ agent: AGENT, cwd: dir })
await launched.spawned
const childFailure = Object.assign(new Error('requested signal failed before fallback'), { code: 'EPERM' })
const originalKill = launched.child.kill.bind(launched.child)
const kill = vi.spyOn(launched.child, 'kill').mockImplementation((signal) => {
if (signal === 'SIGTERM') return true
return originalKill('SIGKILL')
})
try {
launched.child.emit('error', childFailure)
await expect(launched.close('SIGTERM')).rejects.toBe(childFailure)
expect(kill).toHaveBeenNthCalledWith(1, 'SIGTERM')
expect(kill).toHaveBeenNthCalledWith(2, 'SIGKILL')
} finally {
kill.mockRestore()
if (launched.child.exitCode === null && launched.child.signalCode === null) originalKill('SIGKILL')
}
})
it('rejects promptly when fallback termination emits an error', async () => {
const { dir } = await scenario({})
const launched = launchAcpTestAgent({ agent: AGENT, cwd: dir })
@@ -294,7 +398,11 @@ describe('runScenario', () => {
expect(result.sessionLogs[0]?.createdAt).toBe(42)
expect(result.sessionLogs[0]?.content).toContain('turn/start')
// The harvested log embeds the run's REAL temp cwd (template-substituted).
expect(result.sessionLogs[0]?.content).toContain(result.cwd)
// The cwd is JSON-encoded in the log line, so compare the parsed field
// rather than substring-matching a raw path (which breaks when the path
// separator is escaped inside JSON text on Windows).
const sessionLine = result.sessionLogs[0]?.content.split('\n').find(l => l.includes('"type":"session"')) ?? '{}'
expect((JSON.parse(sessionLine) as { cwd?: string }).cwd).toBe(result.cwd)
})
it('forwards override/child fixture paths into the child env and captures stderr', { timeout: 20_000 }, async () => {
@@ -315,7 +423,17 @@ describe('runScenario', () => {
expect(result.stderr).toContain('fake bin booted')
expect(result.rawStdout).toContain('replay.override.json')
// Child paths ride one env var, joined with the platform delimiter.
expect(result.rawStdout).toContain(JSON.stringify(childFiles.join(delimiter)).slice(1, -1))
// Parse the fake bin's env-probe chunk rather than substring-matching a
// JSON-encoded path (the escaping breaks raw-substring compares on Windows).
const envChunk = result.rawStdout.split('\n')
.map(l => l.trim())
.filter(l => l.length > 0)
.map(l => JSON.parse(l) as { params?: { update?: { content?: { text?: string } } } })
.find(f => f.params?.update?.content?.text?.startsWith('env:'))
const env = JSON.parse((envChunk?.params?.update?.content?.text ?? 'env:{}').slice('env:'.length)) as {
childFiles: string | null
}
expect(env.childFiles).toBe(childFiles.join(delimiter))
})
it('gives concurrent scenarios distinct equal-length spill roots', { timeout: 20_000 }, async () => {
@@ -328,7 +446,10 @@ describe('runScenario', () => {
expect(roots.every(root => typeof root === 'string')).toBe(true)
expect(new Set(roots).size).toBe(2)
expect((roots[0] as string).length).toBe((roots[1] as string).length)
expect((roots[0] as string).length).toBe('/tmp/dsh-acp-snapshot-spill'.length)
expect(roots).toEqual([
snapshotSpillRoot(first.fixtureFile),
snapshotSpillRoot(second.fixtureFile),
])
})
it('seeds the workspace dir into the temp cwd before the run', { timeout: 20_000 }, async () => {
@@ -612,6 +733,69 @@ describe('runScenario', () => {
expect(result.sessionLogs).toHaveLength(0)
})
it('drives session/set_mode and swallows the expected rejection of setModeExpectError', { timeout: 20_000 }, async () => {
const { fixtureFile } = await scenario({})
const result = await runScenario(
{ steps: [...boot, { op: 'setMode', modeId: 'plan' }] },
{ agent: AGENT, mode: 'replay', fixtureFile },
)
expect(result.rawStdout).toContain('setMode:plan')
const rejecting = await scenario({ setMode: 'error' })
const rejected = await runScenario(
{ steps: [...boot, { op: 'setModeExpectError', modeId: 'yolo' }] },
{ agent: AGENT, mode: 'replay', fixtureFile: rejecting.fixtureFile },
)
expect(rejected.rawStdout).toContain('unknown mode')
})
it('fails the run when setModeExpectError unexpectedly succeeds, and both mode ops require a session', { timeout: 20_000 }, async () => {
const { fixtureFile } = await scenario({})
await expect(runScenario(
{ steps: [...boot, { op: 'setModeExpectError', modeId: 'plan' }] },
{ agent: AGENT, mode: 'replay', fixtureFile },
)).rejects.toThrow(/expected session\/set_mode to be rejected/)
await expect(runScenario(
{ steps: [{ op: 'initialize' }, { op: 'setMode', modeId: 'plan' }] },
{ agent: AGENT, mode: 'replay', fixtureFile },
)).rejects.toThrow(/setMode before newSession/)
await expect(runScenario(
{ steps: [{ op: 'initialize' }, { op: 'setModeExpectError', modeId: 'plan' }] },
{ agent: AGENT, mode: 'replay', fixtureFile },
)).rejects.toThrow(/setModeExpectError before newSession/)
})
it('answers elicitations from the scripted queue, falling back to cancel on exhaustion', { timeout: 20_000 }, async () => {
const { fixtureFile } = await scenario({ elicitationProbe: true })
// Three prompts → three elicitations: an accept-with-choice, an
// accept-with-custom (feedback), then the exhausted-queue cancel.
const result = await runScenario(
{
steps: [...boot, { op: 'prompt', text: 'one' }, { op: 'prompt', text: 'two' }, { op: 'prompt', text: 'three' }],
elicitationAnswers: [
{ action: 'accept', choice: 'Approve' },
{ action: 'accept', custom: 'add tests first' },
],
},
{ agent: AGENT, mode: 'replay', fixtureFile },
)
const first = result.rawStdout.indexOf('elicitation:{\\"action\\":\\"accept\\",\\"content\\":{\\"choice\\":\\"Approve\\"}}')
const second = result.rawStdout.indexOf('elicitation:{\\"action\\":\\"accept\\",\\"content\\":{\\"custom\\":\\"add tests first\\"}}')
const third = result.rawStdout.indexOf('elicitation:{\\"action\\":\\"cancel\\"}')
expect(first).toBeGreaterThanOrEqual(0)
expect(second).toBeGreaterThan(first)
expect(third).toBeGreaterThan(second)
})
it('a scripted elicitation cancel answers cancel', { timeout: 20_000 }, async () => {
const { fixtureFile } = await scenario({ elicitationProbe: true })
const result = await runScenario(
{ steps: [...boot, { op: 'prompt', text: 'one' }], elicitationAnswers: [{ action: 'cancel' }] },
{ agent: AGENT, mode: 'replay', fixtureFile },
)
expect(result.rawStdout).toContain('elicitation:{\\"action\\":\\"cancel\\"}')
})
it('answers permission requests from the scripted queue by option kind, falling back to cancelled', { timeout: 20_000 }, async () => {
const { fixtureFile } = await scenario({ permissionProbe: true })
// Two prompts → two permission round-trips; one scripted answer, so the
@@ -640,8 +824,11 @@ describe('runScenario', () => {
it('rejects the run on a scripted permission kind the agent never offered', { timeout: 20_000 }, async () => {
const { fixtureFile } = await scenario({ permissionProbe: true })
// The fake offers only allow_once/reject_once. The harness must reject an impossible click,
// not merely send an RPC error that a tolerant agent could absorb.
// The fake bin offers allow_once/reject_once; scripting allow_always is a
// scenario bug. The agent is answered `cancelled` (it must not be able to
// absorb the bug as an error-means-denial), and the RUN fails: a callback
// throw would only reach the agent as a JSON-RPC error response, letting
// a tolerant agent carry on and the scenario pass — or record.
await expect(runScenario(
{ steps: [...boot, { op: 'prompt', text: 'impossible click' }], permissionAnswers: [{ kind: 'allow_always' }] },
{ agent: AGENT, mode: 'replay', fixtureFile },

View File

@@ -44,6 +44,56 @@ describe('normalizeStdout', () => {
expect(out).not.toContain(ctx.sessionIds[0] as string)
})
it('canonicalizes only cwd-rooted path separators', () => {
const windowsCtx: NormalizeContext = {
sessionIds: [],
cwd: String.raw`C:\Users\runner\AppData\Local\Temp\acp-snapshot`,
}
const raw = JSON.stringify({
jsonrpc: '2.0',
method: 'session/update',
params: {
path: `${windowsCtx.cwd}\\nested\\proof.txt`,
regex: String.raw`\d+\w+`,
command: String.raw`printf "\\n"`,
},
})
const frame = JSON.parse(normalizeStdout(raw, windowsCtx)) as {
params: { path: string; regex: string; command: string }
}
expect(frame.params).toEqual({
path: '{{cwd}}/nested/proof.txt',
regex: String.raw`\d+\w+`,
command: String.raw`printf "\\n"`,
})
})
it('canonicalizes generated relative path fields and text markers without rewriting other text', () => {
const raw = JSON.stringify({
path: String.raw`nested\AGENTS.md`,
content: String.raw`<path>.\nested\task.txt</path>
Additional instructions from: nested\AGENTS.md`,
regex: String.raw`\d+\w+`,
})
const frame = JSON.parse(normalizeStdout(raw, { sessionIds: [], cwd: '/unused' })) as {
path: string
content: string
regex: string
}
expect(frame).toEqual({
path: 'nested/AGENTS.md',
content: '<path>./nested/task.txt</path>\nAdditional instructions from: nested/AGENTS.md',
regex: String.raw`\d+\w+`,
})
})
it('can preserve native cwd-rooted separators for a platform golden', () => {
const windowsCtx: NormalizeContext = { sessionIds: [], cwd: String.raw`C:\work\snapshot` }
const raw = JSON.stringify({ path: `${windowsCtx.cwd}\\nested\\proof.txt` })
const frame = JSON.parse(normalizeStdout(raw, windowsCtx, { cwdPathMode: 'native' })) as { path: string }
expect(frame.path).toBe(String.raw`{{cwd}}\nested\proof.txt`)
})
it('scrubs a stray UUID not in the known list', () => {
const raw = JSON.stringify({ jsonrpc: '2.0', method: 'x', params: { id: 'aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee' } })
expect(normalizeStdout(raw, ctx)).toContain('{{sessionId}}')
@@ -55,6 +105,24 @@ describe('normalizeStdout', () => {
expect(out).not.toContain('"id"')
})
it('stabilizes the timestamp carried by session title updates', () => {
const raw = JSON.stringify({
jsonrpc: '2.0',
method: 'session/update',
params: {
sessionId: ctx.sessionIds[0],
update: {
sessionUpdate: 'session_info_update',
title: 'Stable title',
updatedAt: '2026-07-20T17:03:13.689Z',
},
},
})
const out = normalizeStdout(raw, ctx)
expect(out).toContain('"updatedAt":"{{updatedAt}}"')
expect(out).not.toContain('2026-07-20T17:03:13.689Z')
})
it('throws on a non-JSON stdout line (the purity check)', () => {
const raw = `${JSON.stringify({ jsonrpc: '2.0', id: 1 })}\noops a log leaked\n`
expect(() => normalizeStdout(raw, ctx)).toThrow()
@@ -154,6 +222,33 @@ describe('normalizeSessionLog', () => {
expect(out).not.toContain('/tmp/dsh-acp-snap-012345678')
})
it('scrubs scenario-owned snapshot spill paths with Windows drive and separators', () => {
const ev = JSON.stringify({
type: 'tool/result', seq: 2, time: 5,
data: {
content: [{
type: 'text',
text: String.raw`Full formatted result stored at: C:\t\dsh-acp-snap-012345678\session-c22bc3f1d2af\8a7b6c5d4e3f-bash.txt. Use read with offset/limit, or grep this path to search within it.`,
}],
},
})
const out = normalizeSessionLog(`${header({ cwd: ctx.cwd })}\n${ev}\n`, ctx)
expect(out).toContain('{{spillLocator:bash.txt}}')
expect(out).not.toContain('C:\\t\\dsh-acp-snap-012345678')
})
it('shares cwd-rooted path handling with stdout normalization', () => {
const windowsCtx: NormalizeContext = { sessionIds: [], cwd: String.raw`C:\work\snapshot` }
const ev = JSON.stringify({
type: 'tool/result', seq: 2, time: 5,
data: { path: `${windowsCtx.cwd}\\nested\\proof.txt` },
})
expect(normalizeSessionLog(`${header({ cwd: windowsCtx.cwd })}\n${ev}\n`, windowsCtx))
.toContain('{{cwd}}/nested/proof.txt')
expect(normalizeSessionLog(`${header({ cwd: windowsCtx.cwd })}\n${ev}\n`, windowsCtx, { cwdPathMode: 'native' }))
.toContain(String.raw`{{cwd}}\\nested\\proof.txt`)
})
it('scrubs the session id in the header', () => {
const out = normalizeSessionLog(`${header({ id: ctx.sessionIds[0] })}\n`, ctx)
expect(out).toContain('{{sessionId}}')

View File

@@ -15,9 +15,11 @@ import {
normalizedToolSchemas,
parseToolSchemasSnapshot,
refreshFixtureReplacements,
scenarioSkipped,
sessionFixtureNames,
restorePinnedToolSchemas,
stabilizeRefreshLog,
stdoutExpectedVariants,
unknownToolCallIds,
} from '../src/suite.ts'
@@ -237,6 +239,48 @@ describe('sessionFixtureNames', () => {
})
})
describe('stdoutExpectedVariants', () => {
const scenario: Scenario = {
name: 'windows-native',
hasModelTurn: true,
recorded: true,
pinsNativeWindowsStdout: true,
}
it('adds the native sidecar after the shared golden on Windows', () => {
expect(stdoutExpectedVariants(scenario, 'win32')).toEqual([
{ file: 'stdout.expected.jsonl', cwdPathMode: 'canonical' },
{ file: 'stdout.expected.windows.jsonl', cwdPathMode: 'native' },
])
})
it('keeps only the shared golden on other platforms or without the declaration', () => {
expect(stdoutExpectedVariants(scenario, 'linux')).toEqual([
{ file: 'stdout.expected.jsonl', cwdPathMode: 'canonical' },
])
expect(stdoutExpectedVariants({ ...scenario, pinsNativeWindowsStdout: false }, 'win32')).toEqual([
{ file: 'stdout.expected.jsonl', cwdPathMode: 'canonical' },
])
})
})
describe('scenarioSkipped', () => {
const authored: Scenario = { name: 'authored', hasModelTurn: true, recorded: false }
const posix: Scenario = { name: 'posix-cancel', hasModelTurn: true, recorded: false, posixOnly: true }
it('skips authored scenarios only while recording', () => {
expect(scenarioSkipped(authored, true, 'linux')).toBe(true)
expect(scenarioSkipped(authored, false, 'linux')).toBe(false)
})
it('skips posixOnly scenarios on Windows and nowhere else', () => {
expect(scenarioSkipped(posix, false, 'win32')).toBe(true)
expect(scenarioSkipped(posix, false, 'linux')).toBe(false)
expect(scenarioSkipped(posix, false, 'darwin')).toBe(false)
expect(scenarioSkipped(authored, false, 'win32')).toBe(false)
})
})
describe('fixtureContext', () => {
it('reads the fixture header id and cwd', () => {
const ctx = fixtureContext('{"type":"session","id":"abc","cwd":"/rec"}\n{"type":"turn/start"}\n')
@@ -414,6 +458,36 @@ describe('refreshFixtureReplacements', () => {
})
describe('stabilizeRefreshLog', () => {
it('aligns volatile times across a newly inserted log event', () => {
const fresh = [
'{"type":"session","id":"same","createdAt":200}',
'{"type":"turn/start","seq":0,"time":21}',
'{"type":"user/message","seq":1,"time":22}',
'{"type":"session/title","seq":2,"time":999}',
'{"type":"step/start","seq":3,"time":1000}',
'{"type":"request/header","seq":4,"time":1001}',
'',
].join('\n')
const existing = [
'{"type":"session","id":"same","createdAt":100}',
'{"type":"turn/start","seq":0,"time":11}',
'{"type":"user/message","seq":1,"time":12}',
'{"type":"step/start","seq":2,"time":13}',
'{"type":"request/header","seq":3,"time":14}',
'',
].join('\n')
expect(stabilizeRefreshLog(fresh, existing, [])).toBe([
'{"type":"session","id":"same","createdAt":100}',
'{"type":"turn/start","seq":0,"time":11}',
'{"type":"user/message","seq":1,"time":12}',
'{"type":"session/title","seq":2,"time":12}',
'{"type":"step/start","seq":3,"time":13}',
'{"type":"request/header","seq":4,"time":14}',
'',
].join('\n'))
})
it('keeps volatile fixture fields while preserving fresh meaningful payloads', () => {
const fresh = [
'{"type":"session","id":"new-child","createdAt":200,"cwd":"/new","parentSession":"new-parent","seedLength":1}',

View File

@@ -24,7 +24,7 @@ export const LOADER_SMOKE_TEST_TIMEOUT_MS = DEFAULT_PROCESS_TIMEOUT_MS + 15_000
/** Which artifact an example bin is booted from: unbuilt `src` via tsx, or built `lib` via plain Node. */
export type ExampleMode = 'src' | 'lib'
/** Environment variable selecting the mode; CI and pre-push set it to `lib`, dev leaves it unset (`src`). */
/** Environment variable selecting the mode; CI sets it to `lib`, dev leaves it unset (`src`). */
export const EXAMPLE_MODE_ENV = 'DSH_EXAMPLE_MODE'
/**
@@ -138,8 +138,6 @@ export interface LoaderSmokeOptions {
readonly mode?: ExampleMode
/** Environment overrides layered over the parent and isolated DSH homes. */
readonly env?: Readonly<NodeJS.ProcessEnv>
/** Lines written to stdin before EOF; omitted means immediate EOF. */
readonly stdinLines?: readonly string[]
/** Process deadline override for harness tests. */
readonly processTimeoutMs?: number
/** Optional world-state setup run in the isolated cwd before process start. */
@@ -157,10 +155,10 @@ export interface LoaderSmokeResult {
}
/**
* Boot one real Loader tree from an isolated cwd, write the requested stdin
* script, close stdin, and await a clean exit. The helper owns process kill and
* temp-directory cleanup on every outcome, and picks src/lib via {@link resolveExampleLaunch}.
* @param options - example paths, mode, environment, stdin, and diagnostic identity.
* Boot one real Loader tree from an isolated cwd, close stdin immediately, and
* await a clean exit. The helper owns process kill and temp-directory cleanup on
* every outcome, and picks src/lib via {@link resolveExampleLaunch}.
* @param options - example paths, mode, environment, and diagnostic identity.
* @returns captured stdout and stderr after a zero exit.
*/
export async function runLoaderSmoke(options: LoaderSmokeOptions): Promise<LoaderSmokeResult> {
@@ -220,7 +218,7 @@ export async function runLoaderSmoke(options: LoaderSmokeOptions): Promise<Loade
})
/* v8 ignore stop */
child.stdin.end((options.stdinLines ?? []).map(line => `${line}\n`).join(''))
child.stdin.end()
})
await options.inspect?.(cwd)
return result

View File

@@ -11,7 +11,7 @@ const fixture = (name: string): string => fileURLToPath(new URL(`./fixtures/${na
const canonicalTempPath = (path: string): string => path.replace(/^\/private(?=\/var\/)/, '')
describe('runLoaderSmoke', () => {
it('isolates the process, writes stdin, captures output, and removes the cwd', async () => {
it('isolates the process, closes stdin, captures output, and removes the cwd', async () => {
const result = await runLoaderSmoke({
label: 'success fixture',
tempDirPrefix: 'loader-smoke-success-',
@@ -20,7 +20,6 @@ describe('runLoaderSmoke', () => {
tsconfigPath,
mode: 'src',
env: { LOADER_SMOKE_MARKER: 'present' },
stdinLines: ['one', 'two'],
})
const output = JSON.parse(result.stdout) as {
configPath: string
@@ -35,10 +34,10 @@ describe('runLoaderSmoke', () => {
configPath,
args: [configPath],
marker: 'present',
input: 'one\ntwo\n',
input: '',
})
expect(canonicalTempPath(output.dshHome)).toBe(`${canonicalTempPath(output.cwd)}/.dsh`)
expect(canonicalTempPath(output.agentsHome)).toBe(`${canonicalTempPath(output.cwd)}/.agents`)
expect(canonicalTempPath(output.dshHome)).toBe(canonicalTempPath(join(output.cwd, '.dsh')))
expect(canonicalTempPath(output.agentsHome)).toBe(canonicalTempPath(join(output.cwd, '.agents')))
expect(result.stderr).toContain('fixture stderr')
expect(existsSync(output.cwd)).toBe(false)
}, LOADER_SMOKE_TEST_TIMEOUT_MS)