|
|
|
|
@@ -21,9 +21,12 @@ import { existsSync } from 'node:fs'
|
|
|
|
|
import { createHash } from 'node:crypto'
|
|
|
|
|
import { tmpdir } from 'node:os'
|
|
|
|
|
import { basename, dirname, join, delimiter } from 'node:path'
|
|
|
|
|
import { setTimeout as delay } from 'node:timers/promises'
|
|
|
|
|
import {
|
|
|
|
|
ClientSideConnection,
|
|
|
|
|
PROTOCOL_VERSION,
|
|
|
|
|
type CreateElicitationRequest,
|
|
|
|
|
type CreateElicitationResponse,
|
|
|
|
|
type RequestPermissionRequest,
|
|
|
|
|
type RequestPermissionResponse,
|
|
|
|
|
type SessionNotification,
|
|
|
|
|
@@ -32,6 +35,9 @@ import { launchAcpTestAgent, type AgentUnderTest, type LaunchedAcpTestAgent } fr
|
|
|
|
|
|
|
|
|
|
export type { AgentUnderTest } from './launcher.ts'
|
|
|
|
|
|
|
|
|
|
const DEFAULT_WAIT_TIMEOUT_MS = 10_000
|
|
|
|
|
const WAIT_POLL_INTERVAL_MS = 10
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* One step of a scenario's deterministic input script (`input.json`). The
|
|
|
|
|
* harness interprets these in order. `newSession` captures the server-issued
|
|
|
|
|
@@ -40,10 +46,13 @@ export type { AgentUnderTest } from './launcher.ts'
|
|
|
|
|
*
|
|
|
|
|
* `promptAndCancel` starts a prompt without awaiting completion, waits until
|
|
|
|
|
* the client observes the selected update (`agent_message_chunk` by default),
|
|
|
|
|
* then cancels and awaits completion. A named `waitForToolCallUpdate` keeps the
|
|
|
|
|
* step open for a terminal tool update that may follow the prompt response.
|
|
|
|
|
* then cancels and awaits completion. An optional `waitForFile` first observes
|
|
|
|
|
* a cwd-relative readiness marker, and a named `waitForToolCallUpdate` keeps
|
|
|
|
|
* the step open for a terminal tool update that may follow the prompt response.
|
|
|
|
|
* `promptAndWaitForAgentMessage` arms an exact text-chunk waiter before sending
|
|
|
|
|
* the prompt, then keeps the application live until that later update arrives.
|
|
|
|
|
* `waitForTurnEnd` holds the subprocess open until the selected session's latest
|
|
|
|
|
* complete raw-JSONL turn boundary is `turn/end`; its timeout defaults to 10s.
|
|
|
|
|
*/
|
|
|
|
|
export type InputStep =
|
|
|
|
|
| { op: 'initialize'; terminalOutput?: boolean }
|
|
|
|
|
@@ -56,9 +65,13 @@ export type InputStep =
|
|
|
|
|
op: 'promptAndCancel'
|
|
|
|
|
text: string
|
|
|
|
|
afterUpdate?: 'agent_message_chunk' | 'tool_call'
|
|
|
|
|
waitForFile?: { path: string; timeoutMs?: number }
|
|
|
|
|
waitForToolCallUpdate?: string
|
|
|
|
|
}
|
|
|
|
|
| { op: 'waitForTurnEnd'; timeoutMs?: number }
|
|
|
|
|
| { op: 'cancel' }
|
|
|
|
|
| { op: 'setMode'; modeId: string }
|
|
|
|
|
| { op: 'setModeExpectError'; modeId: string }
|
|
|
|
|
| { op: 'setConfigOption'; configId: string; value: string }
|
|
|
|
|
| { op: 'setConfigOptionExpectError'; configId: string; value: string }
|
|
|
|
|
|
|
|
|
|
@@ -78,6 +91,16 @@ export interface InputScript {
|
|
|
|
|
* agent itself just sees `cancelled`, so it cannot absorb the bug).
|
|
|
|
|
*/
|
|
|
|
|
permissionAnswers?: PermissionAnswer[]
|
|
|
|
|
/**
|
|
|
|
|
* Ordered answers for the agent's `elicitation/create` round-trips (the
|
|
|
|
|
* ask_user_question / plan-review forms), consumed FIFO — the Nth request
|
|
|
|
|
* gets the Nth answer. Exhaustion (or no queue) answers `cancel`, the same
|
|
|
|
|
* fail-closed stub an elicitation-free scenario relies on. Unlike permission
|
|
|
|
|
* kinds, the scripted strings are not validated against the offered form —
|
|
|
|
|
* a stray `choice` reaches the agent verbatim, which reads it as a custom
|
|
|
|
|
* (non-consenting) answer, so a scenario bug fails safe in the transcript.
|
|
|
|
|
*/
|
|
|
|
|
elicitationAnswers?: ElicitationAnswer[]
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/** One scripted answer to a permission request: which offered option kind to select. */
|
|
|
|
|
@@ -86,6 +109,16 @@ export interface PermissionAnswer {
|
|
|
|
|
kind: 'allow_once' | 'allow_always' | 'reject_once' | 'reject_always'
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/** One scripted answer to an elicitation form (accept with choice/custom content, or cancel). */
|
|
|
|
|
export interface ElicitationAnswer {
|
|
|
|
|
/** Accept the form with the content below, or cancel it. */
|
|
|
|
|
action: 'accept' | 'cancel'
|
|
|
|
|
/** The selected option label (the form's `choice` field). */
|
|
|
|
|
choice?: string
|
|
|
|
|
/** Free-form text (the form's `custom` field). */
|
|
|
|
|
custom?: string
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/** One harvested session log plus the identifying facts off its header line. */
|
|
|
|
|
export interface HarvestedLog {
|
|
|
|
|
/** The recorded session id (header `id`). */
|
|
|
|
|
@@ -106,7 +139,7 @@ export interface RunResult {
|
|
|
|
|
stderr: string
|
|
|
|
|
/** The session id the server issued (undefined if no session was created). */
|
|
|
|
|
sessionId?: string
|
|
|
|
|
/** The temp cwd the session ran in (the bash workspace). */
|
|
|
|
|
/** The generated cwd the session ran in (the bash workspace). */
|
|
|
|
|
cwd: string
|
|
|
|
|
/**
|
|
|
|
|
* Every persisted session log harvested after the run, ordered primary-first:
|
|
|
|
|
@@ -137,11 +170,19 @@ export interface RunOptions {
|
|
|
|
|
childFiles?: string[]
|
|
|
|
|
/**
|
|
|
|
|
* Optional `<scenario>/workspace/` directory whose contents are copied into
|
|
|
|
|
* the temp cwd BEFORE the run — the standard way to seed files the agent
|
|
|
|
|
* the generated cwd BEFORE the run — the standard way to seed files the agent
|
|
|
|
|
* operates on (a file to read, edit, or grep). Absent for scenarios that
|
|
|
|
|
* start from an empty workspace.
|
|
|
|
|
*/
|
|
|
|
|
workspaceDir?: string
|
|
|
|
|
/**
|
|
|
|
|
* Parent directory for the generated session cwd. Defaults to
|
|
|
|
|
* `os.tmpdir()`. A scenario that must distinguish its workspace from the
|
|
|
|
|
* sandbox's always-writable temporary roots can place the generated child
|
|
|
|
|
* under `os.homedir()` instead. The harness removes only that generated
|
|
|
|
|
* child, never the supplied parent.
|
|
|
|
|
*/
|
|
|
|
|
workspaceParent?: string
|
|
|
|
|
/**
|
|
|
|
|
* Alternate LIVE config path for the boot (absolute), overriding
|
|
|
|
|
* {@link AgentUnderTest.configPath} for this run. A scenario needing a
|
|
|
|
|
@@ -172,15 +213,15 @@ export function snapshotSpillRoot(
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Run a scenario end-to-end against a freshly-spawned subprocess. Owns the
|
|
|
|
|
* child and its temp dirs; always tears them down. Returns the captured stdout
|
|
|
|
|
* child and its generated dirs; always tears them down. Returns the captured stdout
|
|
|
|
|
* and (record mode) the harvested session-log path.
|
|
|
|
|
*
|
|
|
|
|
* @param input The scenario's input script (steps + optional permission answers).
|
|
|
|
|
* @param opts The agent to boot, the mode, and the fixture wiring.
|
|
|
|
|
* @returns The captured stdout/stderr, session id, temp cwd, and harvested logs.
|
|
|
|
|
* @returns The captured stdout/stderr, session id, generated cwd, and harvested logs.
|
|
|
|
|
*/
|
|
|
|
|
export async function runScenario(input: InputScript, opts: RunOptions): Promise<RunResult> {
|
|
|
|
|
const cwd = await mkdtemp(join(tmpdir(), 'acp-snap-cwd-'))
|
|
|
|
|
const cwd = await mkdtemp(join(opts.workspaceParent ?? tmpdir(), 'acp-snap-cwd-'))
|
|
|
|
|
const sessionsRoot = await mkdtemp(join(tmpdir(), 'acp-snap-sessions-'))
|
|
|
|
|
// Fixed path length: spill-policy budgets the preview against the REAL path
|
|
|
|
|
// before stdout normalization, so tmpdir() length differences churn expected outputs.
|
|
|
|
|
@@ -194,7 +235,7 @@ export async function runScenario(input: InputScript, opts: RunOptions): Promise
|
|
|
|
|
let sessionLogs: HarvestedLog[] = []
|
|
|
|
|
const outcome = await (async (): Promise<RunResult> => {
|
|
|
|
|
// Seed the workspace if the scenario ships one (a file the agent reads/edits).
|
|
|
|
|
// Copied into the temp cwd so the agent's bash tools see it; the expected outputs
|
|
|
|
|
// Copied into the generated cwd so the agent's bash tools see it; the expected outputs
|
|
|
|
|
// normalize the cwd, so the seeded paths stay stable across runs.
|
|
|
|
|
if (opts.workspaceDir !== undefined && existsSync(opts.workspaceDir)) {
|
|
|
|
|
await cp(opts.workspaceDir, cwd, { recursive: true })
|
|
|
|
|
@@ -215,6 +256,8 @@ export async function runScenario(input: InputScript, opts: RunOptions): Promise
|
|
|
|
|
// Permission answers are consumed FIFO across the whole run; exhaustion
|
|
|
|
|
// falls back to `cancelled` so approval-free scenarios keep the plain stub.
|
|
|
|
|
const permissionQueue = [...input.permissionAnswers ?? []]
|
|
|
|
|
// Elicitation answers mirror the permission queue: FIFO, cancel on exhaustion.
|
|
|
|
|
const elicitationQueue = [...input.elicitationAnswers ?? []]
|
|
|
|
|
// A scenario bug detected inside a client callback (a scripted permission
|
|
|
|
|
// kind the agent never offered). It cannot fail the run from in there: a
|
|
|
|
|
// callback throw only becomes a JSON-RPC error RESPONSE to the agent, and
|
|
|
|
|
@@ -244,13 +287,32 @@ export async function runScenario(input: InputScript, opts: RunOptions): Promise
|
|
|
|
|
}
|
|
|
|
|
return Promise.resolve({ outcome: { outcome: 'selected', optionId: option.optionId } })
|
|
|
|
|
},
|
|
|
|
|
createElicitation(_params: CreateElicitationRequest): Promise<CreateElicitationResponse> {
|
|
|
|
|
const answer = elicitationQueue.shift()
|
|
|
|
|
if (answer === undefined || answer.action !== 'accept') return Promise.resolve({ action: 'cancel' })
|
|
|
|
|
return Promise.resolve({
|
|
|
|
|
action: 'accept',
|
|
|
|
|
content: {
|
|
|
|
|
...answer.choice !== undefined ? { choice: answer.choice } : {},
|
|
|
|
|
...answer.custom !== undefined ? { custom: answer.custom } : {},
|
|
|
|
|
},
|
|
|
|
|
})
|
|
|
|
|
},
|
|
|
|
|
})
|
|
|
|
|
const active = launched
|
|
|
|
|
await active.spawned
|
|
|
|
|
const { client } = active
|
|
|
|
|
|
|
|
|
|
for (const step of input.steps) {
|
|
|
|
|
await runStep(client, step, cwd, match => active.waitForUpdate(match), () => sessionId, (id) => { sessionId = id })
|
|
|
|
|
await runStep(
|
|
|
|
|
client,
|
|
|
|
|
step,
|
|
|
|
|
cwd,
|
|
|
|
|
match => active.waitForUpdate(match),
|
|
|
|
|
() => sessionId,
|
|
|
|
|
(id) => { sessionId = id },
|
|
|
|
|
(id, timeoutMs) => waitForPersistedTurnEnd(sessionsRoot, id, timeoutMs),
|
|
|
|
|
)
|
|
|
|
|
// A permission exchange happens while a step's request is in flight, so
|
|
|
|
|
// by the time the step settles any script bug it exposed is captured —
|
|
|
|
|
// fail the run HERE, as a harness error, rather than hoping the agent's
|
|
|
|
|
@@ -261,7 +323,7 @@ export async function runScenario(input: InputScript, opts: RunOptions): Promise
|
|
|
|
|
// persistence) and exits. Then await exit so the harvested log is complete.
|
|
|
|
|
await active.close()
|
|
|
|
|
// Harvest EVERY persisted log (parent + any subagent children) while the
|
|
|
|
|
// temp dirs still exist, ordered primary-first.
|
|
|
|
|
// generated dirs still exist, ordered primary-first.
|
|
|
|
|
sessionLogs = await harvestSessionLogs(sessionsRoot)
|
|
|
|
|
return {
|
|
|
|
|
rawStdout: launched.rawStdout(),
|
|
|
|
|
@@ -320,6 +382,7 @@ async function runStep(
|
|
|
|
|
waitForUpdate: (match: (u: SessionNotification['update']) => boolean) => Promise<SessionNotification['update']>,
|
|
|
|
|
getSessionId: () => string | undefined,
|
|
|
|
|
setSessionId: (id: string) => void,
|
|
|
|
|
waitForTurnEnd: (sessionId: string, timeoutMs?: number) => Promise<void>,
|
|
|
|
|
): Promise<void> {
|
|
|
|
|
switch (step.op) {
|
|
|
|
|
case 'initialize':
|
|
|
|
|
@@ -384,6 +447,9 @@ async function runStep(
|
|
|
|
|
const promptDone = client.prompt({ sessionId, prompt: [{ type: 'text', text: step.text }] })
|
|
|
|
|
const afterUpdate = step.afterUpdate ?? 'agent_message_chunk'
|
|
|
|
|
await waitForUpdate(u => u.sessionUpdate === afterUpdate)
|
|
|
|
|
if (step.waitForFile !== undefined) {
|
|
|
|
|
await waitForWorkspaceFile(cwd, step.waitForFile.path, step.waitForFile.timeoutMs)
|
|
|
|
|
}
|
|
|
|
|
// Arm this before cancellation so a fast tool drain cannot outrun the waiter.
|
|
|
|
|
const toolCallUpdateDone = step.waitForToolCallUpdate === undefined
|
|
|
|
|
? undefined
|
|
|
|
|
@@ -393,12 +459,36 @@ async function runStep(
|
|
|
|
|
if (toolCallUpdateDone !== undefined) await toolCallUpdateDone
|
|
|
|
|
return
|
|
|
|
|
}
|
|
|
|
|
case 'waitForTurnEnd': {
|
|
|
|
|
const sessionId = getSessionId()
|
|
|
|
|
if (sessionId === undefined) throw new Error('snapshot-harness: waitForTurnEnd before newSession')
|
|
|
|
|
await waitForTurnEnd(sessionId, step.timeoutMs)
|
|
|
|
|
return
|
|
|
|
|
}
|
|
|
|
|
case 'cancel': {
|
|
|
|
|
const sessionId = getSessionId()
|
|
|
|
|
if (sessionId === undefined) throw new Error('snapshot-harness: cancel before newSession')
|
|
|
|
|
await client.cancel({ sessionId })
|
|
|
|
|
return
|
|
|
|
|
}
|
|
|
|
|
case 'setMode': {
|
|
|
|
|
const sessionId = getSessionId()
|
|
|
|
|
if (sessionId === undefined) throw new Error('snapshot-harness: setMode before newSession')
|
|
|
|
|
await client.setSessionMode({ sessionId, modeId: step.modeId })
|
|
|
|
|
return
|
|
|
|
|
}
|
|
|
|
|
case 'setModeExpectError': {
|
|
|
|
|
const sessionId = getSessionId()
|
|
|
|
|
if (sessionId === undefined) throw new Error('snapshot-harness: setModeExpectError before newSession')
|
|
|
|
|
// The bridge rejects an unknown/uncomposed mode id with invalidParams;
|
|
|
|
|
// that rejection IS the expected wire behavior — swallow it so the run
|
|
|
|
|
// completes and the error frame is captured in the transcript.
|
|
|
|
|
await client.setSessionMode({ sessionId, modeId: step.modeId }).then(
|
|
|
|
|
() => { throw new Error('snapshot-harness: expected session/set_mode to be rejected but it succeeded') },
|
|
|
|
|
() => { /* expected: the bridge rejected the mode id */ },
|
|
|
|
|
)
|
|
|
|
|
return
|
|
|
|
|
}
|
|
|
|
|
case 'setConfigOption': {
|
|
|
|
|
const sessionId = getSessionId()
|
|
|
|
|
if (sessionId === undefined) throw new Error('snapshot-harness: setConfigOption before newSession')
|
|
|
|
|
@@ -422,6 +512,51 @@ async function runStep(
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Wait until the raw JSONL backend exposes one complete closing turn boundary.
|
|
|
|
|
* The ACP cancel notification settles its prompt before the agent necessarily
|
|
|
|
|
* reaches quiescence, so cancellation snapshots use this external boundary to
|
|
|
|
|
* keep subprocess disposal from changing an `aborted` turn into `disposed`.
|
|
|
|
|
*/
|
|
|
|
|
async function waitForPersistedTurnEnd(
|
|
|
|
|
root: string,
|
|
|
|
|
sessionId: string,
|
|
|
|
|
timeoutMs = DEFAULT_WAIT_TIMEOUT_MS,
|
|
|
|
|
): Promise<void> {
|
|
|
|
|
const deadline = Date.now() + timeoutMs
|
|
|
|
|
while (true) {
|
|
|
|
|
const log = (await harvestSessionLogs(root)).find(candidate => candidate.id === sessionId)
|
|
|
|
|
if (log !== undefined && latestTurnIsClosed(log.content)) return
|
|
|
|
|
if (Date.now() >= deadline) {
|
|
|
|
|
throw new Error(`snapshot-harness: session "${sessionId}" did not persist turn/end within ${timeoutMs}ms`)
|
|
|
|
|
}
|
|
|
|
|
await delay(WAIT_POLL_INTERVAL_MS)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/** Wait for a cwd-relative marker proving an external action reached readiness. */
|
|
|
|
|
async function waitForWorkspaceFile(
|
|
|
|
|
cwd: string,
|
|
|
|
|
path: string,
|
|
|
|
|
timeoutMs = DEFAULT_WAIT_TIMEOUT_MS,
|
|
|
|
|
): Promise<void> {
|
|
|
|
|
const target = join(cwd, path)
|
|
|
|
|
const deadline = Date.now() + timeoutMs
|
|
|
|
|
while (!existsSync(target)) {
|
|
|
|
|
if (Date.now() >= deadline) {
|
|
|
|
|
throw new Error(`snapshot-harness: workspace file "${path}" did not appear within ${timeoutMs}ms`)
|
|
|
|
|
}
|
|
|
|
|
await delay(WAIT_POLL_INTERVAL_MS)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/** Return whether the last complete raw-JSONL turn boundary closes its turn. */
|
|
|
|
|
function latestTurnIsClosed(content: string): boolean {
|
|
|
|
|
const complete = content.slice(0, content.lastIndexOf('\n') + 1)
|
|
|
|
|
return complete.lastIndexOf('\n{"type":"turn/end",')
|
|
|
|
|
> complete.lastIndexOf('\n{"type":"turn/start",')
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Harvest EVERY persisted `.jsonl` session log under a sessions root, parse each
|
|
|
|
|
* header line, and return them ordered primary-first: the top-level session (no
|
|
|
|
|
|