feat(acp-snapshot): scripted permission answers in the harness client

InputScript gains an optional permissionAnswers queue, consumed FIFO by
the harness's requestPermission handler. Each entry selects by option
KIND (allow_once, reject_once, …): option ids are agent-issued randoms a
committed script cannot know, while kinds are the ACP-stable vocabulary,
so the client maps kind → the offered optionId at answer time. An absent
or exhausted queue answers cancelled — existing scenarios and goldens
are untouched — and a scripted kind the request never offered throws,
surfacing as a JSON-RPC error on the permission request: the scenario
scripted an impossible click.

This is what lets an approval-flow suite (the sandbox composition) drive
allow/reject round-trips deterministically from input.json, per the
shared-acp-snapshot RFC.
This commit is contained in:
kingwl
2026-07-08 02:07:14 +08:00
parent 610c8e3709
commit b0144eaccd
4 changed files with 75 additions and 3 deletions

View File

@@ -33,4 +33,4 @@ defineAcpSnapshotSuite({
The example also ships a `cordis.snapshot.yml` replay overlay next to its `cordis.yml` (the bin swaps them under `DSH_SNAPSHOT=replay` — [single-source replay config RFC](../../../docs/rfc/implemented/testing/2026-07-04-single-source-acp-replay-config.md)); replay fixtures are served by [`dsh-llm-replay`](../llm-replay/README.md), which this package points at via the `DSH_SNAPSHOT_*` env vars it sets on the child. Fixture roles, record/replay semantics, and scenario-table fields are documented on `Scenario` and in the [snapshot RFC](../../../docs/rfc/implemented/testing/2026-06-19-acp-snapshot-tests.md).
Constraints: `suite.ts` imports vitest, so the package is importable only inside a vitest run (the harness and normalizers have no such dependency but ship from the same entry). ACP-specific by design — the harness speaks the SDK's `ClientSideConnection` and answers `requestPermission` with `cancelled`.
Constraints: `suite.ts` imports vitest, so the package is importable only inside a vitest run (the harness and normalizers have no such dependency but ship from the same entry). ACP-specific by design — the harness speaks the SDK's `ClientSideConnection`. Permission round-trips are scriptable: `InputScript.permissionAnswers` is a FIFO queue of option-kind selections (`allow_once`, `reject_once`, …) the client maps to the agent-issued `optionId` at answer time; an absent or exhausted queue answers `cancelled`, and a kind the request never offered fails loud.

View File

@@ -89,6 +89,23 @@ export type InputStep =
/** A scenario's `input.json`: an ordered list of input steps. */
export interface InputScript {
steps: InputStep[]
/**
* Ordered answers for the agent's `session/request_permission` round-trips,
* consumed FIFO — the Nth request gets the Nth answer. Each answer selects
* by option KIND: option ids are agent-issued randoms a committed script
* cannot know, while kinds are the ACP-stable vocabulary, so the client maps
* kind → the offered `optionId` at answer time. A request beyond the queue
* (or with no queue at all) is answered `cancelled` — the stub behavior a
* scenario without approvals relies on. A scripted kind the request does
* not offer fails loud: the scenario scripted an impossible click.
*/
permissionAnswers?: PermissionAnswer[]
}
/** One scripted answer to a permission request: which offered option kind to select. */
export interface PermissionAnswer {
/** The `PermissionOption.kind` to select (`allow_once`, `reject_always`, …). */
kind: 'allow_once' | 'allow_always' | 'reject_once' | 'reject_always'
}
/** One harvested session log plus the identifying facts off its header line. */
@@ -220,6 +237,9 @@ export async function runScenario(input: InputScript, opts: RunOptions): Promise
const waitForUpdate = (match: (u: SessionNotification['update']) => boolean): Promise<void> =>
new Promise<void>(resolve => updateWaiters.push({ match, resolve }))
// Permission answers are consumed FIFO across the whole run; exhaustion
// falls back to `cancelled` so approval-free scenarios keep the plain stub.
const permissionQueue = [...input.permissionAnswers ?? []]
const makeClient = (_agent: AcpAgent): Client => ({
sessionUpdate(params: SessionNotification): Promise<void> {
for (let i = updateWaiters.length - 1; i >= 0; i--) {
@@ -236,8 +256,20 @@ export async function runScenario(input: InputScript, opts: RunOptions): Promise
}
return Promise.resolve()
},
requestPermission(_params: RequestPermissionRequest): Promise<RequestPermissionResponse> {
return Promise.resolve({ outcome: { outcome: 'cancelled' } })
requestPermission(params: RequestPermissionRequest): Promise<RequestPermissionResponse> {
const answer = permissionQueue.shift()
if (answer === undefined) return Promise.resolve({ outcome: { outcome: 'cancelled' } })
const option = params.options.find(o => o.kind === answer.kind)
if (option === undefined) {
// The scenario scripted a click the agent never offered — a scenario
// bug. Throwing here surfaces as a JSON-RPC error on the permission
// request, which the transcript (and usually the run) fails on.
throw new Error(
`snapshot-harness: scripted permission answer ${answer.kind} not among `
+ `the offered options [${params.options.map(o => o.kind).join(', ')}]`,
)
}
return Promise.resolve({ outcome: { outcome: 'selected', optionId: option.optionId } })
},
})
const client = new ClientSideConnection(makeClient, stream)

View File

@@ -21,6 +21,7 @@ export {
type HarvestedLog,
type InputScript,
type InputStep,
type PermissionAnswer,
type RunOptions,
type RunResult,
} from './harness.ts'

View File

@@ -230,4 +230,43 @@ describe('runScenario', () => {
)
expect(result.sessionLogs).toHaveLength(0)
})
it('answers permission requests from the scripted queue by option kind, falling back to cancelled', { timeout: 20_000 }, async () => {
const { fixtureFile } = await scenario({ permissionProbe: true })
// Two prompts → two permission round-trips; one scripted answer, so the
// second request exercises the exhausted-queue fallback.
const result = await runScenario(
{
steps: [...boot, { op: 'prompt', text: 'one' }, { op: 'prompt', text: 'two' }],
permissionAnswers: [{ kind: 'allow_once' }],
},
{ agent: AGENT, mode: 'replay', fixtureFile },
)
const first = result.rawStdout.indexOf('permission:{\\"outcome\\":\\"selected\\",\\"optionId\\":\\"opt-allow\\"}')
const second = result.rawStdout.indexOf('permission:{\\"outcome\\":\\"cancelled\\"}')
expect(first).toBeGreaterThanOrEqual(0)
expect(second).toBeGreaterThan(first)
})
it('selects a non-first offered option by kind', { timeout: 20_000 }, async () => {
const { fixtureFile } = await scenario({ permissionProbe: true })
const result = await runScenario(
{ steps: [...boot, { op: 'prompt', text: 'deny it' }], permissionAnswers: [{ kind: 'reject_once' }] },
{ agent: AGENT, mode: 'replay', fixtureFile },
)
expect(result.rawStdout).toContain('permission:{\\"outcome\\":\\"selected\\",\\"optionId\\":\\"opt-reject\\"}')
})
it('fails loud on a scripted permission kind the agent never offered', { timeout: 20_000 }, async () => {
const { fixtureFile } = await scenario({ permissionProbe: true })
// The fake bin offers allow_once/reject_once; scripting allow_always is a
// scenario bug. The client handler throws, the SDK surfaces it as a
// JSON-RPC error on the permission request, and the fake bin echoes the
// missing outcome as null.
const result = await runScenario(
{ steps: [...boot, { op: 'prompt', text: 'impossible click' }], permissionAnswers: [{ kind: 'allow_always' }] },
{ agent: AGENT, mode: 'replay', fixtureFile },
)
expect(result.rawStdout).toContain('permission:null')
})
})