diff --git a/packages/support/acp-snapshot/README.md b/packages/support/acp-snapshot/README.md index 324cfa9f1d..a47a51d053 100644 --- a/packages/support/acp-snapshot/README.md +++ b/packages/support/acp-snapshot/README.md @@ -33,4 +33,4 @@ defineAcpSnapshotSuite({ The example also ships a `cordis.snapshot.yml` replay overlay next to its `cordis.yml` (the bin swaps them under `DSH_SNAPSHOT=replay` — [single-source replay config RFC](../../../docs/rfc/implemented/testing/2026-07-04-single-source-acp-replay-config.md)); replay fixtures are served by [`dsh-llm-replay`](../llm-replay/README.md), which this package points at via the `DSH_SNAPSHOT_*` env vars it sets on the child. Fixture roles, record/replay semantics, and scenario-table fields are documented on `Scenario` and in the [snapshot RFC](../../../docs/rfc/implemented/testing/2026-06-19-acp-snapshot-tests.md). -Constraints: `suite.ts` imports vitest, so the package is importable only inside a vitest run (the harness and normalizers have no such dependency but ship from the same entry). ACP-specific by design — the harness speaks the SDK's `ClientSideConnection` and answers `requestPermission` with `cancelled`. +Constraints: `suite.ts` imports vitest, so the package is importable only inside a vitest run (the harness and normalizers have no such dependency but ship from the same entry). ACP-specific by design — the harness speaks the SDK's `ClientSideConnection`. Permission round-trips are scriptable: `InputScript.permissionAnswers` is a FIFO queue of option-kind selections (`allow_once`, `reject_once`, …) the client maps to the agent-issued `optionId` at answer time; an absent or exhausted queue answers `cancelled`, and a kind the request never offered fails loud. diff --git a/packages/support/acp-snapshot/src/harness.ts b/packages/support/acp-snapshot/src/harness.ts index 652e60a469..da640f2dce 100644 --- a/packages/support/acp-snapshot/src/harness.ts +++ b/packages/support/acp-snapshot/src/harness.ts @@ -89,6 +89,23 @@ export type InputStep = /** A scenario's `input.json`: an ordered list of input steps. */ export interface InputScript { steps: InputStep[] + /** + * Ordered answers for the agent's `session/request_permission` round-trips, + * consumed FIFO — the Nth request gets the Nth answer. Each answer selects + * by option KIND: option ids are agent-issued randoms a committed script + * cannot know, while kinds are the ACP-stable vocabulary, so the client maps + * kind → the offered `optionId` at answer time. A request beyond the queue + * (or with no queue at all) is answered `cancelled` — the stub behavior a + * scenario without approvals relies on. A scripted kind the request does + * not offer fails loud: the scenario scripted an impossible click. + */ + permissionAnswers?: PermissionAnswer[] +} + +/** One scripted answer to a permission request: which offered option kind to select. */ +export interface PermissionAnswer { + /** The `PermissionOption.kind` to select (`allow_once`, `reject_always`, …). */ + kind: 'allow_once' | 'allow_always' | 'reject_once' | 'reject_always' } /** One harvested session log plus the identifying facts off its header line. */ @@ -220,6 +237,9 @@ export async function runScenario(input: InputScript, opts: RunOptions): Promise const waitForUpdate = (match: (u: SessionNotification['update']) => boolean): Promise => new Promise(resolve => updateWaiters.push({ match, resolve })) + // Permission answers are consumed FIFO across the whole run; exhaustion + // falls back to `cancelled` so approval-free scenarios keep the plain stub. + const permissionQueue = [...input.permissionAnswers ?? []] const makeClient = (_agent: AcpAgent): Client => ({ sessionUpdate(params: SessionNotification): Promise { for (let i = updateWaiters.length - 1; i >= 0; i--) { @@ -236,8 +256,20 @@ export async function runScenario(input: InputScript, opts: RunOptions): Promise } return Promise.resolve() }, - requestPermission(_params: RequestPermissionRequest): Promise { - return Promise.resolve({ outcome: { outcome: 'cancelled' } }) + requestPermission(params: RequestPermissionRequest): Promise { + const answer = permissionQueue.shift() + if (answer === undefined) return Promise.resolve({ outcome: { outcome: 'cancelled' } }) + const option = params.options.find(o => o.kind === answer.kind) + if (option === undefined) { + // The scenario scripted a click the agent never offered — a scenario + // bug. Throwing here surfaces as a JSON-RPC error on the permission + // request, which the transcript (and usually the run) fails on. + throw new Error( + `snapshot-harness: scripted permission answer ${answer.kind} not among ` + + `the offered options [${params.options.map(o => o.kind).join(', ')}]`, + ) + } + return Promise.resolve({ outcome: { outcome: 'selected', optionId: option.optionId } }) }, }) const client = new ClientSideConnection(makeClient, stream) diff --git a/packages/support/acp-snapshot/src/index.ts b/packages/support/acp-snapshot/src/index.ts index a0d3380086..bbe74030f2 100644 --- a/packages/support/acp-snapshot/src/index.ts +++ b/packages/support/acp-snapshot/src/index.ts @@ -21,6 +21,7 @@ export { type HarvestedLog, type InputScript, type InputStep, + type PermissionAnswer, type RunOptions, type RunResult, } from './harness.ts' diff --git a/packages/support/acp-snapshot/tests/harness.spec.ts b/packages/support/acp-snapshot/tests/harness.spec.ts index 3884e36095..d674e01818 100644 --- a/packages/support/acp-snapshot/tests/harness.spec.ts +++ b/packages/support/acp-snapshot/tests/harness.spec.ts @@ -230,4 +230,43 @@ describe('runScenario', () => { ) expect(result.sessionLogs).toHaveLength(0) }) + + it('answers permission requests from the scripted queue by option kind, falling back to cancelled', { timeout: 20_000 }, async () => { + const { fixtureFile } = await scenario({ permissionProbe: true }) + // Two prompts → two permission round-trips; one scripted answer, so the + // second request exercises the exhausted-queue fallback. + const result = await runScenario( + { + steps: [...boot, { op: 'prompt', text: 'one' }, { op: 'prompt', text: 'two' }], + permissionAnswers: [{ kind: 'allow_once' }], + }, + { agent: AGENT, mode: 'replay', fixtureFile }, + ) + const first = result.rawStdout.indexOf('permission:{\\"outcome\\":\\"selected\\",\\"optionId\\":\\"opt-allow\\"}') + const second = result.rawStdout.indexOf('permission:{\\"outcome\\":\\"cancelled\\"}') + expect(first).toBeGreaterThanOrEqual(0) + expect(second).toBeGreaterThan(first) + }) + + it('selects a non-first offered option by kind', { timeout: 20_000 }, async () => { + const { fixtureFile } = await scenario({ permissionProbe: true }) + const result = await runScenario( + { steps: [...boot, { op: 'prompt', text: 'deny it' }], permissionAnswers: [{ kind: 'reject_once' }] }, + { agent: AGENT, mode: 'replay', fixtureFile }, + ) + expect(result.rawStdout).toContain('permission:{\\"outcome\\":\\"selected\\",\\"optionId\\":\\"opt-reject\\"}') + }) + + it('fails loud on a scripted permission kind the agent never offered', { timeout: 20_000 }, async () => { + const { fixtureFile } = await scenario({ permissionProbe: true }) + // The fake bin offers allow_once/reject_once; scripting allow_always is a + // scenario bug. The client handler throws, the SDK surfaces it as a + // JSON-RPC error on the permission request, and the fake bin echoes the + // missing outcome as null. + const result = await runScenario( + { steps: [...boot, { op: 'prompt', text: 'impossible click' }], permissionAnswers: [{ kind: 'allow_always' }] }, + { agent: AGENT, mode: 'replay', fixtureFile }, + ) + expect(result.rawStdout).toContain('permission:null') + }) })