From 7677d49c9346663c52977f94546ea40f6b07bc7f Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Tue, 21 Jul 2026 01:16:29 +0800 Subject: [PATCH] test(snapshot): cover session-scoped sandbox roots --- .../implemented/feature/2026-07-06-sandbox.md | 2 +- .../testing/2026-06-19-acp-snapshot-tests.md | 4 +-- examples/acp-agent/README.md | 2 +- .../session-sandbox-root.cordis.snapshot.yml | 36 +++++++++++++++++++ .../acp-agent/session-sandbox-root.cordis.yml | 14 ++++++++ examples/acp-agent/tests/acp.snapshot.ts | 15 ++++++++ .../snapshots/session-sandbox-root/input.json | 8 +++++ .../session-sandbox-root/replay.override.json | 22 ++++++++++++ .../session-sandbox-root/session.jsonl | 26 ++++++++++++++ .../stdout.expected.jsonl | 8 +++++ packages/support/acp-snapshot/README.md | 6 ++-- packages/support/acp-snapshot/src/harness.ts | 22 ++++++++---- .../support/acp-snapshot/src/normalize.ts | 4 +-- packages/support/acp-snapshot/src/suite.ts | 9 ++++- .../acp-snapshot/tests/harness.spec.ts | 18 +++++++++- .../support/acp-snapshot/tests/suite.spec.ts | 9 ++++- 16 files changed, 186 insertions(+), 19 deletions(-) create mode 100644 examples/acp-agent/session-sandbox-root.cordis.snapshot.yml create mode 100644 examples/acp-agent/session-sandbox-root.cordis.yml create mode 100644 examples/acp-agent/tests/snapshots/session-sandbox-root/input.json create mode 100644 examples/acp-agent/tests/snapshots/session-sandbox-root/replay.override.json create mode 100644 examples/acp-agent/tests/snapshots/session-sandbox-root/session.jsonl create mode 100644 examples/acp-agent/tests/snapshots/session-sandbox-root/stdout.expected.jsonl diff --git a/.agents/notes/implemented/feature/2026-07-06-sandbox.md b/.agents/notes/implemented/feature/2026-07-06-sandbox.md index 1c46f357e8..d5d0c6d5aa 100644 --- a/.agents/notes/implemented/feature/2026-07-06-sandbox.md +++ b/.agents/notes/implemented/feature/2026-07-06-sandbox.md @@ -118,7 +118,7 @@ fs/web/todo execute in-process, so their sandbox semantics are policy at their s - **Unit:** pin platform selection and profiles, fail-closed runner classification, per-call mode/root resolution, per-process facts, escalation validation and outcomes, permission preset folding and write-through, narrator coalescing, ACP advertisement and validation, and turn-enclosed config writes. - **Keyless real-runner:** exercise bwrap, Landlock, and Seatbelt against real filesystem effects at provider and bash-consumer layers; one real Cordis context concurrently drives two project sessions through shipped bash and fs tools, proving own-root success and sibling-root denial. Packed-install coverage proves the registry launcher remains executable. The real ACP composition pins permission switching and rejects unknown presets. CI rejects a silent all-skip. - **With-key:** drive a real model, runner, bridge answerer, and disk effect through granted and rejected escalation; unavailable credentials or runners self-skip. -- **Snapshot:** pin the permission config-option wire, preset and knob events, prompt deltas and notices, and both scripted approval branches. Snapshot mode starts unconfined so unrelated fixtures remain platform-independent; policy scenarios switch explicitly. Real denial stderr stays on platform tests because its dialect is runner-specific. +- **Snapshot:** pin the permission config-option wire, preset and knob events, prompt deltas and notices, and both scripted approval branches. A real ACP example scenario places its session under the user home while the deployment fallback points at `/tmp`, then pins a successful workspace-write mutation; this distinguishes session-root resolution from the process fallback without depending on runner-specific denial text. Other snapshots start unconfined so unrelated fixtures remain platform-independent, and policy scenarios switch explicitly. ## Deferred phases diff --git a/.agents/notes/implemented/testing/2026-06-19-acp-snapshot-tests.md b/.agents/notes/implemented/testing/2026-06-19-acp-snapshot-tests.md index f400701fcd..1e20d26464 100644 --- a/.agents/notes/implemented/testing/2026-06-19-acp-snapshot-tests.md +++ b/.agents/notes/implemented/testing/2026-06-19-acp-snapshot-tests.md @@ -57,7 +57,7 @@ Normalization replaces session, cwd, protocol-id, timestamp, path, and process v ### Isolation: normalization now, sandbox later -Tool determinism comes from a temporary cwd, scrubbed environment, fresh non-login shell, constrained commands, and normalization. Concurrent replay runs own separate cwd, persistence, and fixed-length scenario-keyed spill roots, so one scenario's teardown cannot delete another's in-flight full-output recovery while real-path preview budgets remain stable. This tier does not claim OS confinement. A sandboxed executor can replace the local backend through the existing [capability seam](../architecture/2026-06-13-capability-seams.md) if a stronger tier is needed. +Tool determinism comes from a generated cwd, scrubbed environment, fresh non-login shell, constrained commands, and normalization. The cwd defaults to the platform temp directory; a scenario can instead supply its parent when temp is an always-writable policy root and the behavior needs an independent project location. Concurrent replay runs own separate cwd, persistence, and fixed-length scenario-keyed spill roots, so one scenario's teardown cannot delete another's in-flight full-output recovery while real-path preview budgets remain stable. This tier does not claim OS confinement. A sandboxed executor can replace the local backend through the existing [capability seam](../architecture/2026-06-13-capability-seams.md) if a stronger tier is needed. ### The replay plugin is its own package @@ -75,6 +75,6 @@ Tool determinism comes from a temporary cwd, scrubbed environment, fresh non-log ## Consequences -The new tier adds reviewed per-scenario input, session, stdout, optional override, and optional workspace fixtures. Workspace seeds are copied into the temporary cwd for both record and replay. In return the tier provides deterministic keyless transcript coverage through the real Loader and tool composition. The subprocess, input, workspace, normalization, and replay harness can support examples beyond ACP. +The new tier adds reviewed per-scenario input, session, stdout, optional override, and optional workspace fixtures. Workspace seeds are copied into the generated cwd for both record and replay. In return the tier provides deterministic keyless transcript coverage through the real Loader and tool composition. The subprocess, input, workspace, normalization, and replay harness can support examples beyond ACP. This Agent Note relates to but does not supersede the [proposed determinism Agent Note](../../proposed/testing/2026-06-11-deterministic-and-stress-testing.md): that proposal's "universal replay fixture" re-derives session *message history* after every test (an internal-consistency invariant), whereas snapshot tests pin the *external protocol output*. They are complementary — one guards the event-sourcing invariant, the other guards the editor-facing contract. diff --git a/examples/acp-agent/README.md b/examples/acp-agent/README.md index b2fe82dd7c..af4a6344f0 100644 --- a/examples/acp-agent/README.md +++ b/examples/acp-agent/README.md @@ -43,7 +43,7 @@ The default tree composes [`@deepseek-ai/dsh-sandbox-local`](../../packages/sand - **Every approval is one-shot**: the choices are `Allow once` and `Reject`; a dismissal, rejection, missing editor, or unavailable runner fails closed. - **The boundary spans bash and the filesystem tools per session**: bash confines through the OS runner and the `read`/`write`/`edit` tools through an in-process path fence ([`dsh-fs-sandbox`](../../packages/fs/fs-sandbox/)); both receive the calling session's cwd as `workspaceRoot`. -`tests/escalation.e2e.ts` boots this default tree keyless, drives the permission select, and—with a key and usable runner—proves both approval outcomes against the filesystem. The agent-spine e2e independently boots one context with two project sessions and world-verifies concurrent own-root success plus sibling-root denial through both shipped tool families. Most snapshots use the ACP tree and start at `danger-full-access` so bash fixtures remain runner-independent; scenarios that call `read`, `write`, or `edit` use the full-access fs overlay and a separate request-header pin. The permission-switching and escalation inputs select `workspace-write` before exercising the bash policy path. No fixture pins real runner denial text because its dialect is platform-specific. +`tests/escalation.e2e.ts` boots this default tree keyless, drives the permission select, and—with a key and usable runner—proves both approval outcomes against the filesystem. The agent-spine e2e independently boots one context with two project sessions and world-verifies concurrent own-root success plus sibling-root denial through both shipped tool families. The keyless `session-sandbox-root` ACP snapshot places its generated project under the user home while an overlay points the deployment fallback at `/tmp`; its successful `workspace-write` call proves the assembled app used the session cwd. Most snapshots start at `danger-full-access` so bash fixtures remain runner-independent. No fixture pins real runner denial text because its dialect is platform-specific. ## MVP limitations diff --git a/examples/acp-agent/session-sandbox-root.cordis.snapshot.yml b/examples/acp-agent/session-sandbox-root.cordis.snapshot.yml new file mode 100644 index 0000000000..565833d528 --- /dev/null +++ b/examples/acp-agent/session-sandbox-root.cordis.snapshot.yml @@ -0,0 +1,36 @@ +# Keyless replay counterpart of session-sandbox-root.cordis.yml. Patches do not +# compose across nested includes, so the replay swap and the deliberately +# distinct sandbox fallback are applied together to the live tree. +- id: base + name: '@cordisjs/plugin-include' + config: + path: ./cordis.yml + patches: + - id: llm-deepseek + name: '@deepseek-ai/dsh-llm-deepseek' + disabled: true + - id: sandbox + name: '@deepseek-ai/dsh-sandbox-local' + config: + runnerCommand: + - bash + - -c + - while [ "$1" != "--" ]; do shift; done; shift; exec "$@" + - passthrough-runner + runnerFailureSignatures: + - 'passthrough-runner: profile rejected' + - id: sandbox-policy + name: '@deepseek-ai/dsh-sandbox-policy' + config: + mode: danger-full-access + workspaceRoot: /tmp + - insert: + - id: llm-replay + name: '@deepseek-ai/dsh-llm-replay' + config: + providers: + - id: deepseek + name: DeepSeek + models: + - id: deepseek-v4-flash + - id: deepseek-v4-pro diff --git a/examples/acp-agent/session-sandbox-root.cordis.yml b/examples/acp-agent/session-sandbox-root.cordis.yml new file mode 100644 index 0000000000..f27732fd68 --- /dev/null +++ b/examples/acp-agent/session-sandbox-root.cordis.yml @@ -0,0 +1,14 @@ +# Session-root sandbox snapshot overlay. The generated ACP session cwd lives +# under the user's home, while this deployment fallback deliberately points at +# /tmp. A workspace-write mutation can therefore succeed only when the calling +# session's cwd replaces the process-level fallback root. +- id: base + name: '@cordisjs/plugin-include' + config: + path: ./cordis.yml + patches: + - id: sandbox-policy + name: '@deepseek-ai/dsh-sandbox-policy' + config: + mode: !!js "process.env.DSH_PERMISSION_MODE ?? (process.env.DSH_SNAPSHOT === undefined ? 'workspace-write' : 'danger-full-access')" + workspaceRoot: /tmp diff --git a/examples/acp-agent/tests/acp.snapshot.ts b/examples/acp-agent/tests/acp.snapshot.ts index aa3cba1f56..3460c3c9af 100644 --- a/examples/acp-agent/tests/acp.snapshot.ts +++ b/examples/acp-agent/tests/acp.snapshot.ts @@ -1,5 +1,6 @@ import { fileURLToPath } from 'node:url' import { dirname, join } from 'node:path' +import { homedir } from 'node:os' import { defineAcpSnapshotSuite, type Scenario, type SnapshotSuiteOptions } from '@deepseek-ai/dsh-acp-snapshot' /** @@ -31,6 +32,7 @@ const WORKSPACE_CONTEXT_CONFIG = fileURLToPath(new URL('../workspace-context.cor const ADVANCED_CONFIG = fileURLToPath(new URL('../advanced.cordis.yml', import.meta.url)) const FS_CONFIG = fileURLToPath(new URL('../fs.cordis.yml', import.meta.url)) const DEPTH_TWO_CONFIG = fileURLToPath(new URL('../depth-two.cordis.yml', import.meta.url)) +const SESSION_SANDBOX_ROOT_CONFIG = fileURLToPath(new URL('../session-sandbox-root.cordis.yml', import.meta.url)) function snapshotModeFromEnv(value: string | undefined): SnapshotSuiteOptions['mode'] { switch (value) { @@ -187,6 +189,19 @@ const SCENARIOS: Scenario[] = [ { name: 'escalation-approved', hasModelTurn: true, recorded: true, headerClass: 'sandbox' }, { name: 'escalation-rejected', hasModelTurn: true, recorded: true, headerClass: 'sandbox' }, { name: 'fs-escalation-approved', hasModelTurn: true, recorded: true, headerClass: 'sandbox' }, + // Unlike ordinary snapshots, this session cwd is outside the platform temp + // roots that workspace-write always grants. The overlay points the + // deployment fallback at /tmp, so a successful relative write proves the + // assembled app replaced that process-level fallback with SessionHeader.cwd. + { + name: 'session-sandbox-root', + hasModelTurn: true, + recorded: false, + overridden: true, + headerClass: 'sandbox', + configPath: SESSION_SANDBOX_ROOT_CONFIG, + workspaceParent: homedir(), + }, ] defineAcpSnapshotSuite({ diff --git a/examples/acp-agent/tests/snapshots/session-sandbox-root/input.json b/examples/acp-agent/tests/snapshots/session-sandbox-root/input.json new file mode 100644 index 0000000000..9cef40f51d --- /dev/null +++ b/examples/acp-agent/tests/snapshots/session-sandbox-root/input.json @@ -0,0 +1,8 @@ +{ + "steps": [ + { "op": "initialize" }, + { "op": "newSession" }, + { "op": "setConfigOption", "configId": "permission", "value": "workspace-write" }, + { "op": "prompt", "text": "Use the write tool (NOT bash) to create session-root.txt in the current directory containing exactly: session root. Then reply with exactly the single word DONE." } + ] +} diff --git a/examples/acp-agent/tests/snapshots/session-sandbox-root/replay.override.json b/examples/acp-agent/tests/snapshots/session-sandbox-root/replay.override.json new file mode 100644 index 0000000000..511613b441 --- /dev/null +++ b/examples/acp-agent/tests/snapshots/session-sandbox-root/replay.override.json @@ -0,0 +1,22 @@ +[ + { + "kind": "chunks", + "chunks": [ + { "type": "block-start", "index": 0, "blockType": "tool-call" }, + { "type": "tool-call-delta", "index": 0, "id": "call_session_root", "name": "write", "argumentsDelta": "{\"file_path\":\"session-root.txt\",\"content\":\"session root\"}" }, + { "type": "block-end", "index": 0, "block": { "type": "tool-call", "id": "call_session_root", "name": "write", "arguments": "{\"file_path\":\"session-root.txt\",\"content\":\"session root\"}" } }, + { "type": "usage", "usage": { "inputTokens": 10, "outputTokens": 5 } }, + { "type": "finish", "reason": { "kind": "tool-calls" } } + ] + }, + { + "kind": "chunks", + "chunks": [ + { "type": "block-start", "index": 0, "blockType": "text" }, + { "type": "text-delta", "index": 0, "text": "DONE" }, + { "type": "block-end", "index": 0, "block": { "type": "text", "text": "DONE" } }, + { "type": "usage", "usage": { "inputTokens": 10, "outputTokens": 2 } }, + { "type": "finish", "reason": { "kind": "stop" } } + ] + } +] diff --git a/examples/acp-agent/tests/snapshots/session-sandbox-root/session.jsonl b/examples/acp-agent/tests/snapshots/session-sandbox-root/session.jsonl new file mode 100644 index 0000000000..a333cb62fc --- /dev/null +++ b/examples/acp-agent/tests/snapshots/session-sandbox-root/session.jsonl @@ -0,0 +1,26 @@ +{"type":"session","version":0,"id":"00000000-0000-0000-0000-000000000000","createdAt":0,"cwd":"/Users/cty/acp-snap-cwd-MABAjO","delegationDepth":0} +{"type":"turn/start","seq":0,"time":1784567324138,"data":{"turn":1,"trigger":{"kind":"message","source":{"kind":"user"}}}} +{"type":"permission/preset","seq":1,"time":1784567324138,"data":{"preset":"workspace-write"}} +{"type":"sandbox/mode","seq":2,"time":1784567324138,"data":{"mode":"workspace-write"}} +{"type":"approval/policy","seq":3,"time":1784567324138,"data":{"policy":"ask"}} +{"type":"user/message","seq":4,"time":1784567324138,"data":{"content":[{"type":"text","text":"Use the write tool (NOT bash) to create session-root.txt in the current directory containing exactly: session root. Then reply with exactly the single word DONE."}],"source":{"kind":"user"}},"surfaceOp":"append"} +{"type":"step/start","seq":5,"time":1784567324142,"data":{"turn":1,"step":1}} +{"type":"request/header","seq":6,"time":1784567324142,"data":{"header":{"config":{"provider":"deepseek","model":"deepseek-v4-flash"},"system":"{{system}}","tools":"{{tools}}"},"reason":"initial"}} +{"type":"assistant/chunk","seq":7,"time":1784567324143,"data":{"turn":1,"step":1,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}}} +{"type":"assistant/chunk","seq":8,"time":1784567324143,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":0,"id":"call_session_root","name":"write","argumentsDelta":"{\"file_path\":\"session-root.txt\",\"content\":\"session root\"}"}}} +{"type":"assistant/chunk","seq":9,"time":1784567324143,"data":{"turn":1,"step":1,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"call_session_root","name":"write","arguments":"{\"file_path\":\"session-root.txt\",\"content\":\"session root\"}"}}}} +{"type":"assistant/chunk","seq":10,"time":1784567324143,"data":{"turn":1,"step":1,"chunk":{"type":"usage","usage":{"inputTokens":10,"outputTokens":5}}}} +{"type":"assistant/chunk","seq":11,"time":1784567324143,"data":{"turn":1,"step":1,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}} +{"type":"assistant/message","seq":12,"time":1784567324144,"data":{"turn":1,"step":1,"content":[{"type":"tool-call","id":"call_session_root","name":"write","arguments":"{\"file_path\":\"session-root.txt\",\"content\":\"session root\"}"}],"provenance":{"provider":"deepseek","model":"deepseek-v4-flash"},"usage":{"inputTokens":10,"outputTokens":5}},"sourceEventSeqs":[7,8,9,10,11],"surfaceOp":"append"} +{"type":"tool/call","seq":13,"time":1784567324145,"data":{"turn":1,"step":1,"callId":"call_session_root","name":"write","arguments":"{\"file_path\":\"session-root.txt\",\"content\":\"session root\"}"}} +{"type":"tool/result","seq":14,"time":1784567324155,"data":{"turn":1,"step":1,"callId":"call_session_root","content":[{"type":"text","text":"/Users/cty/acp-snap-cwd-MABAjO/session-root.txt\nfile\n\nCreated file\n"}],"isError":false},"sourceEventSeqs":[13],"surfaceOp":"append"} +{"type":"step/end","seq":15,"time":1784567324157,"data":{"turn":1,"step":1}} +{"type":"step/start","seq":16,"time":1784567324157,"data":{"turn":1,"step":2}} +{"type":"assistant/chunk","seq":17,"time":1784567324157,"data":{"turn":1,"step":2,"chunk":{"type":"block-start","index":0,"blockType":"text"}}} +{"type":"assistant/chunk","seq":18,"time":1784567324157,"data":{"turn":1,"step":2,"chunk":{"type":"text-delta","index":0,"text":"DONE"}}} +{"type":"assistant/chunk","seq":19,"time":1784567324158,"data":{"turn":1,"step":2,"chunk":{"type":"block-end","index":0,"block":{"type":"text","text":"DONE"}}}} +{"type":"assistant/chunk","seq":20,"time":1784567324158,"data":{"turn":1,"step":2,"chunk":{"type":"usage","usage":{"inputTokens":10,"outputTokens":2}}}} +{"type":"assistant/chunk","seq":21,"time":1784567324158,"data":{"turn":1,"step":2,"chunk":{"type":"finish","reason":{"kind":"stop"}}}} +{"type":"assistant/message","seq":22,"time":1784567324158,"data":{"turn":1,"step":2,"content":[{"type":"text","text":"DONE"}],"provenance":{"provider":"deepseek","model":"deepseek-v4-flash"},"usage":{"inputTokens":10,"outputTokens":2}},"sourceEventSeqs":[17,18,19,20,21],"surfaceOp":"append"} +{"type":"step/end","seq":23,"time":1784567324158,"data":{"turn":1,"step":2}} +{"type":"turn/end","seq":24,"time":1784567324158,"data":{"turn":1,"reason":{"kind":"completed"}}} diff --git a/examples/acp-agent/tests/snapshots/session-sandbox-root/stdout.expected.jsonl b/examples/acp-agent/tests/snapshots/session-sandbox-root/stdout.expected.jsonl new file mode 100644 index 0000000000..bc05df4f5b --- /dev/null +++ b/examples/acp-agent/tests/snapshots/session-sandbox-root/stdout.expected.jsonl @@ -0,0 +1,8 @@ +{"jsonrpc":"2.0","id":1,"result":{"protocolVersion":1,"agentInfo":{"name":"deepseek-harness-acp","version":"0.0.1"},"agentCapabilities":{"loadSession":true,"promptCapabilities":{"image":false,"audio":false,"embeddedContext":false}},"authMethods":[]}} +{"jsonrpc":"2.0","id":2,"result":{"sessionId":"{{sessionId}}","configOptions":[{"id":"model","name":"Model","description":"Sets this session's provider and model.","category":"model","type":"select","currentValue":"[\"deepseek\",\"deepseek-v4-flash\"]","options":[{"value":"[\"deepseek\",\"deepseek-v4-flash\"]","name":"deepseek-v4-flash"},{"value":"[\"deepseek\",\"deepseek-v4-pro\"]","name":"deepseek-v4-pro"}]},{"id":"permission","name":"Permissions","description":"The session permission preset: each choice bundles a sandbox mode and an approval policy.","category":"mode","type":"select","currentValue":"danger-full-access","options":[{"value":"workspace-write","name":"workspace-write","description":"Write inside the workspace and permitted temporary directories; wider retries require approval."},{"value":"danger-full-access","name":"danger-full-access","description":"Full file access without approval prompts."}]}]}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"available_commands_update","availableCommands":[{"name":"goal","description":"set or view the goal for a long-running task","input":{"hint":"[|clear|edit |pause|resume]"}}]}}} +{"jsonrpc":"2.0","id":3,"result":{"configOptions":[{"id":"model","name":"Model","description":"Sets this session's provider and model.","category":"model","type":"select","currentValue":"[\"deepseek\",\"deepseek-v4-flash\"]","options":[{"value":"[\"deepseek\",\"deepseek-v4-flash\"]","name":"deepseek-v4-flash"},{"value":"[\"deepseek\",\"deepseek-v4-pro\"]","name":"deepseek-v4-pro"}]},{"id":"permission","name":"Permissions","description":"The session permission preset: each choice bundles a sandbox mode and an approval policy.","category":"mode","type":"select","currentValue":"workspace-write","options":[{"value":"workspace-write","name":"workspace-write","description":"Write inside the workspace and permitted temporary directories; wider retries require approval."},{"value":"danger-full-access","name":"danger-full-access","description":"Full file access without approval prompts."}]}]}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"tool_call","toolCallId":"call_session_root","title":"Write session-root.txt","kind":"edit","status":"in_progress","locations":[{"path":"session-root.txt"}],"content":[{"type":"diff","path":"session-root.txt","oldText":null,"newText":"session root"}]}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"tool_call_update","toolCallId":"call_session_root","status":"completed","content":[{"type":"diff","path":"session-root.txt","oldText":null,"newText":"session root"}],"title":"Write session-root.txt"}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":"DONE"}}}} +{"jsonrpc":"2.0","id":4,"result":{"stopReason":"end_turn"}} diff --git a/packages/support/acp-snapshot/README.md b/packages/support/acp-snapshot/README.md index b109ca3afa..e4fec33956 100644 --- a/packages/support/acp-snapshot/README.md +++ b/packages/support/acp-snapshot/README.md @@ -4,8 +4,8 @@ The ACP snapshot suite kit: the shared machinery behind the keyless snapshot tie Four layers, importable separately: -- **`launchAcpTestAgent` (launcher)** — boots an unbuilt ACP agent from a temp cwd, pins tsx to the repo tsconfig, connects the SDK client over a raw-byte stdout tee, collects session updates and stderr, surfaces asynchronous spawn failures through its startup lifecycle, fails closed on unhandled permission requests, and owns graceful or signalled shutdown. Shutdown waits for process exit, inherited stdio closure, and ACP parser exhaustion before resolving or propagating a child error, so captures are complete and callers can remove owned paths after either outcome. Snapshot and ordinary e2e suites share this process boundary; a test supplies only agent paths, cwd, environment overrides, and any permission policy. -- **`runScenario` (harness)** — boots the real agent bin as a subprocess via tsx (unbuilt, Loader path), drives it over ACP JSON-RPC stdio from a deterministic `input.json` script, tees raw stdout for the expected-output and purity checks, and harvests every persisted session JSONL (parent + subagent children, primary-first) after a graceful stdin-EOF shutdown. Parameterized by `AgentUnderTest` (`binScript`, `configPath`, `tsconfigPath` — absolute paths; the subprocess cwd is a temp dir outside the repo). Startup failures preserve captured agent stderr in the rejected diagnostic. +- **`launchAcpTestAgent` (launcher)** — boots an unbuilt ACP agent from a supplied cwd, pins tsx to the repo tsconfig, connects the SDK client over a raw-byte stdout tee, collects session updates and stderr, surfaces asynchronous spawn failures through its startup lifecycle, fails closed on unhandled permission requests, and owns graceful or signalled shutdown. Shutdown waits for process exit, inherited stdio closure, and ACP parser exhaustion before resolving or propagating a child error, so captures are complete and callers can remove owned paths after either outcome. Snapshot and ordinary e2e suites share this process boundary; a test supplies only agent paths, cwd, environment overrides, and any permission policy. +- **`runScenario` (harness)** — boots the real agent bin as a subprocess via tsx (unbuilt, Loader path), drives it over ACP JSON-RPC stdio from a deterministic `input.json` script, tees raw stdout for the expected-output and purity checks, and harvests every persisted session JSONL (parent + subagent children, primary-first) after a graceful stdin-EOF shutdown. Parameterized by `AgentUnderTest` (`binScript`, `configPath`, `tsconfigPath` — absolute paths) plus an optional parent for its generated session cwd; the default is the platform temp directory. Startup failures preserve captured agent stderr in the rejected diagnostic. - **Normalizers** — pure functions turning the two captured surfaces into stable text: `normalizeStdout` (JSON-RPC ids → first-seen sequence; UUIDs/cwd → tokens; doubles as the stdout-purity check), `normalizeSessionLog` (times zeroed, `seq` kept), `scrubSystemPrompts` (prompt text → `{{system}}`), `scrubToolSchemas` (schema bulk → `{{tools}}`), and `scrubRequestHeaders` (all header bulk → `{{system}}`/`{{tools}}`/`{{messagePrefix}}` outside each pin, structure kept — [pinned-header Agent Note](../../../.agents/notes/implemented/testing/2026-07-06-pin-request-header-content-in-one-scenario.md)). - **`defineAcpSnapshotSuite` (factory)** — registers the whole describe/it tree for a scenario table: per-scenario expected-output and re-persisted-log comparisons, record/refresh fixture write-back, rejection of structured `UNKNOWN_TOOL` results, the per-header-class pin (`system-prompt.expected.md` plus `tool-schemas.expected.json`) with its live uniformity guard, and the fixture guard block (no orphan scenario dirs, required files present, exactly one pin per class, every JSONL prompt/schema-scrubbed, non-pinning fixtures fully header-scrubbed). Each scenario directory's `session.jsonl` plus contiguous `session..jsonl` siblings are the ordered primary/child inventory; the scenario table does not duplicate their count. Must be called at vitest collection time. @@ -36,7 +36,7 @@ defineAcpSnapshotSuite({ }) ``` -A scenario booting a differently-composed tree sets its own `configPath` (an overlay whose basename still ends in `cordis.yml`, so the bin's replay swap finds the sibling `*cordis.snapshot.yml`) and, when that composition changes the request header, its own `headerClass` with its own pinning scenario — the acp-agent example's Code Mode and filesystem scenarios are templates. Each pinning directory stores the normalized full prompt sequence in generated `system-prompt.expected.md` and the corresponding full tool-schema sequence in generated `tool-schemas.expected.json`; `session.jsonl` stores `"system":"{{system}}","tools":"{{tools}}"` while retaining config, reason, and any model-visible prefix. A pin with legitimate mid-run header changes declares `expectedHeaderChanges`, which fixes the length of both sidecar sequences. +A scenario booting a differently-composed tree sets its own `configPath` (an overlay whose basename still ends in `cordis.yml`, so the bin's replay swap finds the sibling `*cordis.snapshot.yml`) and, when that composition changes the request header, its own `headerClass` with its own pinning scenario — the acp-agent example's Code Mode and filesystem scenarios are templates. `workspaceParent` moves the generated cwd outside the platform temp area when temporary-directory grants are themselves under test; the harness still owns and removes only the generated child. Each pinning directory stores the normalized full prompt sequence in generated `system-prompt.expected.md` and the corresponding full tool-schema sequence in generated `tool-schemas.expected.json`; `session.jsonl` stores `"system":"{{system}}","tools":"{{tools}}"` while retaining config, reason, and any model-visible prefix. A pin with legitimate mid-run header changes declares `expectedHeaderChanges`, which fixes the length of both sidecar sequences. The example also ships a `cordis.snapshot.yml` replay overlay next to its `cordis.yml` (the bin swaps them under `DSH_SNAPSHOT=replay` — [single-source replay config Agent Note](../../../.agents/notes/implemented/testing/2026-07-04-single-source-acp-replay-config.md)); replay fixtures are served by [`dsh-llm-replay`](../llm-replay/README.md), which this package points at via the `DSH_SNAPSHOT_*` env vars it sets on the child. `pnpm run test:snapshot:record` calls the live LLM and rewrites the recorded scenarios' model fixtures; `pnpm run test:snapshot:refresh` stays keyless, runs the replay overlay, and rewrites stdout, comparable session-log expected outputs, and each pin's prompt and tool-schema sidecars from the committed model scripts. Fixture roles, record/replay/refresh semantics, and scenario-table fields are documented on `Scenario` and in the [snapshot Agent Note](../../../.agents/notes/implemented/testing/2026-06-19-acp-snapshot-tests.md). diff --git a/packages/support/acp-snapshot/src/harness.ts b/packages/support/acp-snapshot/src/harness.ts index 435d352789..ffa81cb7d9 100644 --- a/packages/support/acp-snapshot/src/harness.ts +++ b/packages/support/acp-snapshot/src/harness.ts @@ -106,7 +106,7 @@ export interface RunResult { stderr: string /** The session id the server issued (undefined if no session was created). */ sessionId?: string - /** The temp cwd the session ran in (the bash workspace). */ + /** The generated cwd the session ran in (the bash workspace). */ cwd: string /** * Every persisted session log harvested after the run, ordered primary-first: @@ -137,11 +137,19 @@ export interface RunOptions { childFiles?: string[] /** * Optional `/workspace/` directory whose contents are copied into - * the temp cwd BEFORE the run — the standard way to seed files the agent + * the generated cwd BEFORE the run — the standard way to seed files the agent * operates on (a file to read, edit, or grep). Absent for scenarios that * start from an empty workspace. */ workspaceDir?: string + /** + * Parent directory for the generated session cwd. Defaults to + * `os.tmpdir()`. A scenario that must distinguish its workspace from the + * sandbox's always-writable temporary roots can place the generated child + * under `os.homedir()` instead. The harness removes only that generated + * child, never the supplied parent. + */ + workspaceParent?: string /** * Alternate LIVE config path for the boot (absolute), overriding * {@link AgentUnderTest.configPath} for this run. A scenario needing a @@ -162,15 +170,15 @@ function scenarioSpillRoot(fixtureFile: string): string { /** * Run a scenario end-to-end against a freshly-spawned subprocess. Owns the - * child and its temp dirs; always tears them down. Returns the captured stdout + * child and its generated dirs; always tears them down. Returns the captured stdout * and (record mode) the harvested session-log path. * * @param input The scenario's input script (steps + optional permission answers). * @param opts The agent to boot, the mode, and the fixture wiring. - * @returns The captured stdout/stderr, session id, temp cwd, and harvested logs. + * @returns The captured stdout/stderr, session id, generated cwd, and harvested logs. */ export async function runScenario(input: InputScript, opts: RunOptions): Promise { - const cwd = await mkdtemp(join(tmpdir(), 'acp-snap-cwd-')) + const cwd = await mkdtemp(join(opts.workspaceParent ?? tmpdir(), 'acp-snap-cwd-')) const sessionsRoot = await mkdtemp(join(tmpdir(), 'acp-snap-sessions-')) // Fixed path length: spill-policy budgets the preview against the REAL path // before stdout normalization, so tmpdir() length differences churn expected outputs. @@ -184,7 +192,7 @@ export async function runScenario(input: InputScript, opts: RunOptions): Promise let sessionLogs: HarvestedLog[] = [] const outcome = await (async (): Promise => { // Seed the workspace if the scenario ships one (a file the agent reads/edits). - // Copied into the temp cwd so the agent's bash tools see it; the expected outputs + // Copied into the generated cwd so the agent's bash tools see it; the expected outputs // normalize the cwd, so the seeded paths stay stable across runs. if (opts.workspaceDir !== undefined && existsSync(opts.workspaceDir)) { await cp(opts.workspaceDir, cwd, { recursive: true }) @@ -251,7 +259,7 @@ export async function runScenario(input: InputScript, opts: RunOptions): Promise // persistence) and exits. Then await exit so the harvested log is complete. await active.close() // Harvest EVERY persisted log (parent + any subagent children) while the - // temp dirs still exist, ordered primary-first. + // generated dirs still exist, ordered primary-first. sessionLogs = await harvestSessionLogs(sessionsRoot) return { rawStdout: launched.rawStdout(), diff --git a/packages/support/acp-snapshot/src/normalize.ts b/packages/support/acp-snapshot/src/normalize.ts index 6671959b29..5beada3ce6 100644 --- a/packages/support/acp-snapshot/src/normalize.ts +++ b/packages/support/acp-snapshot/src/normalize.ts @@ -1,5 +1,5 @@ /** - * Pure ACP transcript and session-log normalizers. They scrub session ids, temp cwd, RPC ids, + * Pure ACP transcript and session-log normalizers. They scrub session ids, run cwd, RPC ids, * timestamps, and hook duration while preserving deterministic event sequence numbers. * Request-header scrubbers stay composable so one scenario per header class can pin prompt and * tool-schema sidecars while retaining any model-visible prefix in the session log. @@ -29,7 +29,7 @@ const SNAPSHOT_SPILL_PATH_RE = new RegExp( export interface NormalizeContext { /** The session id(s) the run issued — replaced with `{{sessionId}}`. */ sessionIds: string[] - /** The temp cwd the run used — replaced with `{{cwd}}`. */ + /** The generated cwd the run used — replaced with `{{cwd}}`. */ cwd: string } diff --git a/packages/support/acp-snapshot/src/suite.ts b/packages/support/acp-snapshot/src/suite.ts index 60f5cfeadb..82e57db35e 100644 --- a/packages/support/acp-snapshot/src/suite.ts +++ b/packages/support/acp-snapshot/src/suite.ts @@ -100,6 +100,12 @@ export interface Scenario { * {@link headerClass}. */ configPath?: string + /** + * Parent directory for the generated session cwd. Defaults to the platform + * temp directory; set this when temp is itself part of the behavior under + * test and the scenario needs an independent project location. + */ + workspaceParent?: string } /** One suite's inputs: the agent to boot, where its fixtures live, and its scenario table. */ @@ -178,7 +184,7 @@ export function fixtureContext(fixture: string): NormalizeContext { * The `data.header` payload of every `request/header` event in a session * JSONL, in log order, with the log's volatile values scrubbed first * ({@link normalizeSessionLog}) so headers harvested from different runs — - * each embedding its own temp cwd in the composed prompt — compare on equal + * each embedding its own generated cwd in the composed prompt — compare on equal * footing. * * @param rawLog The session `.jsonl` content to extract headers from. @@ -488,6 +494,7 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void { // replays from its own script. In RECORD they are harvested, not read. ...!RECORDING && childFixtureFiles.length > 0 ? { childFiles: childFixtureFiles.map(file => join(dir, file)) } : {}, ...existsSync(workspaceDir) ? { workspaceDir } : {}, + ...scenario.workspaceParent !== undefined ? { workspaceParent: scenario.workspaceParent } : {}, // A scenario booting an overlay tree passes its own live config; the // bin's replay swap derives the sibling `*cordis.snapshot.yml` from it. ...scenario.configPath !== undefined ? { configPath: scenario.configPath } : {}, diff --git a/packages/support/acp-snapshot/tests/harness.spec.ts b/packages/support/acp-snapshot/tests/harness.spec.ts index 4aa24e290a..0d21d9b6a1 100644 --- a/packages/support/acp-snapshot/tests/harness.spec.ts +++ b/packages/support/acp-snapshot/tests/harness.spec.ts @@ -1,7 +1,7 @@ import { mkdtemp, rm, writeFile } from 'node:fs/promises' import { once } from 'node:events' import { tmpdir } from 'node:os' -import { delimiter, join } from 'node:path' +import { delimiter, join, relative, sep } from 'node:path' import { fileURLToPath } from 'node:url' import { afterAll, describe, expect, it, vi } from 'vitest' import { PROTOCOL_VERSION } from '@agentclientprotocol/sdk' @@ -345,6 +345,22 @@ describe('runScenario', () => { expect(result.rawStdout).toContain('workspace:seeded.txt') }) + it('creates the generated workspace under an explicit parent', { timeout: 20_000 }, async () => { + const { fixtureFile } = await scenario({}) + const workspaceParent = await mkdtemp(join(tmpdir(), 'acp-snap-parent-')) + tempDirs.push(workspaceParent) + + const result = await runScenario( + { steps: boot }, + { agent: AGENT, mode: 'replay', fixtureFile, workspaceParent }, + ) + + const child = relative(workspaceParent, result.cwd) + expect(child).not.toBe('') + expect(child).not.toBe('..') + expect(child.startsWith(`..${sep}`)).toBe(false) + }) + it('promptAndCancel waits for the streamed chunk, cancels, and settles the prompt', { timeout: 20_000 }, async () => { const { fixtureFile } = await scenario({ prompt: 'hang-until-cancel' }) const result = await runScenario( diff --git a/packages/support/acp-snapshot/tests/suite.spec.ts b/packages/support/acp-snapshot/tests/suite.spec.ts index 6fbb06bb9d..d425470f83 100644 --- a/packages/support/acp-snapshot/tests/suite.spec.ts +++ b/packages/support/acp-snapshot/tests/suite.spec.ts @@ -46,7 +46,14 @@ const RECORD_SRC = fileURLToPath(new URL('./fixtures/record-suite', import.meta. // Replay pins explicit header classes; recording covers the default fallback. const REPLAY_SCENARIOS: Scenario[] = [ { name: 'pin-turn', hasModelTurn: true, recorded: true, pinsHeader: true, expectedHeaderChanges: 1, headerClass: 'main' }, - { name: 'plain-turn', hasModelTurn: true, recorded: true, headerClass: 'main', configPath: AGENT.configPath }, + { + name: 'plain-turn', + hasModelTurn: true, + recorded: true, + headerClass: 'main', + configPath: AGENT.configPath, + workspaceParent: tmpdir(), + }, { name: 'no-model', hasModelTurn: false, recorded: false, headerClass: 'main' }, { name: 'blocked-log', hasModelTurn: false, comparesLog: true, recorded: false, headerClass: 'main' }, { name: 'authored-error', hasModelTurn: true, recorded: false, overridden: true, headerClass: 'main' },