Merge remote-tracking branch 'origin/master' into codex/skill-system
# Conflicts: # docs/rfc/INDEX.md # examples/acp-agent/tests/snapshots/both-mode-turn/session.jsonl # examples/acp-agent/tests/snapshots/code-mode-turn/session.jsonl # examples/acp-agent/tests/snapshots/text-turn/session.jsonl # packages/README.md
This commit is contained in:
Tianyi Cui
@@ -39,4 +39,4 @@ A scenario booting a differently-composed tree sets its own `configPath` (an ove
|
||||
|
||||
The example also ships a `cordis.snapshot.yml` replay overlay next to its `cordis.yml` (the bin swaps them under `DSH_SNAPSHOT=replay` — [single-source replay config RFC](../../../docs/rfc/implemented/testing/2026-07-04-single-source-acp-replay-config.md)); replay fixtures are served by [`dsh-llm-replay`](../llm-replay/README.md), which this package points at via the `DSH_SNAPSHOT_*` env vars it sets on the child. `pnpm run test:snapshot:record` calls the live LLM and rewrites the recorded scenarios' model fixtures; `pnpm run test:snapshot:refresh` stays keyless, runs the replay overlay, and rewrites stdout plus comparable session-log goldens from the committed model scripts. Fixture roles, record/replay/refresh semantics, and scenario-table fields are documented on `Scenario` and in the [snapshot RFC](../../../docs/rfc/implemented/testing/2026-06-19-acp-snapshot-tests.md).
|
||||
|
||||
Constraints: `suite.ts` imports vitest, so the package is importable only inside a vitest run (the harness and normalizers have no such dependency but ship from the same entry). ACP-specific by design — the harness speaks the SDK's `ClientSideConnection`. Permission round-trips are scriptable: `InputScript.permissionAnswers` is a FIFO queue of option-kind selections (`allow_once`, `reject_once`, …) the client maps to the agent-issued `optionId` at answer time; an absent or exhausted queue answers `cancelled`, and a kind the request never offered rejects the run (the agent is answered `cancelled`, so a tolerant agent cannot absorb the scenario bug).
|
||||
Constraints: `suite.ts` imports vitest, so the package is importable only inside a vitest run (the harness and normalizers have no such dependency but ship from the same entry). ACP-specific by design — the harness speaks the SDK's `ClientSideConnection`. Permission round-trips are scriptable: `InputScript.permissionAnswers` is a FIFO queue of option-kind selections (`allow_once`, `reject_once`, …) the client maps to the agent-issued `optionId` at answer time; an absent or exhausted queue answers `cancelled`, and a kind the request never offered rejects the run (the agent is answered `cancelled`, so a tolerant agent cannot absorb the scenario bug). Session config options are scriptable too: the `setConfigOption` step switches a knob over `session/set_config_option`, and `setConfigOptionExpectError` asserts the bridge rejects an unknown id or out-of-vocabulary value (the error frame stays in the transcript).
|
||||
|
||||
@@ -85,6 +85,8 @@ export type InputStep =
|
||||
| { op: 'promptExpectError'; text: string }
|
||||
| { op: 'promptAndCancel'; text: string }
|
||||
| { op: 'cancel' }
|
||||
| { op: 'setConfigOption'; configId: string; value: string }
|
||||
| { op: 'setConfigOptionExpectError'; configId: string; value: string }
|
||||
|
||||
/** A scenario's `input.json`: an ordered list of input steps. */
|
||||
export interface InputScript {
|
||||
@@ -408,6 +410,24 @@ async function runStep(
|
||||
await client.cancel({ sessionId })
|
||||
return
|
||||
}
|
||||
case 'setConfigOption': {
|
||||
const sessionId = getSessionId()
|
||||
if (sessionId === undefined) throw new Error('snapshot-harness: setConfigOption before newSession')
|
||||
await client.setSessionConfigOption({ sessionId, configId: step.configId, value: step.value })
|
||||
return
|
||||
}
|
||||
case 'setConfigOptionExpectError': {
|
||||
const sessionId = getSessionId()
|
||||
if (sessionId === undefined) throw new Error('snapshot-harness: setConfigOptionExpectError before newSession')
|
||||
// The bridge rejects an unknown id / out-of-vocabulary value; the SDK
|
||||
// surfaces that as a rejected RPC — swallow it so the run completes and
|
||||
// the error frame is captured in the transcript.
|
||||
await client.setSessionConfigOption({ sessionId, configId: step.configId, value: step.value }).then(
|
||||
() => { throw new Error('snapshot-harness: expected set_config_option to be rejected but it succeeded') },
|
||||
() => { /* expected: the bridge rejected the id or value */ },
|
||||
)
|
||||
return
|
||||
}
|
||||
default:
|
||||
throw new Error(`snapshot-harness: unknown input op ${JSON.stringify(step)}`)
|
||||
}
|
||||
|
||||
@@ -98,6 +98,15 @@ export interface Scenario {
|
||||
* Defaults to false.
|
||||
*/
|
||||
pinsHeader?: boolean
|
||||
/**
|
||||
* How many `request/header-delta` events this PINNING scenario's fixture
|
||||
* legitimately carries (default 0). A recorded mid-run header change — a
|
||||
* config-option switch rewriting a prompt section — is part of the pinned
|
||||
* surface, committed verbatim like the header itself; any OTHER count
|
||||
* still fails, so fixture rot stays caught. Meaningless off the pin (the
|
||||
* live uniformity guard keeps non-pinning scenarios delta-free).
|
||||
*/
|
||||
expectedHeaderDeltas?: number
|
||||
/**
|
||||
* Which header-composition class this scenario belongs to. Scenarios that
|
||||
* boot the same config compose the same header; each class has exactly one
|
||||
@@ -511,17 +520,19 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
|
||||
}
|
||||
})
|
||||
|
||||
it('every pinning fixture carries exactly one request/header and no deltas', async () => {
|
||||
it('every pinning fixture carries exactly one request/header and its declared deltas', async () => {
|
||||
// The live uniformity guard runs only in NON-pinning scenarios, so a
|
||||
// class made of just its pinning scenario would otherwise accept a
|
||||
// re-recorded pin with several headers or a mid-run header-delta —
|
||||
// shapes the pin design cannot represent. Assert the committed pins
|
||||
// directly.
|
||||
// re-recorded pin with several headers or an undeclared mid-run
|
||||
// header-delta — shapes the pin design cannot represent. Assert the
|
||||
// committed pins directly; a scenario whose arc legitimately rewrites
|
||||
// a prompt section declares the exact count via expectedHeaderDeltas.
|
||||
for (const scenario of pinningByClass.values()) {
|
||||
const fixture = await readFile(join(snapshotsDir, scenario.name, 'session.jsonl'), 'utf8')
|
||||
const headers = normalizedHeaders(fixture, fixtureContext(fixture))
|
||||
expect(headers.length, `${scenario.name}: a pinning fixture must carry exactly one request/header`).toBe(1)
|
||||
expect(headerDeltaCount(fixture), `${scenario.name}: a pinning fixture must carry no request/header-delta`).toBe(0)
|
||||
expect(headerDeltaCount(fixture), `${scenario.name}: a pinning fixture must carry exactly its declared request/header-deltas`)
|
||||
.toBe(scenario.expectedHeaderDeltas ?? 0)
|
||||
}
|
||||
})
|
||||
|
||||
|
||||
@@ -58,6 +58,13 @@ interface Behavior {
|
||||
strayBucketFile?: boolean
|
||||
/** Delete the sessions root entirely (harvest must yield no logs). */
|
||||
deleteSessionsRoot?: boolean
|
||||
/**
|
||||
* Vocabulary for `session/set_config_option`: allowed values per config id.
|
||||
* A set naming an unknown id or an out-of-vocabulary value rejects (the
|
||||
* real bridge's rule); a valid set answers with the complete refreshed
|
||||
* option state, `currentValue` updated. Absent: every set rejects.
|
||||
*/
|
||||
configOptions?: Record<string, string[]>
|
||||
}
|
||||
|
||||
const sessionsRoot = process.env.DSH_SNAPSHOT_SESSIONS_ROOT ?? ''
|
||||
@@ -81,6 +88,8 @@ let sessionCwd = ''
|
||||
let parkedPromptId: number | string | null = null
|
||||
/** Resolvers for permission-probe responses, keyed by outbound request id. */
|
||||
const pendingPermission = new Map<number, (outcome: unknown) => void>()
|
||||
/** Per-run `session/set_config_option` state: config id → current value (first vocabulary entry until set). */
|
||||
const currentConfig: Record<string, string> = {}
|
||||
|
||||
function send(frame: Record<string, unknown>): void {
|
||||
process.stdout.write(`${JSON.stringify({ jsonrpc: '2.0', ...frame })}\n`)
|
||||
@@ -195,6 +204,32 @@ function handleFrame(frame: Record<string, unknown>): void {
|
||||
case 'session/prompt':
|
||||
void handlePrompt(id as number | string)
|
||||
return
|
||||
case 'session/set_config_option': {
|
||||
const vocabulary = behavior.configOptions
|
||||
const configId = params.configId as string
|
||||
const value = params.value as string
|
||||
const values = vocabulary?.[configId]
|
||||
if (values === undefined) {
|
||||
respondError(id as number | string, `unknown config option ${configId}`)
|
||||
return
|
||||
}
|
||||
if (!values.includes(value)) {
|
||||
respondError(id as number | string, `unknown ${configId} value ${value}`)
|
||||
return
|
||||
}
|
||||
currentConfig[configId] = value
|
||||
// The real bridge's contract: every set answers with the COMPLETE
|
||||
// refreshed option state, not just the changed entry.
|
||||
respond(id as number | string, {
|
||||
configOptions: Object.entries(vocabulary as Record<string, string[]>).map(([cid, vs]) => ({
|
||||
id: cid,
|
||||
type: 'select',
|
||||
currentValue: currentConfig[cid] ?? vs[0],
|
||||
options: vs.map(v => ({ value: v, name: v })),
|
||||
})),
|
||||
})
|
||||
return
|
||||
}
|
||||
case 'session/cancel':
|
||||
if (parkedPromptId !== null) {
|
||||
const parked = parkedPromptId
|
||||
|
||||
@@ -168,6 +168,8 @@ describe('runScenario', () => {
|
||||
[{ op: 'promptExpectError', text: 'x' }, /promptExpectError before newSession/],
|
||||
[{ op: 'promptAndCancel', text: 'x' }, /promptAndCancel before newSession/],
|
||||
[{ op: 'cancel' }, /cancel before newSession/],
|
||||
[{ op: 'setConfigOption', configId: 'sandbox-mode', value: 'read-only' }, /setConfigOption before newSession/],
|
||||
[{ op: 'setConfigOptionExpectError', configId: 'sandbox-mode', value: 'yolo' }, /setConfigOptionExpectError before newSession/],
|
||||
] as [InputStep, RegExp][])('rejects %j before newSession', { timeout: 20_000 }, async (step, message) => {
|
||||
const { fixtureFile } = await scenario({})
|
||||
await expect(runScenario(
|
||||
@@ -176,6 +178,53 @@ describe('runScenario', () => {
|
||||
)).rejects.toThrow(message)
|
||||
})
|
||||
|
||||
it('setConfigOption switches a value and receives the complete refreshed option state', { timeout: 20_000 }, async () => {
|
||||
const { fixtureFile } = await scenario({
|
||||
configOptions: { 'sandbox-mode': ['read-only', 'workspace-write'], 'approval-policy': ['ask', 'never'] },
|
||||
})
|
||||
const result = await runScenario(
|
||||
{
|
||||
steps: [...boot,
|
||||
{ op: 'setConfigOption', configId: 'sandbox-mode', value: 'workspace-write' },
|
||||
{ op: 'setConfigOption', configId: 'approval-policy', value: 'never' }],
|
||||
},
|
||||
{ agent: AGENT, mode: 'replay', fixtureFile },
|
||||
)
|
||||
// Every set answers with the FULL state: the second response carries the
|
||||
// first switch's value too — the complete-refreshed-state contract.
|
||||
const frames = result.rawStdout.trim().split('\n').map(line => JSON.parse(line) as { result?: { configOptions?: { id: string; currentValue: string }[] } })
|
||||
const states = frames
|
||||
.map(f => f.result?.configOptions)
|
||||
.filter(options => options !== undefined)
|
||||
.map(options => Object.fromEntries((options as { id: string; currentValue: string }[]).map(o => [o.id, o.currentValue])))
|
||||
expect(states).toEqual([
|
||||
{ 'sandbox-mode': 'workspace-write', 'approval-policy': 'ask' },
|
||||
{ 'sandbox-mode': 'workspace-write', 'approval-policy': 'never' },
|
||||
])
|
||||
})
|
||||
|
||||
it('setConfigOptionExpectError swallows the rejection for unknown ids and out-of-vocabulary values', { timeout: 20_000 }, async () => {
|
||||
const { fixtureFile } = await scenario({ configOptions: { 'sandbox-mode': ['read-only'] } })
|
||||
const result = await runScenario(
|
||||
{
|
||||
steps: [...boot,
|
||||
{ op: 'setConfigOptionExpectError', configId: 'sandbox-mode', value: 'yolo' },
|
||||
{ op: 'setConfigOptionExpectError', configId: 'reasoning-effort', value: 'max' }],
|
||||
},
|
||||
{ agent: AGENT, mode: 'replay', fixtureFile },
|
||||
)
|
||||
expect(result.rawStdout).toContain('unknown sandbox-mode value yolo')
|
||||
expect(result.rawStdout).toContain('unknown config option reasoning-effort')
|
||||
})
|
||||
|
||||
it('setConfigOptionExpectError throws when the set unexpectedly succeeds', { timeout: 20_000 }, async () => {
|
||||
const { fixtureFile } = await scenario({ configOptions: { 'sandbox-mode': ['read-only'] } })
|
||||
await expect(runScenario(
|
||||
{ steps: [...boot, { op: 'setConfigOptionExpectError', configId: 'sandbox-mode', value: 'read-only' }] },
|
||||
{ agent: AGENT, mode: 'replay', fixtureFile },
|
||||
)).rejects.toThrow(/expected set_config_option to be rejected/)
|
||||
})
|
||||
|
||||
it('rejects an unknown input op', { timeout: 20_000 }, async () => {
|
||||
const { fixtureFile } = await scenario({})
|
||||
const bogus = { op: 'reticulate' } as unknown as InputStep
|
||||
|
||||
Reference in New Issue
Block a user