Merge remote-tracking branch 'origin/master' into codex/system-prompt-snapshots
# Conflicts: # examples/acp-agent/tests/snapshots/both-mode-turn/session.jsonl # examples/acp-agent/tests/snapshots/code-mode-turn/session.jsonl # examples/acp-agent/tests/snapshots/text-turn/session.jsonl # packages/support/acp-snapshot/src/suite.ts
This commit is contained in:
@@ -39,4 +39,4 @@ A scenario booting a differently-composed tree sets its own `configPath` (an ove
|
||||
|
||||
The example also ships a `cordis.snapshot.yml` replay overlay next to its `cordis.yml` (the bin swaps them under `DSH_SNAPSHOT=replay` — [single-source replay config RFC](../../../docs/rfc/implemented/testing/2026-07-04-single-source-acp-replay-config.md)); replay fixtures are served by [`dsh-llm-replay`](../llm-replay/README.md), which this package points at via the `DSH_SNAPSHOT_*` env vars it sets on the child. `pnpm run test:snapshot:record` calls the live LLM and rewrites the recorded scenarios' model fixtures; `pnpm run test:snapshot:refresh` stays keyless, runs the replay overlay, and rewrites stdout, comparable session-log goldens, and each pin's Markdown prompt snapshot from the committed model scripts. Fixture roles, record/replay/refresh semantics, and scenario-table fields are documented on `Scenario` and in the [snapshot RFC](../../../docs/rfc/implemented/testing/2026-06-19-acp-snapshot-tests.md).
|
||||
|
||||
Constraints: `suite.ts` imports vitest, so the package is importable only inside a vitest run (the harness and normalizers have no such dependency but ship from the same entry). ACP-specific by design — the harness speaks the SDK's `ClientSideConnection`. Permission round-trips are scriptable: `InputScript.permissionAnswers` is a FIFO queue of option-kind selections (`allow_once`, `reject_once`, …) the client maps to the agent-issued `optionId` at answer time; an absent or exhausted queue answers `cancelled`, and a kind the request never offered rejects the run (the agent is answered `cancelled`, so a tolerant agent cannot absorb the scenario bug).
|
||||
Constraints: `suite.ts` imports vitest, so the package is importable only inside a vitest run (the harness and normalizers have no such dependency but ship from the same entry). ACP-specific by design — the harness speaks the SDK's `ClientSideConnection`. Permission round-trips are scriptable: `InputScript.permissionAnswers` is a FIFO queue of option-kind selections (`allow_once`, `reject_once`, …) the client maps to the agent-issued `optionId` at answer time; an absent or exhausted queue answers `cancelled`, and a kind the request never offered rejects the run (the agent is answered `cancelled`, so a tolerant agent cannot absorb the scenario bug). Session config options are scriptable too: the `setConfigOption` step switches a knob over `session/set_config_option`, and `setConfigOptionExpectError` asserts the bridge rejects an unknown id or out-of-vocabulary value (the error frame stays in the transcript).
|
||||
|
||||
@@ -85,6 +85,8 @@ export type InputStep =
|
||||
| { op: 'promptExpectError'; text: string }
|
||||
| { op: 'promptAndCancel'; text: string }
|
||||
| { op: 'cancel' }
|
||||
| { op: 'setConfigOption'; configId: string; value: string }
|
||||
| { op: 'setConfigOptionExpectError'; configId: string; value: string }
|
||||
|
||||
/** A scenario's `input.json`: an ordered list of input steps. */
|
||||
export interface InputScript {
|
||||
@@ -210,6 +212,8 @@ export async function runScenario(input: InputScript, opts: RunOptions): Promise
|
||||
DSH_SNAPSHOT: opts.mode,
|
||||
DSH_SNAPSHOT_FILE: opts.fixtureFile,
|
||||
DSH_SNAPSHOT_SESSIONS_ROOT: sessionsRoot,
|
||||
DSH_HOME: join(cwd, '.dsh'),
|
||||
DSH_AGENTS_HOME: join(cwd, '.agents'),
|
||||
...opts.overrideFile !== undefined ? { DSH_SNAPSHOT_OVERRIDE: opts.overrideFile } : {},
|
||||
...opts.childFiles !== undefined && opts.childFiles.length > 0
|
||||
? { DSH_SNAPSHOT_CHILD_FILES: opts.childFiles.join(delimiter) }
|
||||
@@ -406,6 +410,24 @@ async function runStep(
|
||||
await client.cancel({ sessionId })
|
||||
return
|
||||
}
|
||||
case 'setConfigOption': {
|
||||
const sessionId = getSessionId()
|
||||
if (sessionId === undefined) throw new Error('snapshot-harness: setConfigOption before newSession')
|
||||
await client.setSessionConfigOption({ sessionId, configId: step.configId, value: step.value })
|
||||
return
|
||||
}
|
||||
case 'setConfigOptionExpectError': {
|
||||
const sessionId = getSessionId()
|
||||
if (sessionId === undefined) throw new Error('snapshot-harness: setConfigOptionExpectError before newSession')
|
||||
// The bridge rejects an unknown id / out-of-vocabulary value; the SDK
|
||||
// surfaces that as a rejected RPC — swallow it so the run completes and
|
||||
// the error frame is captured in the transcript.
|
||||
await client.setSessionConfigOption({ sessionId, configId: step.configId, value: step.value }).then(
|
||||
() => { throw new Error('snapshot-harness: expected set_config_option to be rejected but it succeeded') },
|
||||
() => { /* expected: the bridge rejected the id or value */ },
|
||||
)
|
||||
return
|
||||
}
|
||||
default:
|
||||
throw new Error(`snapshot-harness: unknown input op ${JSON.stringify(step)}`)
|
||||
}
|
||||
|
||||
@@ -105,6 +105,15 @@ export interface Scenario {
|
||||
* Defaults to false.
|
||||
*/
|
||||
pinsHeader?: boolean
|
||||
/**
|
||||
* How many `request/header-delta` events this PINNING scenario's fixture
|
||||
* legitimately carries (default 0). A recorded mid-run header change — a
|
||||
* config-option switch rewriting a prompt section — is part of the pinned
|
||||
* surface, with readable prompt text in Markdown; any OTHER count
|
||||
* still fails, so fixture rot stays caught. Meaningless off the pin (the
|
||||
* live uniformity guard keeps non-pinning scenarios delta-free).
|
||||
*/
|
||||
expectedHeaderDeltas?: number
|
||||
/**
|
||||
* Which header-composition class this scenario belongs to. Scenarios that
|
||||
* boot the same config compose the same header; each class has exactly one
|
||||
@@ -216,16 +225,67 @@ export function normalizedSystemPrompts(rawLog: string, ctx: NormalizeContext):
|
||||
})
|
||||
}
|
||||
|
||||
/** One normalized system-prompt edit carried by a `request/header-delta`. */
|
||||
export interface SystemPromptDeltaSnapshot {
|
||||
/** How many leading lines remain from the prior prompt. */
|
||||
keepStart: number
|
||||
/** How many trailing lines remain from the prior prompt. */
|
||||
keepEnd: number
|
||||
/** The normalized replacement lines inserted between the retained ranges. */
|
||||
insert: string[]
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract normalized system-prompt edits from request-header deltas in log
|
||||
* order. Deltas without a well-formed system edit are omitted; their non-prompt
|
||||
* structure remains pinned in JSONL.
|
||||
*
|
||||
* @param rawLog The session `.jsonl` content to inspect.
|
||||
* @param ctx The volatile values of the run that produced it.
|
||||
* @returns The normalized system-prompt edits, in event order.
|
||||
*/
|
||||
export function normalizedSystemPromptDeltas(rawLog: string, ctx: NormalizeContext): SystemPromptDeltaSnapshot[] {
|
||||
return normalizeSessionLog(rawLog, ctx)
|
||||
.split('\n')
|
||||
.filter(line => line.trim().length > 0)
|
||||
.map(line => JSON.parse(line) as { type?: unknown; data?: { system?: unknown } })
|
||||
.filter(record => record.type === 'request/header-delta')
|
||||
.flatMap((record) => {
|
||||
const system = record.data?.system
|
||||
if (system === null || typeof system !== 'object') return []
|
||||
const { keepStart, keepEnd, insert } = system as { keepStart?: unknown; keepEnd?: unknown; insert?: unknown }
|
||||
if (typeof keepStart !== 'number' || typeof keepEnd !== 'number' || !Array.isArray(insert)) return []
|
||||
if (!insert.every(line => typeof line === 'string')) return []
|
||||
return [{ keepStart, keepEnd, insert: insert }]
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Render a normalized prompt as a repository-friendly Markdown snapshot.
|
||||
* Prompt text is unchanged except that a missing terminal newline is added so
|
||||
* the committed file follows the repository newline contract.
|
||||
*
|
||||
* @param prompt The normalized system prompt.
|
||||
* @param deltas Normalized prompt edits to append as readable sections.
|
||||
* @returns Markdown snapshot text ending in a newline.
|
||||
*/
|
||||
export function formatSystemPromptSnapshot(prompt: string): string {
|
||||
return prompt.endsWith('\n') ? prompt : `${prompt}\n`
|
||||
export function formatSystemPromptSnapshot(
|
||||
prompt: string,
|
||||
deltas: readonly SystemPromptDeltaSnapshot[] = [],
|
||||
): string {
|
||||
let snapshot = prompt.endsWith('\n') ? prompt : `${prompt}\n`
|
||||
for (const [index, delta] of deltas.entries()) {
|
||||
snapshot += `\n<!-- request/header-delta ${index + 1}: keepStart=${delta.keepStart}, keepEnd=${delta.keepEnd} -->\n\n`
|
||||
const insert = delta.insert.join('\n')
|
||||
snapshot += insert.endsWith('\n') ? insert : `${insert}\n`
|
||||
}
|
||||
return snapshot
|
||||
}
|
||||
|
||||
/** Return the initial-prompt portion of a possibly delta-bearing snapshot. */
|
||||
function initialSystemPromptSnapshot(snapshot: string): string {
|
||||
const marker = snapshot.indexOf('\n<!-- request/header-delta ')
|
||||
return marker < 0 ? snapshot : snapshot.slice(0, marker)
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -432,11 +492,16 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
|
||||
if (scenario.pinsHeader === true) {
|
||||
const prompts = result.sessionLogs.flatMap(log => normalizedSystemPrompts(log.content, ctx))
|
||||
expect(prompts.length, `${mode} produced no system prompt to snapshot`).toBeGreaterThan(0)
|
||||
const snapshot = formatSystemPromptSnapshot(prompts[0] as string)
|
||||
const initialSnapshot = formatSystemPromptSnapshot(prompts[0] as string)
|
||||
for (const prompt of prompts) {
|
||||
expect(formatSystemPromptSnapshot(prompt), 'the pinning run produced divergent system prompts')
|
||||
.toEqual(snapshot)
|
||||
.toEqual(initialSnapshot)
|
||||
}
|
||||
const primary = result.sessionLogs[0] as HarvestedLog
|
||||
const snapshot = formatSystemPromptSnapshot(
|
||||
prompts[0] as string,
|
||||
normalizedSystemPromptDeltas(primary.content, ctx),
|
||||
)
|
||||
await writeFile(join(dir, SYSTEM_PROMPT_SNAPSHOT), snapshot)
|
||||
}
|
||||
}
|
||||
@@ -469,19 +534,24 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
|
||||
|
||||
// Header-uniformity guard: every live header in a class must equal the
|
||||
// class pin split across its JSONL header (system token + real tools)
|
||||
// and readable Markdown prompt. No header delta is representable by
|
||||
// those two static artifacts, so any delta fails loud.
|
||||
// and readable Markdown prompt. A pinning scenario may carry its
|
||||
// declared header deltas; their prompt edits live in the Markdown
|
||||
// golden while JSONL retains the tokenized edit structure.
|
||||
/* v8 ignore next -- construction guarantees the pin exists; a miss would fail the one-header assertion loudly. */
|
||||
const pinningScenario = pinningByClass.get(classOf(scenario)) ?? scenario
|
||||
const pinningDir = join(snapshotsDir, pinningScenario.name)
|
||||
const pinnedFixture = await readFile(join(pinningDir, 'session.jsonl'), 'utf8')
|
||||
const pinned = normalizedHeaders(pinnedFixture, fixtureContext(pinnedFixture))
|
||||
const promptSnapshot = await readFile(join(pinningDir, SYSTEM_PROMPT_SNAPSHOT), 'utf8')
|
||||
const initialPromptSnapshot = initialSystemPromptSnapshot(promptSnapshot)
|
||||
expect(pinned.length, `the pinning fixture (${pinningScenario.name}) must carry exactly one request/header`)
|
||||
.toBe(1)
|
||||
for (const log of result.sessionLogs) {
|
||||
expect(headerDeltaCount(log.content), `session ${log.id}: a request/header-delta is not represented by the class pin`)
|
||||
.toBe(0)
|
||||
for (const [logIndex, log] of result.sessionLogs.entries()) {
|
||||
const expectedDeltas = scenario.pinsHeader === true && logIndex === 0
|
||||
? scenario.expectedHeaderDeltas ?? 0
|
||||
: 0
|
||||
expect(headerDeltaCount(log.content), `session ${log.id}: request/header-delta count`)
|
||||
.toBe(expectedDeltas)
|
||||
const headers = normalizedHeaders(scrubSystemPrompts(log.content), ctx)
|
||||
const prompts = normalizedSystemPrompts(log.content, ctx)
|
||||
expect(prompts.length, `session ${log.id}: every request/header must carry a string system prompt`)
|
||||
@@ -489,7 +559,14 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
|
||||
for (const [k, header] of headers.entries()) {
|
||||
expect(header, `session ${log.id}: request/header #${k + 1} diverged from the pinned (${pinningScenario.name}) header`)
|
||||
.toEqual(pinned[0])
|
||||
expect(formatSystemPromptSnapshot(prompts[k] as string), `session ${log.id}: system prompt #${k + 1} diverged from ${pinningScenario.name}/${SYSTEM_PROMPT_SNAPSHOT}`)
|
||||
expect(formatSystemPromptSnapshot(prompts[k] as string), `session ${log.id}: initial system prompt #${k + 1} diverged from ${pinningScenario.name}/${SYSTEM_PROMPT_SNAPSHOT}`)
|
||||
.toEqual(initialPromptSnapshot)
|
||||
}
|
||||
if (scenario.pinsHeader === true && logIndex === 0) {
|
||||
expect(formatSystemPromptSnapshot(
|
||||
prompts[0] as string,
|
||||
normalizedSystemPromptDeltas(log.content, ctx),
|
||||
), `session ${log.id}: system-prompt deltas diverged from ${pinningScenario.name}/${SYSTEM_PROMPT_SNAPSHOT}`)
|
||||
.toEqual(promptSnapshot)
|
||||
}
|
||||
}
|
||||
@@ -555,12 +632,13 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
|
||||
}
|
||||
})
|
||||
|
||||
it('every pinning fixture carries exactly one request/header, one readable prompt, and no deltas', async () => {
|
||||
it('every pinning fixture carries one request/header, one readable prompt, and its declared deltas', async () => {
|
||||
// The live uniformity guard runs only in NON-pinning scenarios, so a
|
||||
// class made of just its pinning scenario would otherwise accept a
|
||||
// re-recorded pin with several headers or a mid-run header-delta —
|
||||
// shapes the pin design cannot represent. Assert the committed pins
|
||||
// directly.
|
||||
// re-recorded pin with several headers or an undeclared mid-run
|
||||
// header-delta — shapes the pin design cannot represent. Assert the
|
||||
// committed pins directly; a scenario whose arc legitimately rewrites
|
||||
// a prompt section declares the exact count via expectedHeaderDeltas.
|
||||
for (const scenario of pinningByClass.values()) {
|
||||
const fixture = await readFile(join(snapshotsDir, scenario.name, 'session.jsonl'), 'utf8')
|
||||
const headers = normalizedHeaders(fixture, fixtureContext(fixture))
|
||||
@@ -568,7 +646,8 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
|
||||
expect(headers.length, `${scenario.name}: a pinning fixture must carry exactly one request/header`).toBe(1)
|
||||
expect(promptSnapshot.length, `${scenario.name}/${SYSTEM_PROMPT_SNAPSHOT} must not be empty`).toBeGreaterThan(0)
|
||||
expect(promptSnapshot.endsWith('\n'), `${scenario.name}/${SYSTEM_PROMPT_SNAPSHOT} must end in a newline`).toBe(true)
|
||||
expect(headerDeltaCount(fixture), `${scenario.name}: a pinning fixture must carry no request/header-delta`).toBe(0)
|
||||
expect(headerDeltaCount(fixture), `${scenario.name}: a pinning fixture must carry exactly its declared request/header-deltas`)
|
||||
.toBe(scenario.expectedHeaderDeltas ?? 0)
|
||||
}
|
||||
})
|
||||
|
||||
|
||||
@@ -58,6 +58,13 @@ interface Behavior {
|
||||
strayBucketFile?: boolean
|
||||
/** Delete the sessions root entirely (harvest must yield no logs). */
|
||||
deleteSessionsRoot?: boolean
|
||||
/**
|
||||
* Vocabulary for `session/set_config_option`: allowed values per config id.
|
||||
* A set naming an unknown id or an out-of-vocabulary value rejects (the
|
||||
* real bridge's rule); a valid set answers with the complete refreshed
|
||||
* option state, `currentValue` updated. Absent: every set rejects.
|
||||
*/
|
||||
configOptions?: Record<string, string[]>
|
||||
}
|
||||
|
||||
const sessionsRoot = process.env.DSH_SNAPSHOT_SESSIONS_ROOT ?? ''
|
||||
@@ -81,6 +88,8 @@ let sessionCwd = ''
|
||||
let parkedPromptId: number | string | null = null
|
||||
/** Resolvers for permission-probe responses, keyed by outbound request id. */
|
||||
const pendingPermission = new Map<number, (outcome: unknown) => void>()
|
||||
/** Per-run `session/set_config_option` state: config id → current value (first vocabulary entry until set). */
|
||||
const currentConfig: Record<string, string> = {}
|
||||
|
||||
function send(frame: Record<string, unknown>): void {
|
||||
process.stdout.write(`${JSON.stringify({ jsonrpc: '2.0', ...frame })}\n`)
|
||||
@@ -195,6 +204,32 @@ function handleFrame(frame: Record<string, unknown>): void {
|
||||
case 'session/prompt':
|
||||
void handlePrompt(id as number | string)
|
||||
return
|
||||
case 'session/set_config_option': {
|
||||
const vocabulary = behavior.configOptions
|
||||
const configId = params.configId as string
|
||||
const value = params.value as string
|
||||
const values = vocabulary?.[configId]
|
||||
if (values === undefined) {
|
||||
respondError(id as number | string, `unknown config option ${configId}`)
|
||||
return
|
||||
}
|
||||
if (!values.includes(value)) {
|
||||
respondError(id as number | string, `unknown ${configId} value ${value}`)
|
||||
return
|
||||
}
|
||||
currentConfig[configId] = value
|
||||
// The real bridge's contract: every set answers with the COMPLETE
|
||||
// refreshed option state, not just the changed entry.
|
||||
respond(id as number | string, {
|
||||
configOptions: Object.entries(vocabulary as Record<string, string[]>).map(([cid, vs]) => ({
|
||||
id: cid,
|
||||
type: 'select',
|
||||
currentValue: currentConfig[cid] ?? vs[0],
|
||||
options: vs.map(v => ({ value: v, name: v })),
|
||||
})),
|
||||
})
|
||||
return
|
||||
}
|
||||
case 'session/cancel':
|
||||
if (parkedPromptId !== null) {
|
||||
const parked = parkedPromptId
|
||||
|
||||
@@ -5,7 +5,8 @@
|
||||
"lines": [
|
||||
{ "type": "session", "id": "{{SID}}", "createdAt": 100, "cwd": "{{CWD}}" },
|
||||
{ "type": "request/header", "seq": 0, "time": 100, "data": { "header": { "config": { "model": "fake" }, "system": "SYS PROMPT", "tools": [{ "name": "t1", "description": "D1", "parameters": { "type": "object" } }] }, "reason": "initial" } },
|
||||
{ "type": "turn/start", "seq": 1, "time": 100, "data": { "turn": 1 } }
|
||||
{ "type": "request/header-delta", "seq": 1, "time": 100, "data": { "system": { "keepStart": 1, "keepEnd": 0, "insert": ["NEW PROMPT LINE"] } } },
|
||||
{ "type": "turn/start", "seq": 2, "time": 100, "data": { "turn": 1 } }
|
||||
]
|
||||
}]
|
||||
}
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
{"type":"session","id":"12121212-3434-4545-8686-787878787878","createdAt":7,"cwd":"/rec/pin-cwd"}
|
||||
{"type":"request/header","seq":0,"time":7,"data":{"header":{"config":{"model":"fake"},"system":"{{system}}","tools":[{"name":"t1","description":"D1","parameters":{"type":"object"}}]},"reason":"initial"}}
|
||||
{"type":"turn/start","seq":1,"time":7,"data":{"turn":1}}
|
||||
{"type":"request/header-delta","seq":1,"time":7,"data":{"system":{"keepStart":1,"keepEnd":0,"insert":["{{system}}"]}}}
|
||||
{"type":"turn/start","seq":2,"time":7,"data":{"turn":1}}
|
||||
|
||||
@@ -1 +1,5 @@
|
||||
SYS PROMPT
|
||||
|
||||
<!-- request/header-delta 1: keepStart=1, keepEnd=0 -->
|
||||
|
||||
NEW PROMPT LINE
|
||||
|
||||
@@ -168,6 +168,8 @@ describe('runScenario', () => {
|
||||
[{ op: 'promptExpectError', text: 'x' }, /promptExpectError before newSession/],
|
||||
[{ op: 'promptAndCancel', text: 'x' }, /promptAndCancel before newSession/],
|
||||
[{ op: 'cancel' }, /cancel before newSession/],
|
||||
[{ op: 'setConfigOption', configId: 'sandbox-mode', value: 'read-only' }, /setConfigOption before newSession/],
|
||||
[{ op: 'setConfigOptionExpectError', configId: 'sandbox-mode', value: 'yolo' }, /setConfigOptionExpectError before newSession/],
|
||||
] as [InputStep, RegExp][])('rejects %j before newSession', { timeout: 20_000 }, async (step, message) => {
|
||||
const { fixtureFile } = await scenario({})
|
||||
await expect(runScenario(
|
||||
@@ -176,6 +178,53 @@ describe('runScenario', () => {
|
||||
)).rejects.toThrow(message)
|
||||
})
|
||||
|
||||
it('setConfigOption switches a value and receives the complete refreshed option state', { timeout: 20_000 }, async () => {
|
||||
const { fixtureFile } = await scenario({
|
||||
configOptions: { 'sandbox-mode': ['read-only', 'workspace-write'], 'approval-policy': ['ask', 'never'] },
|
||||
})
|
||||
const result = await runScenario(
|
||||
{
|
||||
steps: [...boot,
|
||||
{ op: 'setConfigOption', configId: 'sandbox-mode', value: 'workspace-write' },
|
||||
{ op: 'setConfigOption', configId: 'approval-policy', value: 'never' }],
|
||||
},
|
||||
{ agent: AGENT, mode: 'replay', fixtureFile },
|
||||
)
|
||||
// Every set answers with the FULL state: the second response carries the
|
||||
// first switch's value too — the complete-refreshed-state contract.
|
||||
const frames = result.rawStdout.trim().split('\n').map(line => JSON.parse(line) as { result?: { configOptions?: { id: string; currentValue: string }[] } })
|
||||
const states = frames
|
||||
.map(f => f.result?.configOptions)
|
||||
.filter(options => options !== undefined)
|
||||
.map(options => Object.fromEntries((options as { id: string; currentValue: string }[]).map(o => [o.id, o.currentValue])))
|
||||
expect(states).toEqual([
|
||||
{ 'sandbox-mode': 'workspace-write', 'approval-policy': 'ask' },
|
||||
{ 'sandbox-mode': 'workspace-write', 'approval-policy': 'never' },
|
||||
])
|
||||
})
|
||||
|
||||
it('setConfigOptionExpectError swallows the rejection for unknown ids and out-of-vocabulary values', { timeout: 20_000 }, async () => {
|
||||
const { fixtureFile } = await scenario({ configOptions: { 'sandbox-mode': ['read-only'] } })
|
||||
const result = await runScenario(
|
||||
{
|
||||
steps: [...boot,
|
||||
{ op: 'setConfigOptionExpectError', configId: 'sandbox-mode', value: 'yolo' },
|
||||
{ op: 'setConfigOptionExpectError', configId: 'reasoning-effort', value: 'max' }],
|
||||
},
|
||||
{ agent: AGENT, mode: 'replay', fixtureFile },
|
||||
)
|
||||
expect(result.rawStdout).toContain('unknown sandbox-mode value yolo')
|
||||
expect(result.rawStdout).toContain('unknown config option reasoning-effort')
|
||||
})
|
||||
|
||||
it('setConfigOptionExpectError throws when the set unexpectedly succeeds', { timeout: 20_000 }, async () => {
|
||||
const { fixtureFile } = await scenario({ configOptions: { 'sandbox-mode': ['read-only'] } })
|
||||
await expect(runScenario(
|
||||
{ steps: [...boot, { op: 'setConfigOptionExpectError', configId: 'sandbox-mode', value: 'read-only' }] },
|
||||
{ agent: AGENT, mode: 'replay', fixtureFile },
|
||||
)).rejects.toThrow(/expected set_config_option to be rejected/)
|
||||
})
|
||||
|
||||
it('rejects an unknown input op', { timeout: 20_000 }, async () => {
|
||||
const { fixtureFile } = await scenario({})
|
||||
const bogus = { op: 'reticulate' } as unknown as InputStep
|
||||
|
||||
@@ -11,6 +11,7 @@ import {
|
||||
formatSystemPromptSnapshot,
|
||||
headerDeltaCount,
|
||||
normalizedHeaders,
|
||||
normalizedSystemPromptDeltas,
|
||||
normalizedSystemPrompts,
|
||||
refreshFixtureReplacements,
|
||||
stabilizeRefreshLog,
|
||||
@@ -50,7 +51,7 @@ const RECORD_SRC = fileURLToPath(new URL('./fixtures/record-suite', import.meta.
|
||||
// is what this suite can exercise; the real overlay boot is the acp-agent
|
||||
// example's code-mode scenarios).
|
||||
const REPLAY_SCENARIOS: Scenario[] = [
|
||||
{ name: 'pin-turn', hasModelTurn: true, recorded: true, pinsHeader: true, headerClass: 'main' },
|
||||
{ name: 'pin-turn', hasModelTurn: true, recorded: true, pinsHeader: true, expectedHeaderDeltas: 1, headerClass: 'main' },
|
||||
{ name: 'plain-turn', hasModelTurn: true, recorded: true, childSessions: 1, headerClass: 'main', configPath: AGENT.configPath },
|
||||
{ name: 'no-model', hasModelTurn: false, recorded: false, headerClass: 'main' },
|
||||
{ name: 'blocked-log', hasModelTurn: false, comparesLog: true, recorded: false, headerClass: 'main' },
|
||||
@@ -128,7 +129,14 @@ describe('defineAcpSnapshotSuite: refresh write-back', () => {
|
||||
expect(authored).toContain('"error":"model exploded"')
|
||||
expect(authored).not.toContain('"error":"stale"')
|
||||
|
||||
expect(readFileSync(join(refreshDir, 'pin-turn', 'system-prompt.golden.md'), 'utf8')).toBe('SYS PROMPT\n')
|
||||
expect(readFileSync(join(refreshDir, 'pin-turn', 'system-prompt.golden.md'), 'utf8')).toBe([
|
||||
'SYS PROMPT',
|
||||
'',
|
||||
'<!-- request/header-delta 1: keepStart=1, keepEnd=0 -->',
|
||||
'',
|
||||
'NEW PROMPT LINE',
|
||||
'',
|
||||
].join('\n'))
|
||||
})
|
||||
})
|
||||
|
||||
@@ -239,11 +247,32 @@ describe('normalizedSystemPrompts', () => {
|
||||
})
|
||||
})
|
||||
|
||||
describe('normalizedSystemPromptDeltas', () => {
|
||||
it('extracts and normalizes well-formed system edits', () => {
|
||||
const log = [
|
||||
'{"type":"request/header-delta","data":{"system":{"keepStart":1,"keepEnd":0,"insert":["work in /w"]}}}',
|
||||
'{"type":"request/header-delta","data":{"tools":{"replace":[]}}}',
|
||||
'{"type":"request/header-delta","data":{"system":{"keepStart":"1","keepEnd":0,"insert":[]}}}',
|
||||
'{"type":"request/header-delta","data":{"system":{"keepStart":1,"keepEnd":0,"insert":[null]}}}',
|
||||
'',
|
||||
].join('\n')
|
||||
expect(normalizedSystemPromptDeltas(log, { sessionIds: [], cwd: '/w' })).toEqual([
|
||||
{ keepStart: 1, keepEnd: 0, insert: ['work in {{cwd}}'] },
|
||||
])
|
||||
})
|
||||
})
|
||||
|
||||
describe('formatSystemPromptSnapshot', () => {
|
||||
it('adds a missing terminal newline without changing an existing one', () => {
|
||||
expect(formatSystemPromptSnapshot('prompt')).toBe('prompt\n')
|
||||
expect(formatSystemPromptSnapshot('prompt\n')).toBe('prompt\n')
|
||||
})
|
||||
|
||||
it('renders readable system-prompt delta sections', () => {
|
||||
expect(formatSystemPromptSnapshot('prompt', [
|
||||
{ keepStart: 1, keepEnd: 0, insert: ['new', 'lines'] },
|
||||
])).toBe('prompt\n\n<!-- request/header-delta 1: keepStart=1, keepEnd=0 -->\n\nnew\nlines\n')
|
||||
})
|
||||
})
|
||||
|
||||
describe('headerDeltaCount', () => {
|
||||
|
||||
Reference in New Issue
Block a user