Merge remote-tracking branch 'origin/master' into jsonl-packed-chunk-rows
Conflicts: the four generated catalog docs (regenerated over merged sources), session index.ts exports (keep chunk-rows exports + master's SessionSurface re-export), stdio/acp demo config schema and persistence wiring (thread packChunks through master's DEFAULT_PERSISTENCE_ROOT/UI shape), stdio README config table, and the jsonl spec import line. The packed-chunk fixture also gains the provenance field master made required on assistant/message.
This commit is contained in:
@@ -6,6 +6,7 @@
|
||||
|
||||
import { mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
|
||||
import { readdirSync } from 'node:fs'
|
||||
import { spawn } from 'node:child_process'
|
||||
import { dirname, join } from 'node:path'
|
||||
import { randomUUID } from 'node:crypto'
|
||||
import { createInterface } from 'node:readline'
|
||||
@@ -24,12 +25,18 @@ interface ScriptedLog {
|
||||
|
||||
/** The whole scripted behavior for one run. Every field defaults to the least surprising choice. */
|
||||
interface Behavior {
|
||||
/** Exit during startup after writing any configured stderr note. */
|
||||
failOnBoot?: boolean
|
||||
/** Reject every `session/new` (exercises the expect-error step without extra dirs). */
|
||||
rejectNewSession?: boolean
|
||||
/** Reject `session/new` only when `additionalDirectories` is non-empty (the real bridge's rule). */
|
||||
rejectExtraDirs?: boolean
|
||||
/** How `session/prompt` settles: a clean response, a JSON-RPC error, or a hang until `session/cancel`. */
|
||||
prompt?: 'respond' | 'error' | 'hang-until-cancel'
|
||||
/** Emit a tool call instead of a message chunk before parking a cancellable prompt. */
|
||||
cancelAtToolCall?: boolean
|
||||
/** Emit the parked tool call's terminal update after answering cancellation. */
|
||||
cancelToolCallUpdate?: boolean
|
||||
/** Before responding to a prompt, send a `session/request_permission` request and echo its outcome as a chunk. */
|
||||
permissionProbe?: boolean
|
||||
/** Echo the `DSH_SNAPSHOT_*` env the harness set as a chunk (spec-side env-plumbing assertions). */
|
||||
@@ -38,6 +45,8 @@ interface Behavior {
|
||||
echoWorkspace?: boolean
|
||||
/** Write a line to stderr on boot (spec-side stderr-capture assertions). */
|
||||
stderrNote?: string
|
||||
/** Let a short-lived descendant retain stdio and emit one final ACP update plus stderr line after this parent exits. */
|
||||
lateInheritedOutput?: boolean
|
||||
/** Session logs to persist on stdin EOF. */
|
||||
logs?: ScriptedLog[]
|
||||
/** Leave a stray FILE directly under the sessions root (harvest must skip it). */
|
||||
@@ -62,6 +71,7 @@ const behavior: Behavior = fixtureFile === ''
|
||||
: JSON.parse(readFileSync(join(dirname(fixtureFile), 'behavior.json'), 'utf8')) as Behavior
|
||||
|
||||
if (behavior.stderrNote !== undefined) process.stderr.write(`${behavior.stderrNote}\n`)
|
||||
if (behavior.failOnBoot === true) process.exit(7)
|
||||
|
||||
let nextOutboundId = 1000
|
||||
let sessionId = ''
|
||||
@@ -120,7 +130,23 @@ async function handlePrompt(id: number | string): Promise<void> {
|
||||
params: { sessionId, update: { sessionUpdate: 'agent_thought_chunk', content: { type: 'text', text: 'mulling' } } },
|
||||
})
|
||||
}
|
||||
chunk('thinking about it')
|
||||
if (behavior.cancelAtToolCall === true) {
|
||||
send({
|
||||
method: 'session/update',
|
||||
params: {
|
||||
sessionId,
|
||||
update: {
|
||||
sessionUpdate: 'tool_call',
|
||||
toolCallId: 'call_fake_1',
|
||||
title: 'fake tool',
|
||||
kind: 'execute',
|
||||
status: 'in_progress',
|
||||
},
|
||||
},
|
||||
})
|
||||
} else {
|
||||
chunk('thinking about it')
|
||||
}
|
||||
if (behavior.echoEnv === true) {
|
||||
chunk(`env:${JSON.stringify({
|
||||
mode: process.env.DSH_SNAPSHOT,
|
||||
@@ -223,6 +249,19 @@ function handleFrame(frame: Record<string, unknown>): void {
|
||||
const parked = parkedPromptId
|
||||
parkedPromptId = null
|
||||
respond(parked, { stopReason: 'cancelled' })
|
||||
if (behavior.cancelToolCallUpdate === true) {
|
||||
send({
|
||||
method: 'session/update',
|
||||
params: {
|
||||
sessionId,
|
||||
update: {
|
||||
sessionUpdate: 'tool_call_update',
|
||||
toolCallId: 'call_fake_1',
|
||||
status: 'failed',
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
}
|
||||
return
|
||||
default:
|
||||
@@ -244,6 +283,24 @@ function flushLogsAndExit(): void {
|
||||
writeFileSync(join(sessionsRoot, 'bucket-noise', 'notes.txt'), 'not a session log\n')
|
||||
}
|
||||
if (behavior.deleteSessionsRoot === true) rmSync(sessionsRoot, { recursive: true, force: true })
|
||||
if (behavior.lateInheritedOutput === true) {
|
||||
const frame = JSON.stringify({
|
||||
jsonrpc: '2.0',
|
||||
method: 'session/update',
|
||||
params: {
|
||||
sessionId,
|
||||
update: {
|
||||
sessionUpdate: 'agent_message_chunk',
|
||||
content: { type: 'text', text: 'late inherited stdout' },
|
||||
},
|
||||
},
|
||||
})
|
||||
const code = [
|
||||
`setTimeout(() => process.stdout.write(${JSON.stringify(`${frame}\n`)}), 50)`,
|
||||
`setTimeout(() => process.stderr.write(${JSON.stringify('late inherited stderr\n')}), 75)`,
|
||||
].join(';')
|
||||
spawn(process.execPath, ['-e', code], { stdio: ['ignore', 1, 2] }).unref()
|
||||
}
|
||||
process.exit(0)
|
||||
}
|
||||
|
||||
|
||||
@@ -8,5 +8,5 @@
|
||||
}
|
||||
}
|
||||
],
|
||||
"deltas": []
|
||||
"changes": []
|
||||
}
|
||||
@@ -5,7 +5,7 @@
|
||||
"lines": [
|
||||
{ "type": "session", "id": "{{SID}}", "createdAt": 100, "cwd": "{{CWD}}" },
|
||||
{ "type": "request/header", "seq": 0, "time": 100, "data": { "header": { "config": { "model": "fake" }, "system": "SYS PROMPT", "tools": [{ "name": "t1", "description": "D1", "parameters": { "type": "object" } }] }, "reason": "initial" } },
|
||||
{ "type": "request/header-delta", "seq": 1, "time": 100, "data": { "system": { "keepStart": 1, "keepEnd": 0, "insert": ["NEW PROMPT LINE"] } } },
|
||||
{ "type": "request/header", "seq": 1, "time": 100, "data": { "header": { "config": { "model": "fake" }, "system": "SYS PROMPT\n\nNEW PROMPT LINE", "tools": [{ "name": "t1", "description": "D1", "parameters": { "type": "object" } }] }, "reason": "change" } },
|
||||
{ "type": "turn/start", "seq": 2, "time": 100, "data": { "turn": 1 } }
|
||||
]
|
||||
}]
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
{"type":"session","id":"12121212-3434-4545-8686-787878787878","createdAt":7,"cwd":"/rec/pin-cwd"}
|
||||
{"type":"request/header","seq":0,"time":7,"data":{"header":{"config":{"model":"fake"},"system":"{{system}}","tools":"{{tools}}"},"reason":"initial"}}
|
||||
{"type":"request/header-delta","seq":1,"time":7,"data":{"system":{"keepStart":1,"keepEnd":0,"insert":["{{system}}"]}}}
|
||||
{"type":"request/header","seq":1,"time":7,"data":{"header":{"config":{"model":"fake"},"system":"{{system}}","tools":"{{tools}}"},"reason":"change"}}
|
||||
{"type":"turn/start","seq":2,"time":7,"data":{"turn":1}}
|
||||
|
||||
7
packages/support/acp-snapshot/tests/fixtures/suite/pin-turn/system-prompt.expected.md
vendored
Normal file
7
packages/support/acp-snapshot/tests/fixtures/suite/pin-turn/system-prompt.expected.md
vendored
Normal file
@@ -0,0 +1,7 @@
|
||||
SYS PROMPT
|
||||
|
||||
<!-- request/header change 1 -->
|
||||
|
||||
SYS PROMPT
|
||||
|
||||
NEW PROMPT LINE
|
||||
@@ -1,5 +0,0 @@
|
||||
SYS PROMPT
|
||||
|
||||
<!-- request/header-delta 1: keepStart=1, keepEnd=0 -->
|
||||
|
||||
NEW PROMPT LINE
|
||||
22
packages/support/acp-snapshot/tests/fixtures/suite/pin-turn/tool-schemas.expected.json
vendored
Normal file
22
packages/support/acp-snapshot/tests/fixtures/suite/pin-turn/tool-schemas.expected.json
vendored
Normal file
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"initial": [
|
||||
{
|
||||
"name": "t1",
|
||||
"description": "D1",
|
||||
"parameters": {
|
||||
"type": "object"
|
||||
}
|
||||
}
|
||||
],
|
||||
"changes": [
|
||||
[
|
||||
{
|
||||
"name": "t1",
|
||||
"description": "D1",
|
||||
"parameters": {
|
||||
"type": "object"
|
||||
}
|
||||
}
|
||||
]
|
||||
]
|
||||
}
|
||||
@@ -1,12 +0,0 @@
|
||||
{
|
||||
"initial": [
|
||||
{
|
||||
"name": "t1",
|
||||
"description": "D1",
|
||||
"parameters": {
|
||||
"type": "object"
|
||||
}
|
||||
}
|
||||
],
|
||||
"deltas": []
|
||||
}
|
||||
@@ -1,23 +1,46 @@
|
||||
import { mkdtemp, rm, writeFile } from 'node:fs/promises'
|
||||
import { once } from 'node:events'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { delimiter, join } from 'node:path'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import { afterAll, describe, expect, it } from 'vitest'
|
||||
import { afterAll, describe, expect, it, vi } from 'vitest'
|
||||
import { PROTOCOL_VERSION } from '@agentclientprotocol/sdk'
|
||||
import { runScenario, type AgentUnderTest, type InputStep } from '../src/harness.ts'
|
||||
import { launchAcpTestAgent } from '../src/launcher.ts'
|
||||
|
||||
const fsControl = vi.hoisted(() => ({ cleanupFailure: undefined as Error | undefined }))
|
||||
|
||||
vi.mock('node:fs/promises', async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import('node:fs/promises')>()
|
||||
return {
|
||||
...actual,
|
||||
async rm(...args: Parameters<typeof actual.rm>): Promise<void> {
|
||||
if (String(args[0]).includes('acp-snap-cwd-') && fsControl.cleanupFailure !== undefined) {
|
||||
const failure = fsControl.cleanupFailure
|
||||
fsControl.cleanupFailure = undefined
|
||||
await actual.rm(...args)
|
||||
throw failure
|
||||
}
|
||||
await actual.rm(...args)
|
||||
},
|
||||
}
|
||||
})
|
||||
|
||||
/**
|
||||
* Unit tests for the subprocess harness, driven through the REAL spawn path
|
||||
* (tsx loader, temp cwd, env plumbing) against the scripted fake ACP bin in
|
||||
* (mode-aware launcher, temp cwd, env plumbing) against the scripted fake ACP bin in
|
||||
* ./fixtures/fake-acp-agent.ts. Each case writes a `behavior.json` next to a
|
||||
* throwaway fixture path; the fake bin echoes observable facts (env, seeded
|
||||
* workspace, permission outcomes) into `agent_message_chunk` text, so the
|
||||
* assertions read plain `rawStdout`.
|
||||
*/
|
||||
|
||||
const fakeAgent = fileURLToPath(new URL('./fixtures/fake-acp-agent.ts', import.meta.url))
|
||||
const AGENT: AgentUnderTest = {
|
||||
binScript: fileURLToPath(new URL('./fixtures/fake-acp-agent.ts', import.meta.url)),
|
||||
binScript: fakeAgent,
|
||||
libBinScript: fakeAgent,
|
||||
// The fake bin ignores its config argv; any real path documents the shape.
|
||||
configPath: fileURLToPath(new URL('./fixtures/fake-acp-agent.ts', import.meta.url)),
|
||||
configPath: fakeAgent,
|
||||
tsconfigPath: fileURLToPath(new URL('../../../../tsconfig.json', import.meta.url)),
|
||||
}
|
||||
|
||||
@@ -38,6 +61,206 @@ async function scenario(behavior: object): Promise<{ dir: string; fixtureFile: s
|
||||
const boot: InputStep[] = [{ op: 'initialize' }, { op: 'newSession' }]
|
||||
|
||||
describe('runScenario', () => {
|
||||
it('surfaces an asynchronous child spawn failure through startup and close', async () => {
|
||||
const { dir } = await scenario({})
|
||||
const launched = launchAcpTestAgent({ agent: AGENT, cwd: join(dir, 'missing') })
|
||||
let stdioClosed = false
|
||||
let clientClosed = false
|
||||
launched.child.once('close', () => { stdioClosed = true })
|
||||
void launched.client.closed.then(
|
||||
() => { clientClosed = true },
|
||||
() => { clientClosed = true },
|
||||
)
|
||||
await expect(launched.spawned).rejects.toMatchObject({ code: 'ENOENT' })
|
||||
await expect(launched.close()).rejects.toMatchObject({ code: 'ENOENT' })
|
||||
expect(stdioClosed).toBe(true)
|
||||
expect(clientClosed).toBe(true)
|
||||
})
|
||||
|
||||
it('centralizes ACP boot, captures, updates, fail-closed permissions, and shutdown', { timeout: 20_000 }, async () => {
|
||||
const { dir, fixtureFile } = await scenario({ permissionProbe: true, echoEnv: true, stderrNote: 'launcher stderr' })
|
||||
const sessionsRoot = await mkdtemp(join(tmpdir(), 'acp-launcher-sessions-'))
|
||||
tempDirs.push(sessionsRoot)
|
||||
const launched = launchAcpTestAgent({
|
||||
agent: AGENT,
|
||||
cwd: dir,
|
||||
configPath: AGENT.configPath,
|
||||
env: {
|
||||
DSH_SNAPSHOT: 'replay',
|
||||
DSH_SNAPSHOT_FILE: fixtureFile,
|
||||
DSH_SNAPSHOT_SESSIONS_ROOT: sessionsRoot,
|
||||
},
|
||||
})
|
||||
await launched.client.initialize({ protocolVersion: PROTOCOL_VERSION, clientCapabilities: {} })
|
||||
const { sessionId } = await launched.client.newSession({ cwd: dir, mcpServers: [] })
|
||||
const nextChunk = launched.waitForUpdate(update => update.sessionUpdate === 'agent_message_chunk')
|
||||
const predicateFailure = new Error('predicate failed')
|
||||
const failedPredicate = launched.waitForUpdate(() => { throw predicateFailure })
|
||||
.catch((error: unknown): unknown => error)
|
||||
await launched.client.prompt({ sessionId, prompt: [{ type: 'text', text: 'go' }] })
|
||||
expect(await failedPredicate).toBe(predicateFailure)
|
||||
expect((await nextChunk).sessionUpdate).toBe('agent_message_chunk')
|
||||
expect(launched.updates.some(update => update.sessionUpdate === 'agent_message_chunk')).toBe(true)
|
||||
expect(launched.rawStdout()).toContain('permission:{\\"outcome\\":\\"cancelled\\"}')
|
||||
expect(launched.stderr()).toContain('launcher stderr')
|
||||
const unmatched = expect(launched.waitForUpdate(() => false)).rejects.toThrow(/update stream closed/)
|
||||
await launched.close()
|
||||
await unmatched
|
||||
await expect(launched.waitForUpdate(() => true)).rejects.toThrow(/update stream closed/)
|
||||
await launched.close('SIGKILL')
|
||||
|
||||
// The minimal shape needs no environment or config override.
|
||||
const minimal = launchAcpTestAgent({ agent: AGENT, cwd: dir })
|
||||
await minimal.client.initialize({ protocolVersion: PROTOCOL_VERSION, clientCapabilities: {} })
|
||||
const childFailure = new Error('child process failed')
|
||||
let exited = false
|
||||
minimal.child.once('exit', () => { exited = true })
|
||||
minimal.child.emit('error', childFailure)
|
||||
await expect(minimal.close('SIGTERM')).rejects.toBe(childFailure)
|
||||
// close rejects only after the fallback SIGKILL has produced an exit edge.
|
||||
expect(exited).toBe(true)
|
||||
})
|
||||
|
||||
it('waits for inherited stdio and buffered ACP parsing after the parent exits', { timeout: 20_000 }, async () => {
|
||||
const { dir, fixtureFile } = await scenario({ lateInheritedOutput: true })
|
||||
const launched = launchAcpTestAgent({
|
||||
agent: AGENT,
|
||||
cwd: dir,
|
||||
env: { DSH_SNAPSHOT_FILE: fixtureFile },
|
||||
})
|
||||
await launched.client.initialize({ protocolVersion: PROTOCOL_VERSION, clientCapabilities: {} })
|
||||
await launched.client.newSession({ cwd: dir, mcpServers: [] })
|
||||
const lateUpdate = launched.waitForUpdate(update =>
|
||||
update.sessionUpdate === 'agent_message_chunk'
|
||||
&& update.content.type === 'text'
|
||||
&& update.content.text === 'late inherited stdout')
|
||||
|
||||
await launched.close()
|
||||
|
||||
await expect(lateUpdate).resolves.toMatchObject({ sessionUpdate: 'agent_message_chunk' })
|
||||
expect(launched.rawStdout()).toContain('late inherited stdout')
|
||||
expect(launched.stderr()).toContain('late inherited stderr')
|
||||
})
|
||||
|
||||
it('rejects promptly when fallback termination is refused', async () => {
|
||||
const { dir } = await scenario({})
|
||||
const launched = launchAcpTestAgent({ agent: AGENT, cwd: dir })
|
||||
await launched.spawned
|
||||
|
||||
const childFailure = Object.assign(new Error('signal refused'), { code: 'EPERM' })
|
||||
const originalKill = launched.child.kill.bind(launched.child)
|
||||
const kill = vi.spyOn(launched.child, 'kill').mockReturnValue(false)
|
||||
const closed = new Promise<void>(resolve => launched.child.once('close', () => { resolve() }))
|
||||
try {
|
||||
launched.child.emit('error', childFailure)
|
||||
const rejection = await launched.close('SIGTERM').catch((error: unknown): unknown => error)
|
||||
expect(rejection).toBeInstanceOf(AggregateError)
|
||||
expect(rejection).toMatchObject({
|
||||
message: 'ACP test agent failed and fallback termination was refused',
|
||||
errors: [
|
||||
childFailure,
|
||||
expect.objectContaining({ message: 'Fallback SIGKILL was not accepted by the child process' }),
|
||||
],
|
||||
})
|
||||
expect(kill).toHaveBeenNthCalledWith(1, 'SIGTERM')
|
||||
expect(kill).toHaveBeenNthCalledWith(2, 'SIGKILL')
|
||||
} finally {
|
||||
kill.mockRestore()
|
||||
originalKill('SIGKILL')
|
||||
await closed
|
||||
}
|
||||
})
|
||||
|
||||
it('rejects promptly when fallback termination emits an error', async () => {
|
||||
const { dir } = await scenario({})
|
||||
const launched = launchAcpTestAgent({ agent: AGENT, cwd: dir })
|
||||
await launched.spawned
|
||||
|
||||
const childFailure = Object.assign(new Error('signal refused'), { code: 'EPERM' })
|
||||
const fallbackFailure = Object.assign(new Error('fallback signal refused'), { code: 'EPERM' })
|
||||
const originalKill = launched.child.kill.bind(launched.child)
|
||||
const kill = vi.spyOn(launched.child, 'kill').mockImplementation((signal) => {
|
||||
if (signal === 'SIGKILL') queueMicrotask(() => launched.child.emit('error', fallbackFailure))
|
||||
return signal === 'SIGKILL'
|
||||
})
|
||||
const closed = new Promise<void>(resolve => launched.child.once('close', () => { resolve() }))
|
||||
try {
|
||||
launched.child.emit('error', childFailure)
|
||||
const rejection = await launched.close('SIGTERM').catch((error: unknown): unknown => error)
|
||||
expect(rejection).toBeInstanceOf(AggregateError)
|
||||
expect(rejection).toMatchObject({
|
||||
message: 'ACP test agent failed and fallback termination was refused',
|
||||
errors: [childFailure, fallbackFailure],
|
||||
})
|
||||
expect(kill).toHaveBeenNthCalledWith(1, 'SIGTERM')
|
||||
expect(kill).toHaveBeenNthCalledWith(2, 'SIGKILL')
|
||||
} finally {
|
||||
kill.mockRestore()
|
||||
originalKill('SIGKILL')
|
||||
await closed
|
||||
}
|
||||
})
|
||||
|
||||
it('waits for in-flight client callbacks after the ACP stream closes', { timeout: 20_000 }, async () => {
|
||||
const { dir, fixtureFile } = await scenario({ permissionProbe: true })
|
||||
let releasePermission: (() => void) | undefined
|
||||
const permissionReleased = new Promise<void>((resolve) => { releasePermission = resolve })
|
||||
let markPermissionStarted: (() => void) | undefined
|
||||
const permissionStarted = new Promise<void>((resolve) => { markPermissionStarted = resolve })
|
||||
let permissionFinished = false
|
||||
const launched = launchAcpTestAgent({
|
||||
agent: AGENT,
|
||||
cwd: dir,
|
||||
env: { DSH_SNAPSHOT_FILE: fixtureFile },
|
||||
async requestPermission() {
|
||||
markPermissionStarted?.()
|
||||
await permissionReleased
|
||||
permissionFinished = true
|
||||
return { outcome: { outcome: 'cancelled' } }
|
||||
},
|
||||
})
|
||||
await launched.client.initialize({ protocolVersion: PROTOCOL_VERSION, clientCapabilities: {} })
|
||||
const { sessionId } = await launched.client.newSession({ cwd: dir, mcpServers: [] })
|
||||
void launched.client.prompt({ sessionId, prompt: [{ type: 'text', text: 'go' }] }).catch(() => undefined)
|
||||
await permissionStarted
|
||||
|
||||
const childClosed = once(launched.child, 'close')
|
||||
let closeSettled = false
|
||||
const closing = launched.close('SIGKILL').then(() => { closeSettled = true })
|
||||
await childClosed
|
||||
await launched.client.closed
|
||||
expect(closeSettled).toBe(false)
|
||||
|
||||
releasePermission?.()
|
||||
await closing
|
||||
expect(permissionFinished).toBe(true)
|
||||
})
|
||||
|
||||
it('includes agent stderr when the ACP connection closes during startup', { timeout: 20_000 }, async () => {
|
||||
const { fixtureFile } = await scenario({ failOnBoot: true, stderrNote: 'fake agent requested startup failure' })
|
||||
await expect(runScenario(
|
||||
{ steps: [{ op: 'initialize' }] },
|
||||
{ agent: AGENT, mode: 'replay', fixtureFile },
|
||||
)).rejects.toThrow(/agent stderr:\nfake agent requested startup failure/)
|
||||
})
|
||||
|
||||
it('preserves launch-resolution errors when no child process exists', async () => {
|
||||
const { dir, fixtureFile } = await scenario({})
|
||||
vi.stubEnv('DSH_EXAMPLE_MODE', 'lib')
|
||||
try {
|
||||
await expect(runScenario(
|
||||
{ steps: [] },
|
||||
{
|
||||
agent: { ...AGENT, binScript: join(dir, 'outside-src.ts'), libBinScript: undefined },
|
||||
mode: 'replay',
|
||||
fixtureFile,
|
||||
},
|
||||
)).rejects.toThrow(/expected a "\/src\/" segment/)
|
||||
} finally {
|
||||
vi.unstubAllEnvs()
|
||||
}
|
||||
})
|
||||
|
||||
it('drives a full turn: initialize (terminal caps), session, prompt, permission stub, harvest', { timeout: 20_000 }, async () => {
|
||||
const { fixtureFile } = await scenario({
|
||||
permissionProbe: true,
|
||||
@@ -111,6 +334,28 @@ describe('runScenario', () => {
|
||||
expect(result.rawStdout.indexOf('thinking about it')).toBeLessThan(result.rawStdout.indexOf('cancelled'))
|
||||
})
|
||||
|
||||
it('promptAndCancel can bracket cancellation with tool-call updates', { timeout: 20_000 }, async () => {
|
||||
const { fixtureFile } = await scenario({
|
||||
prompt: 'hang-until-cancel',
|
||||
cancelAtToolCall: true,
|
||||
cancelToolCallUpdate: true,
|
||||
})
|
||||
const result = await runScenario(
|
||||
{
|
||||
steps: [...boot, {
|
||||
op: 'promptAndCancel',
|
||||
text: 'hang',
|
||||
afterUpdate: 'tool_call',
|
||||
waitForToolCallUpdate: 'call_fake_1',
|
||||
}],
|
||||
},
|
||||
{ agent: AGENT, mode: 'replay', fixtureFile },
|
||||
)
|
||||
expect(result.rawStdout).toContain('"sessionUpdate":"tool_call"')
|
||||
expect(result.rawStdout.indexOf('"sessionUpdate":"tool_call"')).toBeLessThan(result.rawStdout.indexOf('cancelled'))
|
||||
expect(result.rawStdout.indexOf('cancelled')).toBeLessThan(result.rawStdout.indexOf('"sessionUpdate":"tool_call_update"'))
|
||||
})
|
||||
|
||||
it('promptExpectError swallows a model-error response as the expected outcome', { timeout: 20_000 }, async () => {
|
||||
const { fixtureFile } = await scenario({ prompt: 'error' })
|
||||
const result = await runScenario(
|
||||
@@ -128,6 +373,39 @@ describe('runScenario', () => {
|
||||
)).rejects.toThrow(/expected the prompt to fail/)
|
||||
})
|
||||
|
||||
it('reports scenario and cleanup failures together', { timeout: 20_000 }, async () => {
|
||||
const { fixtureFile } = await scenario({ prompt: 'respond' })
|
||||
const cleanupFailure = new Error('cleanup failed')
|
||||
fsControl.cleanupFailure = cleanupFailure
|
||||
|
||||
const failure = await runScenario(
|
||||
{ steps: [...boot, { op: 'promptExpectError', text: 'fine' }] },
|
||||
{ agent: AGENT, mode: 'replay', fixtureFile },
|
||||
).catch((error: unknown): unknown => error)
|
||||
|
||||
expect(failure).toBeInstanceOf(AggregateError)
|
||||
const failures = (failure as AggregateError).errors as unknown[]
|
||||
expect(failures).toHaveLength(2)
|
||||
expect(failures[0]).toBeInstanceOf(Error)
|
||||
expect((failures[0] as Error).message).toMatch(/expected the prompt to fail/)
|
||||
expect(failures[1]).toBe(cleanupFailure)
|
||||
})
|
||||
|
||||
it('reports cleanup failure after an otherwise successful scenario', { timeout: 20_000 }, async () => {
|
||||
const { fixtureFile } = await scenario({})
|
||||
const cleanupFailure = new Error('cleanup failed')
|
||||
fsControl.cleanupFailure = cleanupFailure
|
||||
|
||||
const failure = await runScenario(
|
||||
{ steps: boot },
|
||||
{ agent: AGENT, mode: 'replay', fixtureFile },
|
||||
).catch((error: unknown): unknown => error)
|
||||
|
||||
expect(failure).toBeInstanceOf(AggregateError)
|
||||
expect((failure as AggregateError).message).toBe('snapshot cleanup failed')
|
||||
expect((failure as AggregateError).errors as unknown[]).toEqual([cleanupFailure])
|
||||
})
|
||||
|
||||
it('newSessionExpectError swallows the rejection, with and without extra dirs', { timeout: 20_000 }, async () => {
|
||||
const { fixtureFile } = await scenario({ rejectExtraDirs: true })
|
||||
const result = await runScenario(
|
||||
|
||||
@@ -93,6 +93,52 @@ describe('normalizeSessionLog', () => {
|
||||
expect(out).not.toContain(ctx.cwd)
|
||||
})
|
||||
|
||||
it('scrubs random local spill paths under the snapshot cwd', () => {
|
||||
const ev = JSON.stringify({
|
||||
type: 'tool/result', seq: 2, time: 5,
|
||||
data: {
|
||||
content: [{
|
||||
type: 'text',
|
||||
text: `Full formatted result stored at: ${ctx.cwd}/.spill/session-c22bc3f1d2af/8a7b6c5d4e3f-bash.txt. Use read with offset/limit, or grep this path to search within it.`,
|
||||
}],
|
||||
},
|
||||
})
|
||||
const out = normalizeSessionLog(`${header({ cwd: ctx.cwd })}\n${ev}\n`, ctx)
|
||||
expect(out).toContain('{{spillLocator:bash.txt}}')
|
||||
expect(out).not.toContain('session-c22bc3f1d2af')
|
||||
expect(out).not.toContain('8a7b6c5d4e3f')
|
||||
})
|
||||
|
||||
it('scrubs macOS /private aliases for local spill paths', () => {
|
||||
const ev = JSON.stringify({
|
||||
type: 'tool/result', seq: 2, time: 5,
|
||||
data: {
|
||||
content: [{
|
||||
type: 'text',
|
||||
text: `Full formatted result stored at: /private${ctx.cwd}/.spill/session-c22bc3f1d2af/8a7b6c5d4e3f-bash.txt. Use read with offset/limit, or grep this path to search within it.`,
|
||||
}],
|
||||
},
|
||||
})
|
||||
const out = normalizeSessionLog(`${header({ cwd: ctx.cwd })}\n${ev}\n`, ctx)
|
||||
expect(out).toContain('{{spillLocator:bash.txt}}')
|
||||
expect(out).not.toContain('/private{{spillLocator')
|
||||
})
|
||||
|
||||
it('scrubs fixed snapshot spill paths', () => {
|
||||
const ev = JSON.stringify({
|
||||
type: 'tool/result', seq: 2, time: 5,
|
||||
data: {
|
||||
content: [{
|
||||
type: 'text',
|
||||
text: 'Full formatted result stored at: /tmp/dsh-acp-snapshot-spill/session-c22bc3f1d2af/8a7b6c5d4e3f-bash.txt. Use read with offset/limit, or grep this path to search within it.',
|
||||
}],
|
||||
},
|
||||
})
|
||||
const out = normalizeSessionLog(`${header({ cwd: ctx.cwd })}\n${ev}\n`, ctx)
|
||||
expect(out).toContain('{{spillLocator:bash.txt}}')
|
||||
expect(out).not.toContain('/tmp/dsh-acp-snapshot-spill')
|
||||
})
|
||||
|
||||
it('scrubs the session id in the header', () => {
|
||||
const out = normalizeSessionLog(`${header({ id: ctx.sessionIds[0] })}\n`, ctx)
|
||||
expect(out).toContain('{{sessionId}}')
|
||||
@@ -202,89 +248,19 @@ describe('scrubRequestHeaders', () => {
|
||||
expect(scrubRequestHeaders(`${headerLine}\n${odd}\n`)).toContain('"messagePrefix":"weird"')
|
||||
})
|
||||
|
||||
it('scrubs a header-delta prefix replacement to one token per message', () => {
|
||||
const delta = JSON.stringify({
|
||||
type: 'request/header-delta', seq: 8, time: 9,
|
||||
data: { messagePrefix: [{ role: 'user', content: [{ type: 'text', text: 'leaked opener' }] }] },
|
||||
})
|
||||
const out = scrubRequestHeaders(`${headerLine}\n${delta}\n`)
|
||||
expect(out).toContain('"messagePrefix":["{{messagePrefix}}"]')
|
||||
expect(out).not.toContain('leaked opener')
|
||||
// The empty-array transition-to-absence stays a structural fact.
|
||||
const toNone = JSON.stringify({ type: 'request/header-delta', seq: 9, time: 9, data: { messagePrefix: [] } })
|
||||
expect(scrubRequestHeaders(`${headerLine}\n${toNone}\n`)).toContain('"messagePrefix":[]')
|
||||
})
|
||||
|
||||
it('leaves a delta with no scrubbable payload byte-identical (config-only, or non-array shapes)', () => {
|
||||
const configOnly = JSON.stringify({ type: 'request/header-delta', seq: 8, time: 9, data: { config: { model: 'm2' } } })
|
||||
const oddShapes = JSON.stringify({ type: 'request/header-delta', seq: 9, time: 9, data: { system: { insert: 'not-an-array' }, tools: null } })
|
||||
it('leaves malformed headers with no scrubbable payload byte-identical', () => {
|
||||
const headerless = JSON.stringify({ type: 'request/header', seq: 10, time: 9, data: { reason: 'initial' } })
|
||||
const nullData = JSON.stringify({ type: 'request/header', seq: 11, time: 9, data: null })
|
||||
const raw = `${headerLine}\n${configOnly}\n${oddShapes}\n${headerless}\n${nullData}\n`
|
||||
const raw = `${headerLine}\n${headerless}\n${nullData}\n`
|
||||
expect(scrubRequestHeaders(raw)).toBe(raw)
|
||||
})
|
||||
|
||||
it('scrubs a one-sided tools delta and passes non-object schema entries through', () => {
|
||||
const addedOnly = JSON.stringify({
|
||||
type: 'request/header-delta', seq: 8, time: 9,
|
||||
data: { tools: { added: [null, 'weird', { name: 'x', description: 'D' }] } },
|
||||
})
|
||||
const out = scrubRequestHeaders(`${headerLine}\n${addedOnly}\n`)
|
||||
// Non-object entries survive untouched; the object entry keeps only name.
|
||||
expect(out).toContain('"added":[null,"weird",{"name":"x","description":"{{tools}}"}]')
|
||||
const changedOnly = JSON.stringify({
|
||||
type: 'request/header-delta', seq: 8, time: 9,
|
||||
data: { tools: { changed: [{ name: 'y', parameters: {} }] } },
|
||||
})
|
||||
expect(scrubRequestHeaders(`${headerLine}\n${changedOnly}\n`))
|
||||
.toContain('"changed":[{"name":"y","parameters":"{{tools}}"}]')
|
||||
})
|
||||
|
||||
it('scrubs a header-delta system payload but keeps its line positions and arity', () => {
|
||||
const delta = JSON.stringify({
|
||||
type: 'request/header-delta', seq: 8, time: 9,
|
||||
data: { system: { keepStart: 1, keepEnd: 4, insert: ['leaked prompt line', 'second line'] }, config: { model: 'm2' } },
|
||||
})
|
||||
const out = scrubRequestHeaders(`${headerLine}\n${delta}\n`)
|
||||
// One token PER inserted line: the edit's position AND extent survive.
|
||||
expect(out).toContain('"insert":["{{system}}","{{system}}"]')
|
||||
expect(out).toContain('"keepStart":1')
|
||||
expect(out).toContain('"keepEnd":4')
|
||||
expect(out).toContain('"config":{"model":"m2"}')
|
||||
expect(out).not.toContain('leaked prompt line')
|
||||
expect(out).not.toContain('{{tools}}') // no tools delta → none invented
|
||||
})
|
||||
|
||||
it('scrubs a header-delta tools payload but keeps the added/removed/changed names', () => {
|
||||
const delta = JSON.stringify({
|
||||
type: 'request/header-delta', seq: 8, time: 9,
|
||||
data: {
|
||||
tools: {
|
||||
added: [{ name: 'grep', description: 'Search files.', parameters: { type: 'object' } }],
|
||||
removed: ['bash_kill'],
|
||||
changed: [{ name: 'read', description: 'Read v2.', parameters: { type: 'object' } }],
|
||||
},
|
||||
},
|
||||
})
|
||||
const out = scrubRequestHeaders(`${headerLine}\n${delta}\n`)
|
||||
// WHICH tools changed is behavior and survives; their bulk does not.
|
||||
expect(out).toContain('"added":[{"name":"grep","description":"{{tools}}","parameters":"{{tools}}"}]')
|
||||
expect(out).toContain('"removed":["bash_kill"]')
|
||||
expect(out).toContain('"changed":[{"name":"read","description":"{{tools}}","parameters":"{{tools}}"}]')
|
||||
expect(out).not.toContain('Search files')
|
||||
expect(out).not.toContain('Read v2')
|
||||
})
|
||||
|
||||
it('passes every other line through byte-for-byte and is idempotent', () => {
|
||||
const other = JSON.stringify({ type: 'assistant/chunk', seq: 4, time: 9, data: { turn: 1, step: 1, chunk: { type: 'text-delta', index: 0, text: 'hi' } } })
|
||||
const delta = JSON.stringify({
|
||||
type: 'request/header-delta', seq: 8, time: 9,
|
||||
data: { system: { keepStart: 0, keepEnd: 0, insert: ['x'] }, tools: { added: [{ name: 't', description: 'd', parameters: {} }], removed: [], changed: [] } },
|
||||
})
|
||||
const raw = `${headerLine}\n${headerEvent({ config: { model: 'm' }, system: 's', tools: [] })}\n${delta}\n${other}\n`
|
||||
const raw = `${headerLine}\n${headerEvent({ config: { model: 'm' }, system: 's', tools: [] })}\n${other}\n`
|
||||
const once = scrubRequestHeaders(raw)
|
||||
expect(once.split('\n')[0]).toBe(headerLine)
|
||||
expect(once.split('\n')[3]).toBe(other)
|
||||
expect(once.split('\n')[2]).toBe(other)
|
||||
expect(scrubRequestHeaders(once)).toBe(once)
|
||||
})
|
||||
})
|
||||
@@ -302,12 +278,15 @@ describe('scrubSystemPrompts', () => {
|
||||
reason: 'initial',
|
||||
},
|
||||
})
|
||||
const delta = JSON.stringify({
|
||||
type: 'request/header-delta', seq: 2, time: 3,
|
||||
const changed = JSON.stringify({
|
||||
type: 'request/header', seq: 2, time: 3,
|
||||
data: {
|
||||
system: { keepStart: 1, keepEnd: 2, insert: ['new prompt line'] },
|
||||
tools: { changed: [{ name: 'read', description: 'changed schema' }] },
|
||||
messagePrefix: [{ role: 'user', content: [{ type: 'text', text: 'changed prefix' }] }],
|
||||
header: {
|
||||
system: 'new prompt',
|
||||
tools: [{ name: 'read', description: 'changed schema' }],
|
||||
messagePrefix: [{ role: 'user', content: [{ type: 'text', text: 'changed prefix' }] }],
|
||||
},
|
||||
reason: 'change',
|
||||
},
|
||||
})
|
||||
const toolsOnly = JSON.stringify({
|
||||
@@ -315,11 +294,10 @@ describe('scrubSystemPrompts', () => {
|
||||
data: { header: { tools: [{ name: 'read', description: 'schema only' }] }, reason: 'resume' },
|
||||
})
|
||||
|
||||
const out = scrubSystemPrompts(`${header}\n${delta}\n${toolsOnly}\n`)
|
||||
const out = scrubSystemPrompts(`${header}\n${changed}\n${toolsOnly}\n`)
|
||||
expect(out).toContain('"system":"{{system}}"')
|
||||
expect(out).toContain('"insert":["{{system}}"]')
|
||||
expect(out).not.toContain('full prompt')
|
||||
expect(out).not.toContain('new prompt line')
|
||||
expect(out).not.toContain('new prompt')
|
||||
expect(out).toContain('full schema')
|
||||
expect(out).toContain('full prefix')
|
||||
expect(out).toContain('changed schema')
|
||||
@@ -342,12 +320,15 @@ describe('scrubToolSchemas', () => {
|
||||
reason: 'initial',
|
||||
},
|
||||
})
|
||||
const delta = JSON.stringify({
|
||||
type: 'request/header-delta', seq: 2, time: 3,
|
||||
const changed = JSON.stringify({
|
||||
type: 'request/header', seq: 2, time: 3,
|
||||
data: {
|
||||
system: { keepStart: 1, keepEnd: 2, insert: ['new prompt line'] },
|
||||
tools: { added: [{ name: 'grep', description: 'new schema' }], changed: [{ name: 'read', description: 'changed schema' }] },
|
||||
messagePrefix: [{ role: 'user', content: [{ type: 'text', text: 'changed prefix' }] }],
|
||||
header: {
|
||||
system: 'new prompt',
|
||||
tools: [{ name: 'grep', description: 'new schema' }],
|
||||
messagePrefix: [{ role: 'user', content: [{ type: 'text', text: 'changed prefix' }] }],
|
||||
},
|
||||
reason: 'change',
|
||||
},
|
||||
})
|
||||
const systemOnly = JSON.stringify({
|
||||
@@ -355,15 +336,12 @@ describe('scrubToolSchemas', () => {
|
||||
data: { header: { system: 'prompt only' }, reason: 'resume' },
|
||||
})
|
||||
|
||||
const out = scrubToolSchemas(`${header}\n${delta}\n${systemOnly}\n`)
|
||||
expect(out).toContain('"tools":"{{tools}}"')
|
||||
expect(out).toContain('"added":[{"name":"grep","description":"{{tools}}"}]')
|
||||
expect(out).toContain('"changed":[{"name":"read","description":"{{tools}}"}]')
|
||||
const out = scrubToolSchemas(`${header}\n${changed}\n${systemOnly}\n`)
|
||||
expect(out.match(/"tools":"{{tools}}"/g)).toHaveLength(2)
|
||||
expect(out).not.toContain('full schema')
|
||||
expect(out).not.toContain('new schema')
|
||||
expect(out).not.toContain('changed schema')
|
||||
expect(out).toContain('full prompt')
|
||||
expect(out).toContain('new prompt line')
|
||||
expect(out).toContain('new prompt')
|
||||
expect(out).toContain('full prefix')
|
||||
expect(out).toContain('changed prefix')
|
||||
expect(out.split('\n')[2]).toBe(systemOnly)
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { cpSync, mkdtempSync, readFileSync, writeFileSync } from 'node:fs'
|
||||
import { cpSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
|
||||
import { rm } from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
@@ -6,18 +6,16 @@ import { fileURLToPath } from 'node:url'
|
||||
import { afterAll, describe, expect, it } from 'vitest'
|
||||
import { defineAcpSnapshotSuite, type HarvestedLog, type Scenario } from '../src/index.ts'
|
||||
import {
|
||||
childFixturePaths,
|
||||
fixtureContext,
|
||||
formatSystemPromptSnapshot,
|
||||
headerChangeCount,
|
||||
formatToolSchemasSnapshot,
|
||||
headerDeltaCount,
|
||||
normalizedHeaders,
|
||||
normalizedSystemPromptDeltas,
|
||||
normalizedSystemPrompts,
|
||||
normalizedToolSchemaDeltas,
|
||||
normalizedToolSchemas,
|
||||
parseToolSchemasSnapshot,
|
||||
refreshFixtureReplacements,
|
||||
sessionFixtureNames,
|
||||
restorePinnedToolSchemas,
|
||||
stabilizeRefreshLog,
|
||||
unknownToolCallIds,
|
||||
@@ -26,7 +24,7 @@ import {
|
||||
/**
|
||||
* Unit tests for the suite factory, by running it: two synthetic suites over the scripted fake
|
||||
* ACP bin (./fixtures/fake-acp-agent.ts) register real describe/it trees at collection time,
|
||||
* so every factory path — golden and log compares, the per-suite header pin and its uniformity
|
||||
* so every factory path — expected-output and log comparisons, the per-suite header pin and its uniformity
|
||||
* guard, record-mode fixture write-back, skip semantics, and the fixture guard block —
|
||||
* executes as an ordinary green test.
|
||||
*
|
||||
@@ -34,9 +32,11 @@ import {
|
||||
* spec once with `ACP_SNAPSHOT_SPEC_BOOTSTRAP=1`, then review and commit the resulting tree.
|
||||
*/
|
||||
|
||||
const fakeAgent = fileURLToPath(new URL('./fixtures/fake-acp-agent.ts', import.meta.url))
|
||||
const AGENT = {
|
||||
binScript: fileURLToPath(new URL('./fixtures/fake-acp-agent.ts', import.meta.url)),
|
||||
configPath: fileURLToPath(new URL('./fixtures/fake-acp-agent.ts', import.meta.url)),
|
||||
binScript: fakeAgent,
|
||||
libBinScript: fakeAgent,
|
||||
configPath: fakeAgent,
|
||||
tsconfigPath: fileURLToPath(new URL('../../../../tsconfig.json', import.meta.url)),
|
||||
}
|
||||
|
||||
@@ -45,8 +45,8 @@ const RECORD_SRC = fileURLToPath(new URL('./fixtures/record-suite', import.meta.
|
||||
|
||||
// Replay pins explicit header classes; recording covers the default fallback.
|
||||
const REPLAY_SCENARIOS: Scenario[] = [
|
||||
{ name: 'pin-turn', hasModelTurn: true, recorded: true, pinsHeader: true, expectedHeaderDeltas: 1, headerClass: 'main' },
|
||||
{ name: 'plain-turn', hasModelTurn: true, recorded: true, childSessions: 1, headerClass: 'main', configPath: AGENT.configPath },
|
||||
{ name: 'pin-turn', hasModelTurn: true, recorded: true, pinsHeader: true, expectedHeaderChanges: 1, headerClass: 'main' },
|
||||
{ name: 'plain-turn', hasModelTurn: true, recorded: true, headerClass: 'main', configPath: AGENT.configPath },
|
||||
{ name: 'no-model', hasModelTurn: false, recorded: false, headerClass: 'main' },
|
||||
{ name: 'blocked-log', hasModelTurn: false, comparesLog: true, recorded: false, headerClass: 'main' },
|
||||
{ name: 'authored-error', hasModelTurn: true, recorded: false, overridden: true, headerClass: 'main' },
|
||||
@@ -54,17 +54,23 @@ const REPLAY_SCENARIOS: Scenario[] = [
|
||||
|
||||
const RECORD_SCENARIOS: Scenario[] = [
|
||||
{ name: 'rec-pin', hasModelTurn: true, recorded: true, pinsHeader: true },
|
||||
{ name: 'rec-child', hasModelTurn: true, recorded: true, childSessions: 1 },
|
||||
{ name: 'rec-child', hasModelTurn: true, recorded: true },
|
||||
// recorded:false in record mode → registered but skipped (never re-recorded).
|
||||
{ name: 'rec-skip', hasModelTurn: true, recorded: false, overridden: true },
|
||||
]
|
||||
|
||||
// Record/refresh modes mutate their snapshots dir, so run them on throwaway
|
||||
// copies — except record's documented bootstrap knob, which regenerates the
|
||||
// committed record fixtures/goldens in place.
|
||||
// committed record fixtures and expected outputs in place.
|
||||
const BOOTSTRAP = process.env.ACP_SNAPSHOT_SPEC_BOOTSTRAP === '1'
|
||||
const recordDir = BOOTSTRAP ? RECORD_SRC : mkdtempSync(join(tmpdir(), 'acp-snap-record-suite-'))
|
||||
if (!BOOTSTRAP) cpSync(RECORD_SRC, recordDir, { recursive: true })
|
||||
if (!BOOTSTRAP) {
|
||||
cpSync(RECORD_SRC, recordDir, { recursive: true })
|
||||
// Record mode owns its output inventory: a new scenario has no primary yet,
|
||||
// while a changed child count can leave old numbered fixtures behind.
|
||||
rmSync(join(recordDir, 'rec-pin', 'session.jsonl'))
|
||||
writeFileSync(join(recordDir, 'rec-child', 'session.2.jsonl'), 'stale child\n')
|
||||
}
|
||||
const refreshDir = mkdtempSync(join(tmpdir(), 'acp-snap-refresh-suite-'))
|
||||
cpSync(REPLAY_DIR, refreshDir, { recursive: true })
|
||||
staleRefreshFixtures(refreshDir)
|
||||
@@ -74,9 +80,9 @@ afterAll(async () => {
|
||||
})
|
||||
|
||||
function staleRefreshFixtures(dir: string): void {
|
||||
writeFileSync(join(dir, 'plain-turn', 'stdout.golden.jsonl'), 'stale stdout\n')
|
||||
writeFileSync(join(dir, 'pin-turn', 'system-prompt.golden.md'), 'STALE PROMPT\n')
|
||||
writeFileSync(join(dir, 'pin-turn', 'tool-schemas.golden.json'), '{"initial":[{"name":"stale"}],"deltas":[]}\n')
|
||||
writeFileSync(join(dir, 'plain-turn', 'stdout.expected.jsonl'), 'stale stdout\n')
|
||||
writeFileSync(join(dir, 'pin-turn', 'system-prompt.expected.md'), 'STALE PROMPT\n')
|
||||
writeFileSync(join(dir, 'pin-turn', 'tool-schemas.expected.json'), '{"initial":[{"name":"stale"}],"changes":[]}\n')
|
||||
|
||||
const plainBehaviorFile = join(dir, 'plain-turn', 'behavior.json')
|
||||
const plainBehavior = JSON.parse(readFileSync(plainBehaviorFile, 'utf8')) as Record<string, unknown>
|
||||
@@ -111,7 +117,7 @@ describe('defineAcpSnapshotSuite: refresh mode', () => {
|
||||
|
||||
describe('defineAcpSnapshotSuite: refresh write-back', () => {
|
||||
it('rewrites stdout and comparable logs from a replay-mode child run', () => {
|
||||
const stdout = readFileSync(join(refreshDir, 'plain-turn', 'stdout.golden.jsonl'), 'utf8')
|
||||
const stdout = readFileSync(join(refreshDir, 'plain-turn', 'stdout.expected.jsonl'), 'utf8')
|
||||
expect(stdout).not.toContain('stale stdout')
|
||||
expect(stdout).toContain('env:{\\"mode\\":\\"replay\\"')
|
||||
expect(stdout).not.toContain('\\"mode\\":\\"refresh\\"')
|
||||
@@ -124,20 +130,29 @@ describe('defineAcpSnapshotSuite: refresh write-back', () => {
|
||||
expect(authored).toContain('"error":"model exploded"')
|
||||
expect(authored).not.toContain('"error":"stale"')
|
||||
|
||||
expect(readFileSync(join(refreshDir, 'pin-turn', 'system-prompt.golden.md'), 'utf8')).toBe([
|
||||
expect(readFileSync(join(refreshDir, 'pin-turn', 'system-prompt.expected.md'), 'utf8')).toBe([
|
||||
'SYS PROMPT',
|
||||
'',
|
||||
'<!-- request/header-delta 1: keepStart=1, keepEnd=0 -->',
|
||||
'<!-- request/header change 1 -->',
|
||||
'',
|
||||
'SYS PROMPT',
|
||||
'',
|
||||
'NEW PROMPT LINE',
|
||||
'',
|
||||
].join('\n'))
|
||||
const schemas = readFileSync(join(refreshDir, 'pin-turn', 'tool-schemas.golden.json'), 'utf8')
|
||||
const schemas = readFileSync(join(refreshDir, 'pin-turn', 'tool-schemas.expected.json'), 'utf8')
|
||||
expect(schemas).toContain('"description": "D1"')
|
||||
expect(schemas).not.toContain('"name":"stale"')
|
||||
})
|
||||
})
|
||||
|
||||
describe('defineAcpSnapshotSuite: record inventory write-back', () => {
|
||||
it('creates a missing primary fixture and prunes stale child fixtures', () => {
|
||||
expect(readFileSync(join(recordDir, 'rec-pin', 'session.jsonl'), 'utf8')).toContain('"type":"session"')
|
||||
expect(() => readFileSync(join(recordDir, 'rec-child', 'session.2.jsonl'), 'utf8')).toThrow()
|
||||
})
|
||||
})
|
||||
|
||||
describe('defineAcpSnapshotSuite: registration contract', () => {
|
||||
it("throws when a scenario's header class has no pinning scenario", () => {
|
||||
expect(() => {
|
||||
@@ -177,13 +192,41 @@ describe('defineAcpSnapshotSuite: registration contract', () => {
|
||||
})
|
||||
})
|
||||
|
||||
describe('childFixturePaths', () => {
|
||||
it('yields one sibling path per child, 1-based', () => {
|
||||
expect(childFixturePaths('/snap/s', 2)).toEqual(['/snap/s/session.1.jsonl', '/snap/s/session.2.jsonl'])
|
||||
describe('sessionFixtureNames', () => {
|
||||
it('orders the primary and contiguous child fixtures while ignoring other files', () => {
|
||||
expect(sessionFixtureNames([
|
||||
'stdout.expected.jsonl',
|
||||
'session.2.jsonl',
|
||||
'session.jsonl',
|
||||
'session.1.jsonl',
|
||||
'input.json',
|
||||
])).toEqual(['session.jsonl', 'session.1.jsonl', 'session.2.jsonl'])
|
||||
})
|
||||
|
||||
it('yields nothing for a single-session scenario', () => {
|
||||
expect(childFixturePaths('/snap/s', 0)).toEqual([])
|
||||
it('accepts a primary-only scenario', () => {
|
||||
expect(sessionFixtureNames(['session.jsonl'])).toEqual(['session.jsonl'])
|
||||
})
|
||||
|
||||
it('rejects a directory without the primary fixture', () => {
|
||||
expect(() => sessionFixtureNames(['session.1.jsonl'])).toThrow('missing session.jsonl')
|
||||
})
|
||||
|
||||
it('rejects gapped child fixtures', () => {
|
||||
expect(() => sessionFixtureNames(['session.jsonl', 'session.2.jsonl']))
|
||||
.toThrow('expected session.1.jsonl, found session.2.jsonl')
|
||||
})
|
||||
|
||||
it.each(['session.0.jsonl', 'session.child.jsonl', 'session.01.jsonl'])(
|
||||
'rejects invalid child fixture name %s',
|
||||
(name) => {
|
||||
expect(() => sessionFixtureNames(['session.jsonl', name]))
|
||||
.toThrow(`invalid child session fixture name: ${name}`)
|
||||
},
|
||||
)
|
||||
|
||||
it('rejects duplicate child indexes', () => {
|
||||
expect(() => sessionFixtureNames(['session.jsonl', 'session.1.jsonl', 'session.1.jsonl']))
|
||||
.toThrow('expected session.2.jsonl, found session.1.jsonl')
|
||||
})
|
||||
})
|
||||
|
||||
@@ -262,65 +305,41 @@ describe('normalizedToolSchemas', () => {
|
||||
})
|
||||
})
|
||||
|
||||
describe('normalizedToolSchemaDeltas', () => {
|
||||
it('extracts and normalizes object-valued schema edits', () => {
|
||||
const log = [
|
||||
'{"type":"request/header-delta","data":{"tools":{"added":[{"name":"read","description":"work in /w"}]}}}',
|
||||
'{"type":"request/header-delta","data":{"tools":null}}',
|
||||
'{"type":"request/header-delta","data":{"tools":"invalid"}}',
|
||||
'{"type":"request/header-delta","data":{"tools":[]}}',
|
||||
'{"type":"request/header-delta","data":{"system":{"insert":[]}}}',
|
||||
'{"type":"request/header","data":{"tools":{"added":[]}}}',
|
||||
'',
|
||||
].join('\n')
|
||||
expect(normalizedToolSchemaDeltas(log, { sessionIds: [], cwd: '/w' })).toEqual([
|
||||
{ added: [{ name: 'read', description: 'work in {{cwd}}' }] },
|
||||
])
|
||||
})
|
||||
})
|
||||
|
||||
describe('normalizedSystemPromptDeltas', () => {
|
||||
it('extracts and normalizes well-formed system edits', () => {
|
||||
const log = [
|
||||
'{"type":"request/header-delta","data":{"system":{"keepStart":1,"keepEnd":0,"insert":["work in /w"]}}}',
|
||||
'{"type":"request/header-delta","data":{"tools":{"replace":[]}}}',
|
||||
'{"type":"request/header-delta","data":{"system":{"keepStart":"1","keepEnd":0,"insert":[]}}}',
|
||||
'{"type":"request/header-delta","data":{"system":{"keepStart":1,"keepEnd":0,"insert":[null]}}}',
|
||||
'',
|
||||
].join('\n')
|
||||
expect(normalizedSystemPromptDeltas(log, { sessionIds: [], cwd: '/w' })).toEqual([
|
||||
{ keepStart: 1, keepEnd: 0, insert: ['work in {{cwd}}'] },
|
||||
])
|
||||
})
|
||||
})
|
||||
|
||||
describe('formatSystemPromptSnapshot', () => {
|
||||
it('adds a missing terminal newline without changing an existing one', () => {
|
||||
expect(formatSystemPromptSnapshot('prompt')).toBe('prompt\n')
|
||||
expect(formatSystemPromptSnapshot('prompt\n')).toBe('prompt\n')
|
||||
})
|
||||
|
||||
it('renders readable system-prompt delta sections', () => {
|
||||
expect(formatSystemPromptSnapshot('prompt', [
|
||||
{ keepStart: 1, keepEnd: 0, insert: ['new', 'lines'] },
|
||||
])).toBe('prompt\n\n<!-- request/header-delta 1: keepStart=1, keepEnd=0 -->\n\nnew\nlines\n')
|
||||
it('renders readable changed-prompt sections', () => {
|
||||
expect(formatSystemPromptSnapshot('prompt', ['new\nlines']))
|
||||
.toBe('prompt\n\n<!-- request/header change 1 -->\n\nnew\nlines\n')
|
||||
})
|
||||
|
||||
it('does not double the newline of a delta insert with a trailing blank line', () => {
|
||||
expect(formatSystemPromptSnapshot('prompt\n', [
|
||||
{ keepStart: 2, keepEnd: 1, insert: ['tail', ''] },
|
||||
])).toBe('prompt\n\n<!-- request/header-delta 1: keepStart=2, keepEnd=1 -->\n\ntail\n')
|
||||
it('does not double the newline of a changed prompt', () => {
|
||||
expect(formatSystemPromptSnapshot('prompt\n', ['changed\n']))
|
||||
.toBe('prompt\n\n<!-- request/header change 1 -->\n\nchanged\n')
|
||||
})
|
||||
})
|
||||
|
||||
describe('headerChangeCount', () => {
|
||||
it('counts changed request headers, ignoring anchors, blanks, and other lines', () => {
|
||||
const change = JSON.stringify({ type: 'request/header', seq: 2, time: 9, data: { reason: 'change' } })
|
||||
const anchor = JSON.stringify({ type: 'request/header', seq: 0, time: 9, data: { reason: 'initial' } })
|
||||
const other = JSON.stringify({ type: 'turn/start', seq: 1, time: 9, data: {} })
|
||||
expect(headerChangeCount(`${anchor}\n${other}\n\n${change}\n${change}\n`)).toBe(2)
|
||||
expect(headerChangeCount(`${anchor}\n`)).toBe(0)
|
||||
})
|
||||
})
|
||||
|
||||
describe('tool-schema snapshots', () => {
|
||||
const snapshot = {
|
||||
initial: [{ name: 'read', description: 'Read a file.' }],
|
||||
deltas: [{ added: [{ name: 'grep', description: 'Search files.' }] }],
|
||||
changes: [[{ name: 'grep', description: 'Search files.' }]],
|
||||
}
|
||||
|
||||
it('formats and parses canonical structured JSON', () => {
|
||||
const formatted = formatToolSchemasSnapshot(snapshot.initial, snapshot.deltas)
|
||||
const formatted = formatToolSchemasSnapshot(snapshot.initial, snapshot.changes)
|
||||
expect(formatted).toBe(`${JSON.stringify(snapshot, null, 2)}\n`)
|
||||
expect(parseToolSchemasSnapshot(formatted)).toEqual(snapshot)
|
||||
})
|
||||
@@ -329,29 +348,21 @@ describe('tool-schema snapshots', () => {
|
||||
expect(() => parseToolSchemasSnapshot('null')).toThrow(/must be an object/)
|
||||
expect(() => parseToolSchemasSnapshot('"invalid"')).toThrow(/must be an object/)
|
||||
expect(() => parseToolSchemasSnapshot('[]')).toThrow(/must be an object/)
|
||||
expect(() => parseToolSchemasSnapshot('{"initial":{},"deltas":[]}')).toThrow(/array-valued/)
|
||||
expect(() => parseToolSchemasSnapshot('{"initial":[],"deltas":{}}')).toThrow(/array-valued/)
|
||||
expect(() => parseToolSchemasSnapshot('{"initial":{},"changes":[]}')).toThrow(/array-valued/)
|
||||
expect(() => parseToolSchemasSnapshot('{"initial":[],"changes":{}}')).toThrow(/array-valued/)
|
||||
expect(() => parseToolSchemasSnapshot('{"initial":[],"changes":[{}]}')).toThrow(/array-valued/)
|
||||
})
|
||||
|
||||
it('restores initial schemas into the pinned header token', () => {
|
||||
expect(restorePinnedToolSchemas({ system: '{{system}}', tools: '{{tools}}' }, snapshot))
|
||||
expect(restorePinnedToolSchemas({ system: '{{system}}', tools: '{{tools}}' }, snapshot.initial))
|
||||
.toEqual({ system: '{{system}}', tools: snapshot.initial })
|
||||
})
|
||||
|
||||
it('rejects invalid headers and a missing tool token', () => {
|
||||
expect(() => restorePinnedToolSchemas(null, snapshot)).toThrow(/must be an object/)
|
||||
expect(() => restorePinnedToolSchemas('invalid', snapshot)).toThrow(/must be an object/)
|
||||
expect(() => restorePinnedToolSchemas([], snapshot)).toThrow(/must be an object/)
|
||||
expect(() => restorePinnedToolSchemas({ tools: [] }, snapshot)).toThrow(/must equal/)
|
||||
})
|
||||
})
|
||||
|
||||
describe('headerDeltaCount', () => {
|
||||
it('counts request/header-delta events, ignoring blanks and other lines', () => {
|
||||
const delta = JSON.stringify({ type: 'request/header-delta', seq: 2, time: 9, data: {} })
|
||||
const other = JSON.stringify({ type: 'request/header', seq: 0, time: 9, data: {} })
|
||||
expect(headerDeltaCount(`${other}\n\n${delta}\n${delta}\n`)).toBe(2)
|
||||
expect(headerDeltaCount(`${other}\n`)).toBe(0)
|
||||
expect(() => restorePinnedToolSchemas(null, snapshot.initial)).toThrow(/must be an object/)
|
||||
expect(() => restorePinnedToolSchemas('invalid', snapshot.initial)).toThrow(/must be an object/)
|
||||
expect(() => restorePinnedToolSchemas([], snapshot.initial)).toThrow(/must be an object/)
|
||||
expect(() => restorePinnedToolSchemas({ tools: [] }, snapshot.initial)).toThrow(/must equal/)
|
||||
})
|
||||
})
|
||||
|
||||
|
||||
Reference in New Issue
Block a user