fix: stabilize spill snapshot path budget
This commit is contained in:
@@ -113,11 +113,13 @@
|
|||||||
# results under the session cwd, and the post-execute policy that replaces the
|
# results under the session cwd, and the post-execute policy that replaces the
|
||||||
# model-facing result with a bounded preview + read path. Snapshots lower the
|
# model-facing result with a bounded preview + read path. Snapshots lower the
|
||||||
# cap so a deterministic bash result exercises this transcript surface without a
|
# cap so a deterministic bash result exercises this transcript surface without a
|
||||||
# real model call; normal demo runs keep the coding-agent cap.
|
# real model call. The snapshot harness supplies a fixed spill root so the
|
||||||
|
# spill-policy preview budget is stable across macOS/Linux path lengths; normal
|
||||||
|
# demo runs keep the session-local `.spill` root and the coding-agent cap.
|
||||||
- id: spill-local
|
- id: spill-local
|
||||||
name: '@deepseek-ai/dsh-spill-local'
|
name: '@deepseek-ai/dsh-spill-local'
|
||||||
config:
|
config:
|
||||||
root: ./.spill
|
root: !!js process.env.DSH_SNAPSHOT_SPILL_ROOT ?? './.spill'
|
||||||
|
|
||||||
- id: spill-policy
|
- id: spill-policy
|
||||||
name: '@deepseek-ai/dsh-spill-policy'
|
name: '@deepseek-ai/dsh-spill-policy'
|
||||||
|
|||||||
@@ -10,7 +10,7 @@
|
|||||||
{"type":"assistant/chunk","seq":8,"time":0,"data":{"turn":1,"step":1,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}}
|
{"type":"assistant/chunk","seq":8,"time":0,"data":{"turn":1,"step":1,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}}
|
||||||
{"type":"assistant/message","seq":9,"time":0,"data":{"turn":1,"step":1,"content":[{"type":"tool-call","id":"call_spill","name":"bash","arguments":"{\"command\":\"node -e \\\"process.stdout.write('SPILL_START-' + 'x'.repeat(2000) + '-SPILL_END')\\\"\",\"description\":\"Print large deterministic output\"}"}],"usage":{"inputTokens":10,"outputTokens":5}},"sourceEventSeqs":[4,5,6,7,8],"surfaceOp":"append"}
|
{"type":"assistant/message","seq":9,"time":0,"data":{"turn":1,"step":1,"content":[{"type":"tool-call","id":"call_spill","name":"bash","arguments":"{\"command\":\"node -e \\\"process.stdout.write('SPILL_START-' + 'x'.repeat(2000) + '-SPILL_END')\\\"\",\"description\":\"Print large deterministic output\"}"}],"usage":{"inputTokens":10,"outputTokens":5}},"sourceEventSeqs":[4,5,6,7,8],"surfaceOp":"append"}
|
||||||
{"type":"tool/call","seq":10,"time":0,"data":{"turn":1,"step":1,"callId":"call_spill","name":"bash","arguments":"{\"command\":\"node -e \\\"process.stdout.write('SPILL_START-' + 'x'.repeat(2000) + '-SPILL_END')\\\"\",\"description\":\"Print large deterministic output\"}"}}
|
{"type":"tool/call","seq":10,"time":0,"data":{"turn":1,"step":1,"callId":"call_spill","name":"bash","arguments":"{\"command\":\"node -e \\\"process.stdout.write('SPILL_START-' + 'x'.repeat(2000) + '-SPILL_END')\\\"\",\"description\":\"Print large deterministic output\"}"}}
|
||||||
{"type":"tool/result","seq":11,"time":0,"data":{"turn":1,"step":1,"callId":"call_spill","content":[{"type":"text","text":"SPILL_START-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx-SPILL_END\n\n(Omitted 1447 bytes. Full formatted result saved to: {{spillPath:bash.txt}}. Use read with offset/limit to inspect it.)"}],"isError":false},"sourceEventSeqs":[10],"surfaceOp":"append"}
|
{"type":"tool/result","seq":11,"time":0,"data":{"turn":1,"step":1,"callId":"call_spill","content":[{"type":"text","text":"SPILL_START-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx-SPILL_END\n\n(Omitted 1391 bytes. Full formatted result saved to: {{spillPath:bash.txt}}. Use read with offset/limit to inspect it.)"}],"isError":false},"sourceEventSeqs":[10],"surfaceOp":"append"}
|
||||||
{"type":"step/end","seq":12,"time":0,"data":{"turn":1,"step":1}}
|
{"type":"step/end","seq":12,"time":0,"data":{"turn":1,"step":1}}
|
||||||
{"type":"step/start","seq":13,"time":0,"data":{"turn":1,"step":2}}
|
{"type":"step/start","seq":13,"time":0,"data":{"turn":1,"step":2}}
|
||||||
{"type":"assistant/chunk","seq":14,"time":0,"data":{"turn":1,"step":2,"chunk":{"type":"block-start","index":0,"blockType":"text"}}}
|
{"type":"assistant/chunk","seq":14,"time":0,"data":{"turn":1,"step":2,"chunk":{"type":"block-start","index":0,"blockType":"text"}}}
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
{"jsonrpc":"2.0","id":1,"result":{"protocolVersion":1,"agentInfo":{"name":"deepseek-harness-acp","version":"0.0.1"},"agentCapabilities":{"loadSession":true,"promptCapabilities":{"image":false,"audio":false,"embeddedContext":false}},"authMethods":[]}}
|
{"jsonrpc":"2.0","id":1,"result":{"protocolVersion":1,"agentInfo":{"name":"deepseek-harness-acp","version":"0.0.1"},"agentCapabilities":{"loadSession":true,"promptCapabilities":{"image":false,"audio":false,"embeddedContext":false}},"authMethods":[]}}
|
||||||
{"jsonrpc":"2.0","id":2,"result":{"sessionId":"{{sessionId}}"}}
|
{"jsonrpc":"2.0","id":2,"result":{"sessionId":"{{sessionId}}"}}
|
||||||
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"tool_call","toolCallId":"call_spill","title":"node -e \"process.stdout.write('SPILL_START-' + 'x'.repeat(2000) + '-SPILL_END')\"","kind":"execute","status":"in_progress","rawInput":"node -e \"process.stdout.write('SPILL_START-' + 'x'.repeat(2000) + '-SPILL_END')\"","content":[{"type":"content","content":{"type":"text","text":"Print large deterministic output"}}]}}}
|
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"tool_call","toolCallId":"call_spill","title":"node -e \"process.stdout.write('SPILL_START-' + 'x'.repeat(2000) + '-SPILL_END')\"","kind":"execute","status":"in_progress","rawInput":"node -e \"process.stdout.write('SPILL_START-' + 'x'.repeat(2000) + '-SPILL_END')\"","content":[{"type":"content","content":{"type":"text","text":"Print large deterministic output"}}]}}}
|
||||||
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"tool_call_update","toolCallId":"call_spill","status":"completed","content":[{"type":"content","content":{"type":"text","text":"```console\nSPILL_START-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx-SPILL_END\n\n(Omitted 1447 bytes. Full formatted result saved to: {{spillPath:bash.txt}}. Use read with offset/limit to inspect it.)\n```"}}]}}}
|
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"tool_call_update","toolCallId":"call_spill","status":"completed","content":[{"type":"content","content":{"type":"text","text":"```console\nSPILL_START-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx-SPILL_END\n\n(Omitted 1391 bytes. Full formatted result saved to: {{spillPath:bash.txt}}. Use read with offset/limit to inspect it.)\n```"}}]}}}
|
||||||
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":"DONE"}}}}
|
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":"DONE"}}}}
|
||||||
{"jsonrpc":"2.0","id":3,"result":{"stopReason":"end_turn"}}
|
{"jsonrpc":"2.0","id":3,"result":{"stopReason":"end_turn"}}
|
||||||
|
|||||||
@@ -180,6 +180,9 @@ export interface RunOptions {
|
|||||||
export async function runScenario(input: InputScript, opts: RunOptions): Promise<RunResult> {
|
export async function runScenario(input: InputScript, opts: RunOptions): Promise<RunResult> {
|
||||||
const cwd = await mkdtemp(join(tmpdir(), 'acp-snap-cwd-'))
|
const cwd = await mkdtemp(join(tmpdir(), 'acp-snap-cwd-'))
|
||||||
const sessionsRoot = await mkdtemp(join(tmpdir(), 'acp-snap-sessions-'))
|
const sessionsRoot = await mkdtemp(join(tmpdir(), 'acp-snap-sessions-'))
|
||||||
|
// Fixed path length: spill-policy budgets the preview against the REAL path
|
||||||
|
// before stdout normalization, so tmpdir() length differences churn goldens.
|
||||||
|
const spillRoot = '/tmp/dsh-acp-snapshot-spill'
|
||||||
// Everything past the temp-dir creation runs under a try/finally that always
|
// Everything past the temp-dir creation runs under a try/finally that always
|
||||||
// removes both dirs — so a failure in workspace seeding, spawn, or any step
|
// removes both dirs — so a failure in workspace seeding, spawn, or any step
|
||||||
// never leaks them (the "e2e tests own their resources" rule).
|
// never leaks them (the "e2e tests own their resources" rule).
|
||||||
@@ -201,6 +204,7 @@ export async function runScenario(input: InputScript, opts: RunOptions): Promise
|
|||||||
DSH_SNAPSHOT: opts.mode,
|
DSH_SNAPSHOT: opts.mode,
|
||||||
DSH_SNAPSHOT_FILE: opts.fixtureFile,
|
DSH_SNAPSHOT_FILE: opts.fixtureFile,
|
||||||
DSH_SNAPSHOT_SESSIONS_ROOT: sessionsRoot,
|
DSH_SNAPSHOT_SESSIONS_ROOT: sessionsRoot,
|
||||||
|
DSH_SNAPSHOT_SPILL_ROOT: spillRoot,
|
||||||
...opts.overrideFile !== undefined ? { DSH_SNAPSHOT_OVERRIDE: opts.overrideFile } : {},
|
...opts.overrideFile !== undefined ? { DSH_SNAPSHOT_OVERRIDE: opts.overrideFile } : {},
|
||||||
...opts.childFiles !== undefined && opts.childFiles.length > 0
|
...opts.childFiles !== undefined && opts.childFiles.length > 0
|
||||||
? { DSH_SNAPSHOT_CHILD_FILES: opts.childFiles.join(delimiter) }
|
? { DSH_SNAPSHOT_CHILD_FILES: opts.childFiles.join(delimiter) }
|
||||||
@@ -310,6 +314,7 @@ export async function runScenario(input: InputScript, opts: RunOptions): Promise
|
|||||||
}
|
}
|
||||||
await rm(cwd, { recursive: true, force: true })
|
await rm(cwd, { recursive: true, force: true })
|
||||||
await rm(sessionsRoot, { recursive: true, force: true })
|
await rm(sessionsRoot, { recursive: true, force: true })
|
||||||
|
await rm(spillRoot, { recursive: true, force: true })
|
||||||
}
|
}
|
||||||
|
|
||||||
return {
|
return {
|
||||||
|
|||||||
@@ -38,6 +38,11 @@ const LOCAL_SPILL_PATH_RE = new RegExp(
|
|||||||
+ String.raw`(?=\. Use read with offset/limit|[\s)]|$)`,
|
+ String.raw`(?=\. Use read with offset/limit|[\s)]|$)`,
|
||||||
'g',
|
'g',
|
||||||
)
|
)
|
||||||
|
const SNAPSHOT_SPILL_PATH_RE = new RegExp(
|
||||||
|
String.raw`/tmp/dsh-acp-snapshot-spill/session-[0-9a-f]{12}/[0-9a-f]{12}-([A-Za-z0-9._~-]+?)`
|
||||||
|
+ String.raw`(?=\. Use read with offset/limit|[\s)]|$)`,
|
||||||
|
'g',
|
||||||
|
)
|
||||||
|
|
||||||
/** Inputs the normalizers need to recognize a run's volatile values. */
|
/** Inputs the normalizers need to recognize a run's volatile values. */
|
||||||
export interface NormalizeContext {
|
export interface NormalizeContext {
|
||||||
@@ -55,6 +60,7 @@ function scrubString(value: string, ctx: NormalizeContext): string {
|
|||||||
out = out.split(ctx.cwd).join(CWD)
|
out = out.split(ctx.cwd).join(CWD)
|
||||||
out = out.split(`/private${CWD}`).join(CWD)
|
out = out.split(`/private${CWD}`).join(CWD)
|
||||||
out = out.replace(LOCAL_SPILL_PATH_RE, (_match, name: string) => `{{spillPath:${name}}}`)
|
out = out.replace(LOCAL_SPILL_PATH_RE, (_match, name: string) => `{{spillPath:${name}}}`)
|
||||||
|
out = out.replace(SNAPSHOT_SPILL_PATH_RE, (_match, name: string) => `{{spillPath:${name}}}`)
|
||||||
for (const id of ctx.sessionIds) out = out.split(id).join(SESSION_ID)
|
for (const id of ctx.sessionIds) out = out.split(id).join(SESSION_ID)
|
||||||
out = out.replace(UUID_RE, SESSION_ID)
|
out = out.replace(UUID_RE, SESSION_ID)
|
||||||
return out
|
return out
|
||||||
|
|||||||
@@ -117,6 +117,21 @@ describe('normalizeSessionLog', () => {
|
|||||||
expect(out).not.toContain('/private{{spillPath')
|
expect(out).not.toContain('/private{{spillPath')
|
||||||
})
|
})
|
||||||
|
|
||||||
|
it('scrubs fixed snapshot spill paths', () => {
|
||||||
|
const ev = JSON.stringify({
|
||||||
|
type: 'tool/result', seq: 2, time: 5,
|
||||||
|
data: {
|
||||||
|
content: [{
|
||||||
|
type: 'text',
|
||||||
|
text: 'Full formatted result saved to: /tmp/dsh-acp-snapshot-spill/session-c22bc3f1d2af/8a7b6c5d4e3f-bash.txt. Use read with offset/limit to inspect it.',
|
||||||
|
}],
|
||||||
|
},
|
||||||
|
})
|
||||||
|
const out = normalizeSessionLog(`${header({ cwd: ctx.cwd })}\n${ev}\n`, ctx)
|
||||||
|
expect(out).toContain('{{spillPath:bash.txt}}')
|
||||||
|
expect(out).not.toContain('/tmp/dsh-acp-snapshot-spill')
|
||||||
|
})
|
||||||
|
|
||||||
it('scrubs the session id in the header', () => {
|
it('scrubs the session id in the header', () => {
|
||||||
const out = normalizeSessionLog(`${header({ id: ctx.sessionIds[0] })}\n`, ctx)
|
const out = normalizeSessionLog(`${header({ id: ctx.sessionIds[0] })}\n`, ctx)
|
||||||
expect(out).toContain('{{sessionId}}')
|
expect(out).toContain('{{sessionId}}')
|
||||||
|
|||||||
Reference in New Issue
Block a user