Coder/deepseek-harness
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

test(snapshot): derive compaction replay from logs

Browse Source
This commit is contained in:
Tianyi Cui
2026-08-08 01:32:27 +08:00
parent a29a0a4773
commit e46b082fee
16 changed files with 317 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
examples/headless-agent/compaction.cordis.snapshot.yml Normal file
View File

@@ -0,0 +1,25 @@
# Keyless context-overflow composition for the assembled compaction snapshot.
- id: base
name: '@cordisjs/plugin-include'
config:
path: ./cordis.yml
patches:
- id: llm-deepseek
name: '@deepseek-ai/dsh-llm-deepseek'
disabled: true
- id: compact-basic
name: '@deepseek-ai/dsh-compact-basic'
config:
thresholdRatio: 0.99
retainTokens: 20
maxTokens: 32
compactionRetries: 1
- insert:
- id: llm-replay
name: '@deepseek-ai/dsh-llm-replay'
config:
providers:
- id: deepseek-official
models:
- id: deepseek-v4-flash
contextWindow: 128000

4
examples/headless-agent/tests/compaction.e2e.ts
View File

@@ -11,8 +11,8 @@ import { SessionId } from '@deepseek-ai/dsh-session'
* Key-gated smoke for mid-session compaction. It verifies the compact event
* pair, replacement of older surface nodes, and a final answer after compaction.
*/
// FIXME(compaction-snapshot): this is the only full compaction coverage because
// replay cannot serve the summarizer's unlogged model call.
// The keyless headless snapshot pins deterministic overflow recovery; this test
// remains the independent live-provider smoke for organic pressure and summary quality.
let workdir: string | undefined
let ctx: Context | undefined

73
examples/headless-agent/tests/headless.snapshot.ts
View File

@@ -29,6 +29,10 @@ const goalScenarioDir = join(snapshotsDir, 'goal-tools')
const goalConfigPath = fileURLToPath(new URL('../goal.cordis.snapshot.yml', import.meta.url))
const retryScenarioDir = join(snapshotsDir, 'provider-retry')
const retryConfigPath = fileURLToPath(new URL('../retry.cordis.snapshot.yml', import.meta.url))
const compactionScenarioDir = join(snapshotsDir, 'compaction-recovery')
const compactionSessionFixture = join(compactionScenarioDir, 'session.jsonl')
const compactionStreamExpected = join(compactionScenarioDir, 'stream-json.expected.jsonl')
const compactionConfigPath = fileURLToPath(new URL('../compaction.cordis.snapshot.yml', import.meta.url))
const credentialsScenarioDir = join(snapshotsDir, 'missing-credential')
const credentialsConfigPath = fileURLToPath(new URL('../credentials.cordis.snapshot.yml', import.meta.url))
// Same keyless composition as the missing-credential scenario: the endpoint is
@@ -227,6 +231,75 @@ describe('headless stream-json snapshots', () => {
expect(normalized).toBe(await readFile(streamExpected, 'utf8'))
}, LOADER_SMOKE_TEST_TIMEOUT_MS)
it('recovers from context overflow through an assembled compaction', async () => {
const prompt = await scenarioPrompt(compactionScenarioDir, 'compaction-recovery')
let expectedSession = await readFile(compactionSessionFixture, 'utf8')
let runCwd = ''
const result = await runLoaderSmoke({
label: 'compaction recovery headless stream-json snapshot',
tempDirPrefix: 'headless-snapshot-compaction-recovery-',
binScript,
configPath: compactionConfigPath,
binArgs: ['--config', compactionConfigPath, '--output-format', 'stream-json', prompt],
tsconfigPath,
env: {
DSH_SNAPSHOT: 'replay',
DSH_SNAPSHOT_FILE: compactionSessionFixture,
NODE_OPTIONS: [process.env.NODE_OPTIONS, '--disable-warning=ExperimentalWarning'].filter(Boolean).join(' '),
},
prepare: (cwd) => { runCwd = cwd },
inspect: async (cwd) => {
const logs = await persistedLogs(cwd)
expect(logs).toHaveLength(1)
const actual = logs[0]
if (actual === undefined) throw new Error('compaction snapshot did not persist its session')
const records = parseJsonl(actual.content)
const types = records.map(record => record.type)
expect(types.filter(type => type === 'compact/start')).toHaveLength(1)
expect(types.filter(type => type === 'compact/summary')).toHaveLength(1)
expect(types.filter(type => type === 'compact/end')).toHaveLength(1)
const start = types.indexOf('compact/start')
const summary = types.indexOf('compact/summary')
const replacement = records.findIndex((record) => {
if (record.type !== 'user/message') return false
const surfaceOp = record.surfaceOp as JsonObject | undefined
return surfaceOp?.op === 'replace'
})
const end = types.indexOf('compact/end')
expect(start).toBeLessThan(summary)
expect(summary).toBeLessThan(replacement)
expect(replacement).toBeLessThan(end)
const summaryRecord = records[summary]
const summaryData = summaryRecord?.data as JsonObject | undefined
expect(summaryData?.shadowedSeqs).toEqual(expect.arrayContaining([expect.any(Number)]))
const final = [...records].reverse().find(record => record.type === 'assistant/message')
expect(JSON.stringify(final)).toContain('COMPACTION RECOVERED')
const actualContext = contextFromLogs([actual.content])
if (refreshing) {
const harvested: HarvestedLog = {
id: String(actual.header.id),
createdAt: Number(actual.header.createdAt),
content: actual.content,
}
const replacements = refreshFixtureReplacements([harvested], [expectedSession])
expectedSession = tokenizeSessionFixtureCwd(
stabilizeRefreshLog(actual.content, expectedSession, replacements, actualContext),
)
await writeFile(compactionSessionFixture, expectedSession)
}
const expectedContext = contextFromLogs([expectedSession])
expect(scrubRequestHeaders(normalizeSessionLog(actual.content, actualContext)))
.toBe(scrubRequestHeaders(normalizeSessionLog(expectedSession, expectedContext)))
},
})
expect(result.stderr).toBe('')
const normalized = normalizeHeadlessStream(result.stdout, runCwd)
if (refreshing) await writeFile(compactionStreamExpected, normalized)
expect(normalized).toBe(await readFile(compactionStreamExpected, 'utf8'))
}, LOADER_SMOKE_TEST_TIMEOUT_MS)
it('logs actionable missing-credential guidance through the one-shot app', async () => {
const streamExpected = join(credentialsScenarioDir, 'stream-json.expected.jsonl')
let runCwd = ''

8
examples/headless-agent/tests/snapshots/compaction-recovery/input.json Normal file
View File

@@ -0,0 +1,8 @@
{
"steps": [
{
"op": "prompt",
"text": "Establish a durable compaction premise before continuing. Record every part of this historical evidence: the snapshot uses keyless replay; persistence uses JSONL; the assembled headless application loads its real Cordis composition; model-visible inputs remain logged; tool calls and results remain paired and ordered; context overflow retains the original failure while recovery is attempted; compaction opens with compact/start and closes with compact/end; a successful auxiliary summary records compact/summary provenance; the replacement surface shadows only an older balanced range; the checkpoint remains smaller than the history it replaces; the newest tool result remains verbatim; the retried request sees that checkpoint; the final response proves the same turn continued; deterministic snapshot evidence stays separate from the live-provider smoke; and no external API key is needed. Emit one alpha marker through bash, then finish the task after any required recovery with the exact words COMPACTION RECOVERED."
}
]
}

32
examples/headless-agent/tests/snapshots/compaction-recovery/session.jsonl Normal file
View File

File diff suppressed because one or more lines are too long

32
examples/headless-agent/tests/snapshots/compaction-recovery/stream-json.expected.jsonl Normal file
View File

@@ -0,0 +1,32 @@
{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"agent/inbox/spliced","seq":0,"time":0,"data":{"target":"next-turn","start":0,"inserted":[{"content":[{"type":"text","text":"Establish a durable compaction premise before continuing. Record every part of this historical evidence: the snapshot uses keyless replay; persistence uses JSONL; the assembled headless application loads its real Cordis composition; model-visible inputs remain logged; tool calls and results remain paired and ordered; context overflow retains the original failure while recovery is attempted; compaction opens with compact/start and closes with compact/end; a successful auxiliary summary records compact/summary provenance; the replacement surface shadows only an older balanced range; the checkpoint remains smaller than the history it replaces; the newest tool result remains verbatim; the retried request sees that checkpoint; the final response proves the same turn continued; deterministic snapshot evidence stays separate from the live-provider smoke; and no external API key is needed. Emit one alpha marker through bash, then finish the task after any required recovery with the exact words COMPACTION RECOVERED."}],"source":{"kind":"user"},"role":"user","id":"{{sessionId}}"}]}}}
{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"turn/start","seq":1,"time":0,"data":{"turn":1}}}
{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"agent/inbox/spliced","seq":2,"time":0,"data":{"target":"next-turn","start":0,"removedCount":1,"inserted":[]}}}
{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"step/start","seq":3,"time":0,"data":{"turn":1,"step":1}}}
{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"user/message","seq":4,"time":0,"data":{"content":[{"type":"text","text":"Establish a durable compaction premise before continuing. Record every part of this historical evidence: the snapshot uses keyless replay; persistence uses JSONL; the assembled headless application loads its real Cordis composition; model-visible inputs remain logged; tool calls and results remain paired and ordered; context overflow retains the original failure while recovery is attempted; compaction opens with compact/start and closes with compact/end; a successful auxiliary summary records compact/summary provenance; the replacement surface shadows only an older balanced range; the checkpoint remains smaller than the history it replaces; the newest tool result remains verbatim; the retried request sees that checkpoint; the final response proves the same turn continued; deterministic snapshot evidence stays separate from the live-provider smoke; and no external API key is needed. Emit one alpha marker through bash, then finish the task after any required recovery with the exact words COMPACTION RECOVERED."}],"source":{"kind":"user"},"role":"user","id":"{{sessionId}}"},"surfaceOp":"append"}}
{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"session/title","seq":5,"time":0,"data":{"title":"Establish a durable compaction premise","messageSeqs":[4],"source":{"kind":"fallback"}}}}
{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"request/header","seq":6,"time":0,"data":{"header":{"config":{"provider":"deepseek-official","model":"deepseek-v4-flash"},"system":"{{system}}","tools":"{{tools}}"},"reason":"initial"}}}
{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"request/context","seq":7,"time":0,"data":{"provider":"deepseek-official","model":"deepseek-v4-flash","contextWindow":128000}}}
{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"assistant/chunk","seq":8,"time":0,"data":{"turn":1,"step":1,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}}}}
{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"assistant/chunk","seq":9,"time":0,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":0,"id":"call_compaction_marker","name":"bash","argumentsDelta":"{\"command\":\"printf 'alpha\\n'\",\"description\":\"Emit compaction premise marker\"}"}}}}
{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"assistant/chunk","seq":10,"time":0,"data":{"turn":1,"step":1,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"call_compaction_marker","name":"bash","arguments":"{\"command\":\"printf 'alpha\\n'\",\"description\":\"Emit compaction premise marker\"}"}}}}}
{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"assistant/chunk","seq":11,"time":0,"data":{"turn":1,"step":1,"chunk":{"type":"usage","usage":{"inputTokens":24,"outputTokens":6}}}}}
{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"assistant/chunk","seq":12,"time":0,"data":{"turn":1,"step":1,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}}}
{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"assistant/message","seq":13,"time":0,"data":{"turn":1,"step":1,"message":{"role":"assistant","content":[{"type":"tool-call","id":"call_compaction_marker","name":"bash","arguments":"{\"command\":\"printf 'alpha\\n'\",\"description\":\"Emit compaction premise marker\"}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash"},"id":"{{sessionId}}"},"usage":{"inputTokens":24,"outputTokens":6}},"sourceEventSeqs":[8,9,10,11,12],"surfaceOp":"append"}}
{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"tool/call","seq":14,"time":0,"data":{"turn":1,"step":1,"callId":"call_compaction_marker","name":"bash","arguments":"{\"command\":\"printf 'alpha\\n'\",\"description\":\"Emit compaction premise marker\"}"}}}
{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"tool/result","seq":15,"time":0,"data":{"turn":1,"step":1,"message":{"source":{"kind":"tool","callId":"call_compaction_marker"},"content":[{"type":"tool-result","toolCallId":"call_compaction_marker","content":[{"type":"text","text":"alpha\n"}],"isError":false}],"role":"user","id":"{{sessionId}}"}},"sourceEventSeqs":[14],"surfaceOp":"append"}}
{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"step/end","seq":16,"time":0,"data":{"turn":1,"step":1}}}
{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"step/start","seq":17,"time":0,"data":{"turn":1,"step":2}}}
{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"assistant/chunk","seq":18,"time":0,"data":{"turn":1,"step":2,"chunk":{"type":"finish","reason":{"kind":"error","failure":{"message":"snapshot request exceeded the model context window","code":"CONTEXT_WINDOW_EXCEEDED"}}}}}}
{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"compact/start","seq":19,"time":0,"data":{"turn":1}}}
{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"compact/summary","seq":20,"time":0,"data":{"summary":[{"type":"text","text":"The request established a durable compaction premise."}],"rawOutput":[{"type":"text","text":"The request established a durable compaction premise."}],"shadowedRange":{"start":4,"end":4},"shadowedSeqs":[4],"shadowedTokenCount":264,"provider":"deepseek-official","model":"deepseek-v4-flash","maxTokens":32,"usage":{"inputTokens":20,"outputTokens":4}}}}
{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"user/message","seq":21,"time":0,"data":{"content":[{"type":"text","text":"This is an automatically generated checkpoint condensing an earlier span of the conversation to free up context. Treat the captured context as established background and build on it without restating it. Continue the task directly from the messages that follow, without acknowledging this checkpoint.\n\n<compacted-summary>"},{"type":"text","text":"The request established a durable compaction premise."},{"type":"text","text":"</compacted-summary>"}],"source":{"kind":"plugin","plugin":"compact"},"role":"user","id":"{{sessionId}}"},"sourceEventSeqs":[19,20,4],"surfaceOp":{"op":"replace","start":4,"end":4}}}
{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"compact/end","seq":22,"time":0,"data":{"turn":1}}}
{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"assistant/chunk","seq":23,"time":0,"data":{"turn":1,"step":2,"chunk":{"type":"block-start","index":0,"blockType":"text"}}}}
{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"assistant/chunk","seq":24,"time":0,"data":{"turn":1,"step":2,"chunk":{"type":"text-delta","index":0,"text":"COMPACTION RECOVERED"}}}}
{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"assistant/chunk","seq":25,"time":0,"data":{"turn":1,"step":2,"chunk":{"type":"block-end","index":0,"block":{"type":"text","text":"COMPACTION RECOVERED"}}}}}
{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"assistant/chunk","seq":26,"time":0,"data":{"turn":1,"step":2,"chunk":{"type":"usage","usage":{"inputTokens":20,"outputTokens":4}}}}}
{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"assistant/chunk","seq":27,"time":0,"data":{"turn":1,"step":2,"chunk":{"type":"finish","reason":{"kind":"stop"}}}}}
{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"assistant/message","seq":28,"time":0,"data":{"turn":1,"step":2,"message":{"role":"assistant","content":[{"type":"text","text":"COMPACTION RECOVERED"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash"},"id":"{{sessionId}}"},"usage":{"inputTokens":20,"outputTokens":4}},"sourceEventSeqs":[23,24,25,26,27],"surfaceOp":"append"}}
{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"step/end","seq":29,"time":0,"data":{"turn":1,"step":2}}}
{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"turn/end","seq":30,"time":0,"data":{"turn":1,"reason":{"kind":"completed"}}}}
{"type":"result","sessionId":"{{sessionId}}","output":"COMPACTION RECOVERED","usage":{"inputTokens":44,"outputTokens":10}}