fix(session): add semantic crash checkpoints

This commit is contained in:
Yichen Jiang
2026-07-21 14:50:06 +08:00
parent 9a5c81f9e5
commit 6d12e3ab41
56 changed files with 1016 additions and 61 deletions

View File

@@ -0,0 +1,41 @@
# dsh-session-checkpoint-policy
Semantic durability policy for persisted agents. It checkpoints the event-sourced session before a model adapter receives a request, before a top-level tool body may produce an external side effect, and after a step has recorded its complete assistant message and ordered tool results. The final `turn/end` checkpoint remains owned by `dsh-agent-loop`.
## Plugin (namespace: `session-checkpoint-policy`)
This zero-config function plugin consumes `ctx.sessions`, `ctx.llm`, `ctx.tools`, and the presence of `ctx.sessionPersistence`. Load it beside one persistence backend:
```yaml
- id: session-persistence
name: '@deepseek-ai/dsh-session-persistence-jsonl'
- id: session-checkpoints
name: '@deepseek-ai/dsh-session-checkpoint-policy'
```
The policy wraps `llm/stream` lazily, so the downstream stream is not constructed until the live session's buffered request events are durable. It wraps `tools/execute` after pre-execute policy and guards; a top-level tool body runs only after its recorded call is durable. Nested tool dispatches reuse the outer model-visible call's checkpoint. `agent/post-step` persists the complete response/result batch before continuation work.
Checkpoint rejection is fail-closed at the model and tool boundaries: neither the adapter nor the top-level tool body runs. A post-step rejection fails the turn before another request starts. Concurrent tool checkpoints share the session store's serialized persistence drain and cannot duplicate sequence numbers.
## Model Experience
### Interrupted calls
#### What the model sees
The plugin adds no prompt or tool schema. A hard crash after a tool checkpoint but before its result leaves a durable unmatched call; session recovery supplies the model-visible `TOOL_OUTCOME_UNKNOWN` result owned by `dsh-session`. The message permits retry for read-only or idempotent work and requires state verification or user confirmation for calls that may have side effects.
#### Token effect
Successful checkpoints add no tokens and do not change the request. Recovery adds one short tool-result message to balance the interrupted transcript.
#### KV Cache effect
The repair result is appended after the reusable prefix, so it does not invalidate earlier cache entries.
## Known Limitations and Deferred Work
- The policy durably records execution intent, not generic exactly-once effects. Side-effecting tools should forward `exec.callId` as an idempotency key when their provider supports one.
- Streaming `assistant/chunk` events have no per-chunk checkpoint. They reach storage with the next semantic checkpoint, so a hard crash may lose the current partial response.
- A persisted call without a result cannot prove whether its external effect completed. Recovery therefore records an unknown outcome instead of retrying automatically.

View File

@@ -0,0 +1,45 @@
{
"name": "@deepseek-ai/dsh-session-checkpoint-policy",
"description": "Semantic session durability checkpoints before model requests and tool side effects",
"version": "0.0.1",
"private": true,
"type": "module",
"main": "lib/index.js",
"types": "lib/types/index.d.ts",
"exports": {
".": {
"types": "./lib/types/index.d.ts",
"default": "./lib/index.js"
},
"./src/*": "./src/*",
"./package.json": "./package.json"
},
"files": [
"lib/index.js",
"lib/types/**/*.d.ts",
"lib/types/**/*.d.ts.map",
"src"
],
"license": "BSD-3-Clause",
"peerDependencies": {
"@deepseek-ai/dsh-agent": "^0.0.1",
"@deepseek-ai/dsh-llm": "^0.0.1",
"@deepseek-ai/dsh-session": "^0.0.1",
"@deepseek-ai/dsh-session-persistence": "^0.0.1",
"@deepseek-ai/dsh-tools": "^0.0.1",
"cordis": "^4.0.0-rc.7"
},
"devDependencies": {
"@cordisjs/plugin-loader": "workspace:^",
"@deepseek-ai/dsh-agent": "workspace:^",
"@deepseek-ai/dsh-agent-loop": "workspace:^",
"@deepseek-ai/dsh-agent-loop-testkit": "workspace:^",
"@deepseek-ai/dsh-llm": "workspace:^",
"@deepseek-ai/dsh-session": "workspace:^",
"@deepseek-ai/dsh-session-persistence": "workspace:^",
"@deepseek-ai/dsh-session-persistence-jsonl": "workspace:^",
"@deepseek-ai/dsh-system-prompt": "workspace:^",
"@deepseek-ai/dsh-tools": "workspace:^",
"cordis": "^4.0.0-rc.7"
}
}

View File

@@ -0,0 +1,65 @@
/**
* Semantic durability checkpoints for model requests, top-level tool dispatch,
* and completed agent steps.
* @module @deepseek-ai/dsh-session-checkpoint-policy
*/
import type { Context } from 'cordis'
import type { Session } from '@deepseek-ai/dsh-session'
import type { StreamChunk } from '@deepseek-ai/dsh-llm'
import type { ToolExecutionResult } from '@deepseek-ai/dsh-tools'
import type {} from '@deepseek-ai/dsh-agent'
import type {} from '@deepseek-ai/dsh-session-persistence'
/** Cordis plugin name used by Loader diagnostics. */
export const name = 'session-checkpoint-policy'
/** Services whose request, tool, session, and persistence boundaries this policy joins. */
export const inject = ['llm', 'sessionPersistence', 'sessions', 'tools']
/**
* Delay construction of the downstream model stream until the complete logged
* request prefix is durable. A checkpoint rejection prevents adapter dispatch.
*
* @param ctx - plugin context that owns the session store.
* @param session - live session named by the model request.
* @param next - downstream `llm/stream` chain.
* @returns a stream that checkpoints before requesting its first chunk.
*/
function afterCheckpoint(
ctx: Context,
session: Session,
next: () => AsyncIterable<StreamChunk>,
): AsyncIterable<StreamChunk> {
return (async function* (): AsyncIterable<StreamChunk> {
await ctx.sessions.flush(session)
yield* next()
})()
}
/**
* Install semantic checkpoint listeners. Loop-built model calls checkpoint the
* logged request before adapter dispatch; top-level tool calls checkpoint their
* recorded call before the tool body; post-step checkpoints retain the complete
* response/result batch. Nested tool dispatches reuse the durable outer call.
*
* Checkpoint failures are fail-closed at the model and tool side-effect
* boundaries: the downstream adapter or tool body is not invoked.
*
* @param ctx - plugin context that owns the listeners.
*/
export function apply(ctx: Context): void {
ctx.on('llm/stream', (options, next): AsyncIterable<StreamChunk> => {
if (options.sessionId === undefined) return next()
const session = ctx.sessions.get(options.sessionId)
return session === undefined ? next() : afterCheckpoint(ctx, session, next)
})
ctx.on('tools/execute', async (exec, next): Promise<ToolExecutionResult> => {
if (exec.agent === undefined || exec.parent !== undefined) return next()
await ctx.sessions.flush(exec.agent.session)
return next()
})
ctx.on('agent/post-step', (agent): Promise<void> => ctx.sessions.flush(agent.session))
}

View File

@@ -0,0 +1,104 @@
import { spawn } from 'node:child_process'
import { access, mkdtemp, readFile, rm } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { fileURLToPath } from 'node:url'
import { Context } from 'cordis'
import { afterEach, describe, expect, it } from 'vitest'
import SessionStore, {
SessionId, TOOL_OUTCOME_UNKNOWN,
type SessionEvent,
} from '@deepseek-ai/dsh-session'
import SessionPersistenceJsonl from '@deepseek-ai/dsh-session-persistence-jsonl'
const repoRoot = fileURLToPath(new URL('../../../../', import.meta.url))
const childScript = fileURLToPath(new URL('./fixtures/crash-child.ts', import.meta.url))
const tsxLoader = fileURLToPath(import.meta.resolve('tsx'))
const sessionId = SessionId('semantic-checkpoint-crash')
const roots: string[] = []
async function waitForFile(path: string): Promise<void> {
for (let attempt = 0; attempt < 500; attempt += 1) {
try {
await access(path)
return
} catch (error: unknown) {
if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error
}
await new Promise(resolve => setTimeout(resolve, 10))
}
throw new Error(`crash child did not reach failpoint ${path}`)
}
async function crashAt(mode: 'request' | 'tool'): Promise<{ root: string; markerText: string }> {
const root = await mkdtemp(join(tmpdir(), `dsh-semantic-${mode}-`))
roots.push(root)
const marker = join(root, 'failpoint')
const child = spawn(process.execPath, ['--import', tsxLoader, childScript, mode, root, marker], {
cwd: repoRoot,
env: { ...process.env, TSX_TSCONFIG_PATH: join(repoRoot, 'tsconfig.json') },
stdio: ['ignore', 'ignore', 'pipe'],
})
let stderr = ''
child.stderr.setEncoding('utf8')
child.stderr.on('data', (chunk: string) => { stderr += chunk })
try {
await waitForFile(marker)
const markerText = await readFile(marker, 'utf8')
const closed = new Promise<{ code: number | null; signal: NodeJS.Signals | null }>((resolve) => {
child.once('close', (code, signal) => { resolve({ code, signal }) })
})
child.kill('SIGKILL')
const exit = await closed
expect(exit).toEqual({ code: null, signal: 'SIGKILL' })
return { root, markerText }
} catch (error: unknown) {
if (child.exitCode === null && child.signalCode === null) child.kill('SIGKILL')
throw new Error(`crash child failed: ${stderr}`, { cause: error })
}
}
async function load(root: string): Promise<SessionEvent[]> {
const ctx = new Context()
await ctx.plugin(SessionStore)
await ctx.plugin(SessionPersistenceJsonl, { root, compression: 'none' })
try {
return (await ctx.sessionPersistence.load(sessionId)).events
} finally {
await ctx.fiber.dispose()
}
}
afterEach(async () => {
await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true })))
})
describe.skipIf(process.platform === 'win32')('semantic checkpoint hard-crash recovery', () => {
it('persists the complete request before model dispatch', async () => {
const crashed = await crashAt('request')
expect(crashed.markerText).toBe('request-dispatched')
const events = await load(crashed.root)
expect(events.map(event => event.type)).toEqual([
'turn/start', 'user/message', 'step/start', 'request/header', 'step/end', 'turn/end',
])
expect(events.at(-1)).toMatchObject({
type: 'turn/end', data: { reason: { kind: 'interrupted' } },
})
})
it('persists tool intent before a side effect and repairs its missing result as unknown', async () => {
const crashed = await crashAt('tool')
expect(crashed.markerText).toBe('tool-side-effect')
const events = await load(crashed.root)
expect(events.some(event => event.type === 'assistant/message')).toBe(true)
expect(events.some(event => event.type === 'tool/call')).toBe(true)
const result = events.find(event => event.type === 'tool/result')
expect(result?.type === 'tool/result' && result.data.error).toEqual({
name: 'ToolOutcomeUnknownError', code: TOOL_OUTCOME_UNKNOWN,
})
if (result?.type !== 'tool/result' || result.data.content[0]?.type !== 'text') {
throw new Error('expected a text tool result')
}
expect(result.data.content[0].text).toContain('Do not retry blindly.')
})
})

View File

@@ -0,0 +1,59 @@
import { writeFile } from 'node:fs/promises'
import { Context } from 'cordis'
import AgentLoop from '@deepseek-ai/dsh-agent-loop'
import { mountAgentLoopTestDependencies } from '@deepseek-ai/dsh-agent-loop-testkit'
import { CallId, type GenerateOptions, LlmAdapter, type StreamChunk } from '@deepseek-ai/dsh-llm'
import { SessionId } from '@deepseek-ai/dsh-session'
import SessionPersistenceJsonl from '@deepseek-ai/dsh-session-persistence-jsonl'
import * as checkpointPolicy from '../../src/index.ts'
function waitForCrash(): Promise<never> {
return new Promise(() => { setInterval(() => {}, 60_000) })
}
const [mode, root, marker] = process.argv.slice(2)
if ((mode !== 'request' && mode !== 'tool') || root === undefined || marker === undefined) {
throw new Error('usage: crash-child.ts <request|tool> <persistence-root> <marker>')
}
const persistenceRoot = root
const failpoint = marker
class CrashAdapter extends LlmAdapter {
async * stream(_options: GenerateOptions): AsyncIterable<StreamChunk> {
if (mode === 'request') {
await writeFile(failpoint, 'request-dispatched')
await waitForCrash()
return
}
yield { type: 'block-start', index: 0, blockType: 'tool-call' }
yield {
type: 'block-end',
index: 0,
block: { type: 'tool-call', id: CallId('crash-call'), name: 'crash_tool', arguments: '{}' },
}
yield { type: 'finish', reason: { kind: 'tool-calls' } }
}
}
const ctx = new Context()
await mountAgentLoopTestDependencies(ctx)
await ctx.plugin(AgentLoop, { agents: [] })
await ctx.plugin(SessionPersistenceJsonl, { root: persistenceRoot, compression: 'none' })
await ctx.plugin(checkpointPolicy)
ctx.llm.registerAdapter(['crash'], new CrashAdapter())
ctx.tools.register({
name: 'crash_tool',
description: 'records an external effect and never returns',
parameters: {},
async execute() {
await writeFile(failpoint, 'tool-side-effect')
return waitForCrash()
},
})
const handle = await ctx.agents.create({
sessionId: SessionId('semantic-checkpoint-crash'),
agentOptions: { provider: 'crash', model: 'crash' },
})
handle.agent.send([{ type: 'text', text: 'exercise the crash boundary' }])
await waitForCrash()

View File

@@ -0,0 +1,212 @@
import { afterEach, describe, expect, it } from 'vitest'
import { Context } from 'cordis'
import Loader from '@cordisjs/plugin-loader'
import { agentEvents, type Agent } from '@deepseek-ai/dsh-agent'
import LlmService, { CallId, type GenerateOptions, LlmAdapter, type StreamChunk } from '@deepseek-ai/dsh-llm'
import SessionStore, { SessionId } from '@deepseek-ai/dsh-session'
import type { SessionEvent, SessionHeader } from '@deepseek-ai/dsh-session'
import SessionPersistence from '@deepseek-ai/dsh-session-persistence'
import SystemPrompt from '@deepseek-ai/dsh-system-prompt'
import ToolRegistry from '@deepseek-ai/dsh-tools'
import * as checkpointPolicy from '../src/index.ts'
const contexts: Context[] = []
class TestPersistence extends SessionPersistence {
locate(_meta: SessionHeader): undefined { return undefined }
create(_meta: SessionHeader): Promise<void> { return Promise.resolve() }
append(_id: SessionId, _events: readonly SessionEvent[]): Promise<void> { return Promise.resolve() }
load(_id: SessionId): Promise<{ meta: SessionHeader; events: SessionEvent[] }> {
return Promise.reject(new Error('not used'))
}
list(): Promise<SessionHeader[]> { return Promise.resolve([]) }
}
class RecordingAdapter extends LlmAdapter {
constructor(private readonly order: string[]) { super() }
async * stream(_options: GenerateOptions): AsyncIterable<StreamChunk> {
this.order.push('adapter')
yield { type: 'finish', reason: { kind: 'stop' } }
}
}
async function setup(): Promise<Context> {
const ctx = new Context()
contexts.push(ctx)
await ctx.plugin(SessionStore)
await ctx.plugin(LlmService)
await ctx.plugin(SystemPrompt)
await ctx.plugin(ToolRegistry)
await ctx.plugin(TestPersistence)
await ctx.plugin(checkpointPolicy)
return ctx
}
async function drain(stream: AsyncIterable<StreamChunk>): Promise<void> {
for await (const _chunk of stream) { /* drain */ }
}
afterEach(async () => {
await Promise.all(contexts.splice(0).map(ctx => ctx.fiber.dispose()))
})
describe('session-checkpoint-policy request boundary', () => {
it('awaits the live session checkpoint before constructing the downstream model stream', async () => {
const ctx = await setup()
const session = ctx.sessions.create(SessionId('request-checkpoint'))
session.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } })
const gate = Promise.withResolvers<undefined>()
const order: string[] = []
ctx.on('session/flush', async () => {
order.push('flush:start')
await gate.promise
order.push('flush:end')
})
ctx.llm.registerAdapter(['mock'], new RecordingAdapter(order))
const pending = drain(ctx.llm.stream({
provider: 'mock', model: 'mock', messages: [], sessionId: session.id,
}))
await Promise.resolve()
expect(order).toEqual(['flush:start'])
gate.resolve(undefined)
await pending
expect(order).toEqual(['flush:start', 'flush:end', 'adapter'])
})
it('delegates a request without a live session without checkpointing', async () => {
const ctx = await setup()
const order: string[] = []
ctx.on('session/flush', () => { order.push('flush') })
ctx.llm.registerAdapter(['mock'], new RecordingAdapter(order))
await drain(ctx.llm.stream({ provider: 'mock', model: 'mock', messages: [] }))
expect(order).toEqual(['adapter'])
})
it('delegates an already-detached session id without checkpointing', async () => {
const ctx = await setup()
const order: string[] = []
ctx.on('session/flush', () => { order.push('flush') })
ctx.llm.registerAdapter(['mock'], new RecordingAdapter(order))
await drain(ctx.llm.stream({
provider: 'mock', model: 'mock', messages: [], sessionId: SessionId('detached'),
}))
expect(order).toEqual(['adapter'])
})
it('does not dispatch the adapter when the checkpoint rejects', async () => {
const ctx = await setup()
const session = ctx.sessions.create(SessionId('request-failure'))
const order: string[] = []
ctx.on('session/flush', () => Promise.reject(new Error('disk unavailable')))
ctx.llm.registerAdapter(['mock'], new RecordingAdapter(order))
await expect(drain(ctx.llm.stream({
provider: 'mock', model: 'mock', messages: [], sessionId: session.id,
}))).rejects.toThrow('disk unavailable')
expect(order).toEqual([])
})
})
describe('session-checkpoint-policy tool and step boundaries', () => {
it('awaits the checkpoint before a top-level tool body', async () => {
const ctx = await setup()
const session = ctx.sessions.create(SessionId('tool-checkpoint'))
const agent = { session } as Agent
const gate = Promise.withResolvers<undefined>()
const order: string[] = []
ctx.on('session/flush', async () => {
order.push('flush:start')
await gate.promise
order.push('flush:end')
})
ctx.tools.register({
name: 'write', description: 'side effect', parameters: {},
execute: async () => { order.push('tool'); return [] },
})
const pending = ctx.tools.execute({
callId: CallId('write-1'), name: 'write', arguments: {}, agent,
})
await Promise.resolve()
expect(order).toEqual(['flush:start'])
gate.resolve(undefined)
await expect(pending).resolves.toMatchObject({ isError: false })
expect(order).toEqual(['flush:start', 'flush:end', 'tool'])
})
it('turns a rejected checkpoint into an error result without running the tool body', async () => {
const ctx = await setup()
const session = ctx.sessions.create(SessionId('tool-failure'))
const agent = { session } as Agent
let ran = false
ctx.on('session/flush', () => Promise.reject(new Error('disk unavailable')))
ctx.tools.register({
name: 'write', description: 'side effect', parameters: {},
execute: async () => { ran = true; return [] },
})
const result = await ctx.tools.execute({
callId: CallId('write-2'), name: 'write', arguments: {}, agent,
})
expect(result.isError).toBe(true)
expect(result.content).toEqual([{ type: 'text', text: 'Error: disk unavailable' }])
expect(ran).toBe(false)
})
it('reuses the outer checkpoint for a nested tool dispatch', async () => {
const ctx = await setup()
const session = ctx.sessions.create(SessionId('nested-tool'))
const agent = { session } as Agent
let flushes = 0
ctx.on('session/flush', () => { flushes += 1 })
ctx.tools.register({ name: 'nested', description: 'nested', parameters: {}, execute: async () => [] })
await ctx.tools.execute({
callId: CallId('nested-1'), name: 'nested', arguments: {}, agent,
parent: Symbol('outer') as never,
})
expect(flushes).toBe(0)
})
it('checkpoints the complete recorded step at agent/post-step', async () => {
const ctx = await setup()
const session = ctx.sessions.create(SessionId('post-step'))
const agent = { session } as Agent
const flushed: string[] = []
ctx.on('session/flush', (current) => { flushed.push(current.id) })
await agentEvents(ctx, agent).serial(
'agent/post-step', 1, 1, new AbortController().signal,
)
expect(flushed).toEqual([session.id])
})
})
describe('session-checkpoint-policy lifecycle', () => {
it('removes its wrappers when the owning fiber is disposed', async () => {
const ctx = new Context()
contexts.push(ctx)
await ctx.plugin(SessionStore)
await ctx.plugin(LlmService)
await ctx.plugin(SystemPrompt)
await ctx.plugin(ToolRegistry)
await ctx.plugin(TestPersistence)
const session = ctx.sessions.create(SessionId('disposed-policy'))
let flushes = 0
ctx.on('session/flush', () => { flushes += 1 })
ctx.llm.registerAdapter(['mock'], new RecordingAdapter([]))
const fiber = await ctx.plugin(checkpointPolicy)
await drain(ctx.llm.stream({ provider: 'mock', model: 'mock', messages: [], sessionId: session.id }))
expect(flushes).toBe(1)
await fiber.dispose()
await drain(ctx.llm.stream({ provider: 'mock', model: 'mock', messages: [], sessionId: session.id }))
expect(flushes).toBe(1)
})
it('keeps the Loader-safe namespace plugin shape', () => {
expect('default' in checkpointPolicy).toBe(false)
const loader = Object.create(Loader.prototype) as Loader
const unwrapped = loader.unwrapExports(checkpointPolicy) as Record<string, unknown>
expect(unwrapped).toBe(checkpointPolicy)
expect(unwrapped.name).toBe('session-checkpoint-policy')
expect(unwrapped.inject).toEqual(['llm', 'sessionPersistence', 'sessions', 'tools'])
expect(typeof unwrapped.apply).toBe('function')
})
})

View File

@@ -0,0 +1,33 @@
{
"extends": "../../../tsconfig.base.json",
"compilerOptions": {
"rootDir": "src",
"outDir": "lib/types"
},
"include": [
"src"
],
"references": [
{
"path": "../../../vendor/cosmokit"
},
{
"path": "../../../vendor/cordis"
},
{
"path": "../../core/agent"
},
{
"path": "../../llm/llm"
},
{
"path": "../../core/session"
},
{
"path": "../../session-persistence/session-persistence"
},
{
"path": "../../core/tools"
}
]
}