Coder/deepseek-harness
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: implement bounded LLM request recovery

Browse Source
This commit is contained in:
Tianyi Cui
2026-07-20 03:34:19 +08:00
parent 7cf966fc0e
commit 3b0b0cefeb
115 changed files with 3311 additions and 366 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
packages/core/agent-loop/tests/contract-regressions.spec.ts
View File

@@ -1,6 +1,6 @@
import { describe, expect, it } from 'vitest'
import { Context } from 'cordis'
import LlmService, { CallId, ContentBlock, MessageSource, StreamChunk } from '@deepseek-ai/dsh-llm'
import LlmService, { CallId, ContentBlock, MessageSource, ProviderRequestId, StreamChunk } from '@deepseek-ai/dsh-llm'
import SessionStore, { Session, SessionEvent, SessionId, TurnEndReason } from '@deepseek-ai/dsh-session'
import SystemPrompt from '@deepseek-ai/dsh-system-prompt'
import ToolRegistry, { defineTool, type PostToolDecision } from '@deepseek-ai/dsh-tools'
@@ -925,8 +925,15 @@ describe('discriminated SessionEvent narrows without casts', () => {
describe('a finish-error stream chunk ends the turn as error, not completed', () => {
it('translates finish {kind:error} into a turn error with a logged error event', async () => {
// A finish-error chunk must not produce a completed assistant turn.
const failure = {
message: 'provider 401',
code: 'AUTH',
status: 401,
retryAfterMs: 2_000,
requestId: ProviderRequestId('finish-request-1'),
}
const errorStream: StreamChunk[] = [
{ type: 'finish', reason: { kind: 'error', message: 'provider 401', code: 'AUTH' } },
{ type: 'finish', reason: { kind: 'error', failure } },
]
const adapter = new MockAdapter([errorStream])
const ctx = await harness(adapter)
@@ -938,20 +945,20 @@ describe('a finish-error stream chunk ends the turn as error, not completed', ()
send(agent, 'go')
await waitForIdle(ctx, agent)
expect(reasons).toEqual([{ kind: 'error', step: 1, message: 'provider 401', code: 'AUTH' }])
expect(reasons).toEqual([{ kind: 'error', step: 1, failure }])
const events = [...agent.session.events]
// The durable failure lives on turn/end.reason (with the failing step), not
// a standalone error event.
const turnEnd = events.find(event => event.type === 'turn/end')
expect(turnEnd?.type === 'turn/end' && turnEnd.data.reason).toEqual({ kind: 'error', step: 1, message: 'provider 401', code: 'AUTH' })
expect(turnEnd?.type === 'turn/end' && turnEnd.data.reason).toEqual({ kind: 'error', step: 1, failure })
// A failed step must not synthesize an assistant message.
expect(events.some(event => event.type === 'assistant/message')).toBe(false)
})
it('translates finish {kind:aborted} into a turn error coded ABORTED', async () => {
const abortedStream: StreamChunk[] = [
{ type: 'finish', reason: { kind: 'aborted' } },
{ type: 'finish', reason: { kind: 'aborted', failure: { message: 'model stream aborted', code: 'ABORTED' } } },
]
const adapter = new MockAdapter([abortedStream])
const ctx = await harness(adapter)
@@ -963,13 +970,13 @@ describe('a finish-error stream chunk ends the turn as error, not completed', ()
send(agent, 'go')
await waitForIdle(ctx, agent)
expect(reasons).toEqual([{ kind: 'error', step: 1, message: 'model stream aborted', code: 'ABORTED' }])
expect(reasons).toEqual([{ kind: 'error', step: 1, failure: { message: 'model stream aborted', code: 'ABORTED' } }])
expect([...agent.session.events].some(event => event.type === 'assistant/message')).toBe(false)
})
it('handles a finish error without a code (code key omitted)', async () => {
const errorStream: StreamChunk[] = [
{ type: 'finish', reason: { kind: 'error', message: 'codeless failure' } },
{ type: 'finish', reason: { kind: 'error', failure: { message: 'codeless failure', code: 'UNKNOWN' } } },
]
const adapter = new MockAdapter([errorStream])
const ctx = await harness(adapter)
@@ -981,7 +988,7 @@ describe('a finish-error stream chunk ends the turn as error, not completed', ()
send(agent, 'go')
await waitForIdle(ctx, agent)
expect(reasons).toEqual([{ kind: 'error', step: 1, message: 'codeless failure' }])
expect(reasons).toEqual([{ kind: 'error', step: 1, failure: { message: 'codeless failure', code: 'UNKNOWN' } }])
})
})
@@ -1101,7 +1108,7 @@ describe('turn and step boundary recovery', () => {
})
it('a one-shot turn/end validation failure preserves the earlier turn error on retry', async () => {
const errorStream: StreamChunk[] = [{ type: 'finish', reason: { kind: 'error', message: 'provider failed' } }]
const errorStream: StreamChunk[] = [{ type: 'finish', reason: { kind: 'error', failure: { message: 'provider failed', code: 'UNKNOWN' } } }]
const adapter = new MockAdapter([errorStream])
const ctx = await balancedHarness(adapter)
const agent = ctx.agentLoop.create(SessionId('a-turnend-veto'), { provider: 'mock', model: 'mock' })
@@ -1131,7 +1138,7 @@ describe('turn and step boundary recovery', () => {
const turnEnd = agent.session.events.findLast(event => event.type === 'turn/end')
expect(turnEnd?.type === 'turn/end' && turnEnd.data.reason).toMatchObject({
kind: 'error',
message: 'provider failed',
failure: { message: 'provider failed', code: 'UNKNOWN' },
})
})
@@ -1167,7 +1174,7 @@ describe('turn and step boundary recovery', () => {
it('a throwing agent/error listener during a step-error path still balances the turn, loop survives', async () => {
// Listener failure cannot interrupt error finalization or the next turn.
const errorStream: StreamChunk[] = [{ type: 'finish', reason: { kind: 'error', message: 'provider 500' } }]
const errorStream: StreamChunk[] = [{ type: 'finish', reason: { kind: 'error', failure: { message: 'provider 500', code: 'SERVER' } } }]
const adapter = new MockAdapter([errorStream, textResponse('turn 2 ok')])
const ctx = await balancedHarness(adapter)
const agent = ctx.agentLoop.create(SessionId('a-errorlistener'), { provider: 'mock', model: 'mock' })
@@ -1183,7 +1190,11 @@ describe('turn and step boundary recovery', () => {
expect(c.turnStart).toBe(1)
expect(c.turnEnd).toBe(1)
expect(c.stepStart).toBe(c.stepEnd)
expect(c.lastTurnEnd?.type === 'turn/end' && c.lastTurnEnd.data.reason).toMatchObject({ kind: 'error', step: 1, message: 'provider 500' })
expect(c.lastTurnEnd?.type === 'turn/end' && c.lastTurnEnd.data.reason).toMatchObject({
kind: 'error',
step: 1,
failure: { message: 'provider 500', code: 'SERVER' },
})
// loop survives: a second turn runs to completion (invariants oracle would
// throw on its turn/start if turn 1 had been left open).
@@ -1328,7 +1339,7 @@ describe('turn and step boundary recovery', () => {
it('a throwing step/end observer cannot interrupt error finalization', async () => {
// Observer failure after step/end commit cannot interrupt turn finalization.
const errorStream: StreamChunk[] = [{ type: 'finish', reason: { kind: 'error', message: 'provider 500' } }]
const errorStream: StreamChunk[] = [{ type: 'finish', reason: { kind: 'error', failure: { message: 'provider 500', code: 'SERVER' } } }]
const adapter = new MockAdapter([errorStream, textResponse('turn 2 ok')])
const ctx = await harness(adapter)
const agent = ctx.agentLoop.create(SessionId('a-stependthrow'), { provider: 'mock', model: 'mock' })

7
packages/core/agent-loop/tests/coverage-edges.spec.ts
View File

@@ -177,7 +177,9 @@ describe('toError normalization', () => {
// String() of { code: 500 } is '[object Object]'
expect(errors[0]!.message).toBe('[object Object]')
const turnEnd = agent.session.events.find(e => e.type === 'turn/end')
expect(turnEnd?.type === 'turn/end' && turnEnd.data.reason.kind === 'error' && turnEnd.data.reason.code).toBe('UNKNOWN')
expect(turnEnd?.type === 'turn/end' && turnEnd.data.reason.kind === 'error'
&& ('failure' in turnEnd.data.reason ? turnEnd.data.reason.failure.code : turnEnd.data.reason.code))
.toBe('UNKNOWN')
})
})
@@ -208,7 +210,8 @@ describe('coded error data emission', () => {
const turnEnd = agent.session.events.find(e => e.type === 'turn/end')
expect(turnEnd).toBeDefined()
if (turnEnd?.type === 'turn/end' && turnEnd.data.reason.kind === 'error') {
expect(turnEnd.data.reason.code).toBe('RATE_LIMIT')
expect('failure' in turnEnd.data.reason ? turnEnd.data.reason.failure.code : turnEnd.data.reason.code)
.toBe('RATE_LIMIT')
}
})
})

91
packages/core/agent-loop/tests/request-recovery.spec.ts
View File

@@ -5,8 +5,9 @@ import LlmService, {
CONTEXT_WINDOW_EXCEEDED_CODE,
LlmAdapter,
LlmError,
ProviderRequestId,
} from '@deepseek-ai/dsh-llm'
import type { GenerateOptions, StreamChunk } from '@deepseek-ai/dsh-llm'
import type { GenerateOptions, LlmFailure, StreamChunk } from '@deepseek-ai/dsh-llm'
import SessionStore, { SessionId } from '@deepseek-ai/dsh-session'
import SystemPrompt from '@deepseek-ai/dsh-system-prompt'
import ToolRegistry, { defineTool } from '@deepseek-ai/dsh-tools'
@@ -258,16 +259,17 @@ describe('agent post-step and request-error lifecycle', () => {
it.each([
['thrown', contextError()],
['in-band', [{ type: 'finish', reason: { kind: 'error', message: 'too large', code: CONTEXT_WINDOW_EXCEEDED_CODE } }] satisfies StreamChunk[]],
['in-band', [{ type: 'finish', reason: { kind: 'error', failure: { message: 'too large', code: CONTEXT_WINDOW_EXCEEDED_CODE, status: 400 } } }] satisfies StreamChunk[]],
] as const)('recovers a %s request failure in a new reconstructable step', async (_style, failure) => {
const adapter = new FailureScriptAdapter([failure, textResponse('recovered')])
const ctx = await harness(adapter)
const agent = ctx.agentLoop.create(SessionId(`recover-${_style}`), { provider: 'mock', model: 'mock' })
const attempts: number[] = []
ctx.on('agent/request-error', async (subject, turn, step, error, attempt) => {
ctx.on('agent/request-error', async (subject, turn, step, error, facts, history) => {
expect(subject).toBe(agent)
expect({ turn, step, code: error.code }).toEqual({ turn: 1, step: 1, code: CONTEXT_WINDOW_EXCEEDED_CODE })
attempts.push(attempt)
expect(facts.code).toBe(CONTEXT_WINDOW_EXCEEDED_CODE)
attempts.push(history.length)
subject.session.append('context/message', {
content: [{ type: 'text', text: 'RECOVERY SURFACE MUTATION' }],
source: { kind: 'plugin', plugin: 'test-recovery' },
@@ -295,7 +297,7 @@ describe('agent post-step and request-error lifecycle', () => {
const agent = ctx.agentLoop.create(SessionId(`stream-plugin-${_name.replaceAll(' ', '-')}`), { provider: 'mock', model: 'mock' })
let recoveries = 0
install(ctx)
ctx.on('agent/request-error', async (_agent, _turn, _step, _error, _attempt, _signal, next) => {
ctx.on('agent/request-error', async (_agent, _turn, _step, _error, _failure, _history, _signal, next) => {
recoveries += 1
return next()
})
@@ -326,7 +328,7 @@ describe('agent post-step and request-error lifecycle', () => {
})
const agent = ctx.agentLoop.create(SessionId('nested-stream-not-recoverable'), { provider: 'mock', model: 'mock' })
let recoveries = 0
ctx.on('agent/request-error', async (_agent, _turn, _step, _error, _attempt, _signal, next) => {
ctx.on('agent/request-error', async (_agent, _turn, _step, _error, _failure, _history, _signal, next) => {
recoveries += 1
return next()
})
@@ -359,7 +361,7 @@ describe('agent post-step and request-error lifecycle', () => {
}
const agent = ctx.agentLoop.create(SessionId(`${boundary}-not-recoverable`), { provider: 'mock', model: 'mock' })
let recoveries = 0
ctx.on('agent/request-error', async (_agent, _turn, _step, _error, _attempt, _signal, next) => {
ctx.on('agent/request-error', async (_agent, _turn, _step, _error, _failure, _history, _signal, next) => {
recoveries += 1
return next()
})
@@ -387,7 +389,7 @@ describe('agent post-step and request-error lifecycle', () => {
}
const agent = ctx.agentLoop.create(SessionId(`${failure}-not-recoverable`), { provider: 'mock', model: 'mock' })
let recoveries = 0
ctx.on('agent/request-error', async (_agent, _turn, _step, _error, _attempt, _signal, next) => {
ctx.on('agent/request-error', async (_agent, _turn, _step, _error, _failure, _history, _signal, next) => {
recoveries += 1
return next()
})
@@ -406,7 +408,7 @@ describe('agent post-step and request-error lifecycle', () => {
const ctx = await harness(makeAdapter(original))
const agent = ctx.agentLoop.create(SessionId(`identity-${_name.replaceAll(' ', '-')}`), { provider: 'mock', model: 'mock' })
let seen: Error | undefined
ctx.on('agent/request-error', async (_agent, _turn, _step, error, _attempt, _signal, next) => {
ctx.on('agent/request-error', async (_agent, _turn, _step, error, _failure, _history, _signal, next) => {
seen = error
return next()
})
@@ -417,12 +419,52 @@ describe('agent post-step and request-error lifecycle', () => {
expect(seen).toBe(original)
})
it('passes structured facts beside the original Error and records them on exhaustion', async () => {
const original = new LlmError('provider busy', 'RATE_LIMIT', {
status: 429,
retryAfterMs: 2_000,
requestId: ProviderRequestId('req-9'),
})
Object.freeze(original)
const ctx = await harness(new SynchronousDispatchFailureAdapter(original))
const agent = ctx.agentLoop.create(SessionId('structured-request-failure'), { provider: 'mock', model: 'mock' })
let seenError: Error | undefined
let seenFailure: LlmFailure | undefined
let seenHistory: readonly LlmFailure[] | undefined
ctx.on('agent/request-error', async (
_agent, _turn, _step, error, failure, history, _signal, next,
) => {
seenError = error
seenFailure = failure
seenHistory = history
return next()
})
send(agent)
await waitForIdle(ctx, agent)
expect(seenError).toBe(original)
expect(seenFailure).toEqual({
message: 'provider busy',
code: 'RATE_LIMIT',
status: 429,
retryAfterMs: 2_000,
requestId: ProviderRequestId('req-9'),
})
expect(seenHistory).toEqual([])
expect(Object.isFrozen(seenHistory)).toBe(true)
expect(agent.session.events.at(-1)).toMatchObject({
type: 'turn/end',
data: { reason: { kind: 'error', step: 1, failure: seenFailure } },
})
})
it('classifies iterator construction and explicit NO_ADAPTER as model-request failures', async () => {
for (const scenario of ['iterator', 'no-adapter'] as const) {
const ctx = scenario === 'iterator' ? await harness(new IteratorConstructionFailureAdapter()) : await harness()
const agent = ctx.agentLoop.create(SessionId(`request-boundary-${scenario}`), { provider: 'mock', model: 'mock' })
let seen = ''
ctx.on('agent/request-error', async (_agent, _turn, _step, error, _attempt, _signal, next) => {
ctx.on('agent/request-error', async (_agent, _turn, _step, error, _failure, _history, _signal, next) => {
seen = error.code ?? ''
return next()
})
@@ -436,14 +478,17 @@ describe('agent post-step and request-error lifecycle', () => {
const capped = new FailureScriptAdapter([contextError('first overflow'), contextError('second overflow')])
const cappedCtx = await harness(capped)
const cappedAgent = cappedCtx.agentLoop.create(SessionId('retry-cap'), { provider: 'mock', model: 'mock' })
const cappedAttempts: number[] = []
cappedCtx.on('agent/request-error', async (_agent, _turn, _step, _error, attempt, _signal, next) => {
cappedAttempts.push(attempt)
return attempt < 1 ? { action: 'retry' } : next()
const cappedHistories: string[][] = []
cappedCtx.on('agent/request-error', async (
_agent, _turn, _step, _error, _failure, history, _signal, next,
) => {
const codes = history.map(entry => entry.code)
cappedHistories.push(codes)
return codes.length < 1 ? { action: 'retry' } : next()
})
send(cappedAgent)
await waitForIdle(cappedCtx, cappedAgent)
expect(cappedAttempts).toEqual([0, 1])
expect(cappedHistories).toEqual([[], [CONTEXT_WINDOW_EXCEEDED_CODE]])
const reset = new FailureScriptAdapter([
contextError('first overflow'),
@@ -458,14 +503,16 @@ describe('agent post-step and request-error lifecycle', () => {
async execute() { return [{ type: 'text', text: 'worked' }] },
}))
const resetAgent = resetCtx.agentLoop.create(SessionId('retry-reset'), { provider: 'mock', model: 'mock' })
const resetAttempts: { step: number; attempt: number }[] = []
resetCtx.on('agent/request-error', async (_agent, _turn, step, _error, attempt, _signal, next) => {
resetAttempts.push({ step, attempt })
return resetAttempts.length === 1 ? { action: 'retry' } : next()
const resetHistories: { step: number; codes: string[] }[] = []
resetCtx.on('agent/request-error', async (
_agent, _turn, step, _error, _failure, history, _signal, next,
) => {
resetHistories.push({ step, codes: history.map(entry => entry.code) })
return resetHistories.length === 1 ? { action: 'retry' } : next()
})
send(resetAgent)
await waitForIdle(resetCtx, resetAgent)
expect(resetAttempts).toEqual([{ step: 1, attempt: 0 }, { step: 3, attempt: 0 }])
expect(resetHistories).toEqual([{ step: 1, codes: [] }, { step: 3, codes: [] }])
})
it('preserves the original provider error when recovery throws', async () => {
@@ -479,7 +526,7 @@ describe('agent post-step and request-error lifecycle', () => {
expect(agent.session.events.at(-1)).toMatchObject({
type: 'turn/end',
data: { reason: { kind: 'error', message: 'original overflow', code: CONTEXT_WINDOW_EXCEEDED_CODE } },
data: { reason: { kind: 'error', failure: { message: 'original overflow', code: CONTEXT_WINDOW_EXCEEDED_CODE } } },
})
})
@@ -489,7 +536,7 @@ describe('agent post-step and request-error lifecycle', () => {
const agent = ctx.agentLoop.create(SessionId(`${action}-recovery`), { provider: 'mock', model: 'mock' })
let entered!: () => void
const recoveryEntered = new Promise<void>((resolve) => { entered = resolve })
ctx.on('agent/request-error', async (_agent, _turn, _step, _error, _attempt, signal) => {
ctx.on('agent/request-error', async (_agent, _turn, _step, _error, _failure, _history, signal) => {
entered()
await new Promise<void>((resolve) => {
signal.addEventListener('abort', () => { resolve() }, { once: true })