feat: implement bounded LLM request recovery
This commit is contained in:
@@ -54,7 +54,7 @@ The driver owns one agent for its lifetime and runs inside `ctx.agents.withIniti
|
||||
|
||||
Every provider call that reaches a successful finish appends exactly one `assistant/message` completion anchor, including content-less calls and `max-tokens` finishes. A successful `agent/step-result` stores its transformed content; a rejected result records empty content before the original failure continues. The anchor retains exact chunk provenance (`[]` for a stream with no chunks) and usage when available, while empty content stays out of derived message history.
|
||||
|
||||
Plugin failure ends the current turn, not the loop. Only final adapter dispatch/iteration failures and terminal in-band error or aborted finishes enter `agent/request-error`; middleware, result processing, tools, and `agent/post-step` remain ordinary turn failures. Recovery observes a closed failed step, and a retry rebuilds the request from the durable log in a new numbered step. Cancellation clears pending work and aborts the current step without leaking to the next prompt; undispatched model tool calls receive synthetic `tool/call` and aborted result pairs. Terminal continuation stops remain authoritative through turn close and durability flush.
|
||||
Plugin failure ends the current turn, not the loop. Only final adapter dispatch/iteration failures and terminal in-band error or aborted finishes enter `agent/request-error`; middleware, result processing, tools, and `agent/post-step` remain ordinary turn failures. Recovery receives the exact live error, immutable provider facts, and immutable prior failures after the failed step closes. A retry rebuilds from the durable log in a new numbered step, success clears the consecutive history, and exhaustion records the structured failure once on `turn/end`. Cancellation clears pending work and aborts the current step without leaking to the next prompt; undispatched model tool calls receive synthetic `tool/call` and aborted result pairs. Terminal continuation stops remain authoritative through turn close and durability flush.
|
||||
|
||||
Within a step, exclusive calls form barriers; parallel-safe calls use a bounded rolling pool and are reclassified before start. Only dispatch/body overlaps. Policy, durable results, and result context remain model-ordered. Abort stops new calls, drains started results, then drains accepted batch context before the turn closes through the normal abort path.
|
||||
|
||||
@@ -63,6 +63,7 @@ Within a step, exclusive calls form barriers; parallel-safe calls use a bounded
|
||||
Everything that goes beyond "call the model, run the tools, repeat" belongs to plugins listening on the event taxonomy:
|
||||
- Hooks and policy: the relevant `agent/*` checkpoints plus the guarded `tools/pre-execute` → `tools/execute` → `tools/post-execute` → `tools/result` pipeline; exact signatures and modes live in the [generated event catalog](../../../docs/cordis-catalog/events.md)
|
||||
- Compaction: pressure on `agent/post-step`; canonical context overflow on `agent/request-error`
|
||||
- Transient model recovery: `dsh-llm-retry` on `agent/request-error`, with finite code-specific budgets and non-surface `llm/retry` status events
|
||||
- Sandbox, permission, plan mode: `tools/pre-execute` for extensible deny/ask, `tools.guard()` for monotonic owner policy, `tools/post-execute` for result decisions, and `tools/result` for final observation
|
||||
- Sub-agents: implemented outside the loop as `ctx.subagents` providers; in-process providers use `ctx.agents.create()` and owned `AgentHandle` teardown, while generic [`ctx.tasks`](../../tasks/tasks/) plus [`dsh-tool-subagent`](../../subagent/tool-subagent/) own background collection.
|
||||
- Persistence: `session/event` + `session/flush`
|
||||
|
||||
@@ -6,9 +6,9 @@
|
||||
*/
|
||||
|
||||
import type { Context } from 'cordis'
|
||||
import type { ContentBlock, FinishReason, GenerateOptions, LlmCallConfig, Message } from '@deepseek-ai/dsh-llm'
|
||||
import type { ContentBlock, FinishReason, GenerateOptions, LlmCallConfig, LlmFailure, Message } from '@deepseek-ai/dsh-llm'
|
||||
import { isDeepStrictEqual } from 'node:util'
|
||||
import { BlockAssembler, HarnessError, assertNever, deepFreeze, isLlmAdapterFailure } from '@deepseek-ai/dsh-llm'
|
||||
import { BlockAssembler, HarnessError, LlmError, assertNever, deepFreeze, llmFailureOf } from '@deepseek-ai/dsh-llm'
|
||||
import { agentEvents, assembleContextFor } from '@deepseek-ai/dsh-agent'
|
||||
import type { AgentEventDispatch, ContinuationDecision, HookContext, PromptDecision, RequestError, RequestErrorDecision } from '@deepseek-ai/dsh-agent'
|
||||
import { canonicalHeader } from '@deepseek-ai/dsh-session'
|
||||
@@ -28,24 +28,27 @@ function toError(error: unknown): RequestError {
|
||||
|
||||
/** Distinguishes final model-request failures from failures in later step processing. */
|
||||
class TerminalModelRequestFailure extends Error {
|
||||
constructor(readonly requestError: RequestError) {
|
||||
constructor(
|
||||
readonly requestError: RequestError,
|
||||
readonly failure: LlmFailure,
|
||||
) {
|
||||
super(requestError.message, { cause: requestError })
|
||||
this.name = 'TerminalModelRequestFailure'
|
||||
}
|
||||
}
|
||||
|
||||
/** Convert terminal failure finishes into step errors; unknown extensible finishes remain successful. */
|
||||
function finishError(finish: FinishReason): RequestError | undefined {
|
||||
function finishError(finish: FinishReason): { error: RequestError; failure: LlmFailure } | undefined {
|
||||
switch (finish.kind) {
|
||||
case 'error': {
|
||||
const error: RequestError = new Error(finish.message)
|
||||
if (finish.code !== undefined) error.code = finish.code
|
||||
return error
|
||||
}
|
||||
case 'error':
|
||||
case 'aborted': {
|
||||
const error: RequestError = new Error('model stream aborted')
|
||||
error.code = 'ABORTED'
|
||||
return error
|
||||
const facts = finish.failure
|
||||
const error = new LlmError(facts.message, facts.code, {
|
||||
...facts.status === undefined ? {} : { status: facts.status },
|
||||
...facts.retryAfterMs === undefined ? {} : { retryAfterMs: facts.retryAfterMs },
|
||||
...facts.requestId === undefined ? {} : { requestId: facts.requestId },
|
||||
})
|
||||
return { error, failure: error.failure }
|
||||
}
|
||||
// stop / tool-calls / max-tokens / plugin-added kinds → not a failure.
|
||||
default:
|
||||
@@ -191,7 +194,7 @@ async function runTurn(
|
||||
|
||||
let reason: TurnEndReason = { kind: 'completed' }
|
||||
let step = 0
|
||||
let requestRetryAttempt = 0
|
||||
let requestFailureHistory: readonly LlmFailure[] = Object.freeze([])
|
||||
let stepOpen = false
|
||||
let errorReported = false
|
||||
let terminalStopped = false
|
||||
@@ -204,10 +207,12 @@ async function runTurn(
|
||||
}
|
||||
|
||||
// Record the durable turn failure once and contain the live error notification.
|
||||
const failTurn = (err: RequestError): void => {
|
||||
const failTurn = (err: RequestError, failure?: LlmFailure): void => {
|
||||
if (errorReported) return
|
||||
errorReported = true
|
||||
reason = { kind: 'error', step, ...errorData(err) }
|
||||
reason = failure === undefined
|
||||
? { kind: 'error', step, ...errorData(err) }
|
||||
: { kind: 'error', step, failure }
|
||||
try {
|
||||
events.emit('agent/error', turn, step, err)
|
||||
} catch {
|
||||
@@ -353,14 +358,14 @@ async function runTurn(
|
||||
|
||||
let stepOutcome:
|
||||
| { hadToolCalls: boolean; finish: FinishReason }
|
||||
| { requestError: RequestError }
|
||||
| { requestError: RequestError; failure: LlmFailure }
|
||||
| { error: RequestError }
|
||||
try {
|
||||
stepOutcome = await runStep(
|
||||
ctx, events, handle, turn, step, assembly, fullSystemPrompt, boundaryMessages, transmission, abort.signal)
|
||||
} catch (error: unknown) {
|
||||
if (error instanceof TerminalModelRequestFailure) {
|
||||
stepOutcome = { requestError: error.requestError }
|
||||
stepOutcome = { requestError: error.requestError, failure: error.failure }
|
||||
} else {
|
||||
stepOutcome = { error: toError(error) }
|
||||
}
|
||||
@@ -383,7 +388,7 @@ async function runTurn(
|
||||
try {
|
||||
recoveryDecision = await events.waterfall(
|
||||
'agent/request-error', turn, step, stepOutcome.requestError,
|
||||
requestRetryAttempt, abort.signal,
|
||||
stepOutcome.failure, requestFailureHistory, abort.signal,
|
||||
() => Promise.resolve(defaultDecision),
|
||||
)
|
||||
} catch (recoveryError: unknown) {
|
||||
@@ -404,10 +409,10 @@ async function runTurn(
|
||||
}
|
||||
switch (recoveryDecision.action) {
|
||||
case 'retry':
|
||||
requestRetryAttempt += 1
|
||||
requestFailureHistory = Object.freeze([...requestFailureHistory, stepOutcome.failure])
|
||||
continue
|
||||
case 'fail':
|
||||
failTurn(stepOutcome.requestError)
|
||||
failTurn(stepOutcome.requestError, stepOutcome.failure)
|
||||
break
|
||||
/* v8 ignore next -- closed-union exhaustiveness guard */
|
||||
default:
|
||||
@@ -435,7 +440,7 @@ async function runTurn(
|
||||
break
|
||||
}
|
||||
|
||||
requestRetryAttempt = 0
|
||||
requestFailureHistory = Object.freeze([])
|
||||
|
||||
// Preserve max-token completion unless a later disposal, abort, or error wins.
|
||||
const stepReason = stepFinishReason(stepOutcome.finish)
|
||||
@@ -635,13 +640,14 @@ async function runStep(
|
||||
assembler.push(chunk)
|
||||
}
|
||||
} catch (error: unknown) {
|
||||
if (isLlmAdapterFailure(stream, error)) throw new TerminalModelRequestFailure(error)
|
||||
const failure = llmFailureOf(stream, error)
|
||||
if (failure !== undefined && error instanceof Error) throw new TerminalModelRequestFailure(error, failure)
|
||||
throw error
|
||||
}
|
||||
|
||||
// Normalize failure finish chunks into the same path as thrown stream errors.
|
||||
const stepError = finishError(assembler.finish)
|
||||
if (stepError) throw new TerminalModelRequestFailure(stepError)
|
||||
if (stepError) throw new TerminalModelRequestFailure(stepError.error, stepError.failure)
|
||||
|
||||
const recordAssistantMessage = (
|
||||
assembledContent: ContentBlock[],
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { Context } from 'cordis'
|
||||
import LlmService, { CallId, ContentBlock, MessageSource, StreamChunk } from '@deepseek-ai/dsh-llm'
|
||||
import LlmService, { CallId, ContentBlock, MessageSource, ProviderRequestId, StreamChunk } from '@deepseek-ai/dsh-llm'
|
||||
import SessionStore, { Session, SessionEvent, SessionId, TurnEndReason } from '@deepseek-ai/dsh-session'
|
||||
import SystemPrompt from '@deepseek-ai/dsh-system-prompt'
|
||||
import ToolRegistry, { defineTool, type PostToolDecision } from '@deepseek-ai/dsh-tools'
|
||||
@@ -925,8 +925,15 @@ describe('discriminated SessionEvent narrows without casts', () => {
|
||||
describe('a finish-error stream chunk ends the turn as error, not completed', () => {
|
||||
it('translates finish {kind:error} into a turn error with a logged error event', async () => {
|
||||
// A finish-error chunk must not produce a completed assistant turn.
|
||||
const failure = {
|
||||
message: 'provider 401',
|
||||
code: 'AUTH',
|
||||
status: 401,
|
||||
retryAfterMs: 2_000,
|
||||
requestId: ProviderRequestId('finish-request-1'),
|
||||
}
|
||||
const errorStream: StreamChunk[] = [
|
||||
{ type: 'finish', reason: { kind: 'error', message: 'provider 401', code: 'AUTH' } },
|
||||
{ type: 'finish', reason: { kind: 'error', failure } },
|
||||
]
|
||||
const adapter = new MockAdapter([errorStream])
|
||||
const ctx = await harness(adapter)
|
||||
@@ -938,20 +945,20 @@ describe('a finish-error stream chunk ends the turn as error, not completed', ()
|
||||
send(agent, 'go')
|
||||
await waitForIdle(ctx, agent)
|
||||
|
||||
expect(reasons).toEqual([{ kind: 'error', step: 1, message: 'provider 401', code: 'AUTH' }])
|
||||
expect(reasons).toEqual([{ kind: 'error', step: 1, failure }])
|
||||
|
||||
const events = [...agent.session.events]
|
||||
// The durable failure lives on turn/end.reason (with the failing step), not
|
||||
// a standalone error event.
|
||||
const turnEnd = events.find(event => event.type === 'turn/end')
|
||||
expect(turnEnd?.type === 'turn/end' && turnEnd.data.reason).toEqual({ kind: 'error', step: 1, message: 'provider 401', code: 'AUTH' })
|
||||
expect(turnEnd?.type === 'turn/end' && turnEnd.data.reason).toEqual({ kind: 'error', step: 1, failure })
|
||||
// A failed step must not synthesize an assistant message.
|
||||
expect(events.some(event => event.type === 'assistant/message')).toBe(false)
|
||||
})
|
||||
|
||||
it('translates finish {kind:aborted} into a turn error coded ABORTED', async () => {
|
||||
const abortedStream: StreamChunk[] = [
|
||||
{ type: 'finish', reason: { kind: 'aborted' } },
|
||||
{ type: 'finish', reason: { kind: 'aborted', failure: { message: 'model stream aborted', code: 'ABORTED' } } },
|
||||
]
|
||||
const adapter = new MockAdapter([abortedStream])
|
||||
const ctx = await harness(adapter)
|
||||
@@ -963,13 +970,13 @@ describe('a finish-error stream chunk ends the turn as error, not completed', ()
|
||||
send(agent, 'go')
|
||||
await waitForIdle(ctx, agent)
|
||||
|
||||
expect(reasons).toEqual([{ kind: 'error', step: 1, message: 'model stream aborted', code: 'ABORTED' }])
|
||||
expect(reasons).toEqual([{ kind: 'error', step: 1, failure: { message: 'model stream aborted', code: 'ABORTED' } }])
|
||||
expect([...agent.session.events].some(event => event.type === 'assistant/message')).toBe(false)
|
||||
})
|
||||
|
||||
it('handles a finish error without a code (code key omitted)', async () => {
|
||||
const errorStream: StreamChunk[] = [
|
||||
{ type: 'finish', reason: { kind: 'error', message: 'codeless failure' } },
|
||||
{ type: 'finish', reason: { kind: 'error', failure: { message: 'codeless failure', code: 'UNKNOWN' } } },
|
||||
]
|
||||
const adapter = new MockAdapter([errorStream])
|
||||
const ctx = await harness(adapter)
|
||||
@@ -981,7 +988,7 @@ describe('a finish-error stream chunk ends the turn as error, not completed', ()
|
||||
send(agent, 'go')
|
||||
await waitForIdle(ctx, agent)
|
||||
|
||||
expect(reasons).toEqual([{ kind: 'error', step: 1, message: 'codeless failure' }])
|
||||
expect(reasons).toEqual([{ kind: 'error', step: 1, failure: { message: 'codeless failure', code: 'UNKNOWN' } }])
|
||||
})
|
||||
})
|
||||
|
||||
@@ -1101,7 +1108,7 @@ describe('turn and step boundary recovery', () => {
|
||||
})
|
||||
|
||||
it('a one-shot turn/end validation failure preserves the earlier turn error on retry', async () => {
|
||||
const errorStream: StreamChunk[] = [{ type: 'finish', reason: { kind: 'error', message: 'provider failed' } }]
|
||||
const errorStream: StreamChunk[] = [{ type: 'finish', reason: { kind: 'error', failure: { message: 'provider failed', code: 'UNKNOWN' } } }]
|
||||
const adapter = new MockAdapter([errorStream])
|
||||
const ctx = await balancedHarness(adapter)
|
||||
const agent = ctx.agentLoop.create(SessionId('a-turnend-veto'), { provider: 'mock', model: 'mock' })
|
||||
@@ -1131,7 +1138,7 @@ describe('turn and step boundary recovery', () => {
|
||||
const turnEnd = agent.session.events.findLast(event => event.type === 'turn/end')
|
||||
expect(turnEnd?.type === 'turn/end' && turnEnd.data.reason).toMatchObject({
|
||||
kind: 'error',
|
||||
message: 'provider failed',
|
||||
failure: { message: 'provider failed', code: 'UNKNOWN' },
|
||||
})
|
||||
})
|
||||
|
||||
@@ -1167,7 +1174,7 @@ describe('turn and step boundary recovery', () => {
|
||||
|
||||
it('a throwing agent/error listener during a step-error path still balances the turn, loop survives', async () => {
|
||||
// Listener failure cannot interrupt error finalization or the next turn.
|
||||
const errorStream: StreamChunk[] = [{ type: 'finish', reason: { kind: 'error', message: 'provider 500' } }]
|
||||
const errorStream: StreamChunk[] = [{ type: 'finish', reason: { kind: 'error', failure: { message: 'provider 500', code: 'SERVER' } } }]
|
||||
const adapter = new MockAdapter([errorStream, textResponse('turn 2 ok')])
|
||||
const ctx = await balancedHarness(adapter)
|
||||
const agent = ctx.agentLoop.create(SessionId('a-errorlistener'), { provider: 'mock', model: 'mock' })
|
||||
@@ -1183,7 +1190,11 @@ describe('turn and step boundary recovery', () => {
|
||||
expect(c.turnStart).toBe(1)
|
||||
expect(c.turnEnd).toBe(1)
|
||||
expect(c.stepStart).toBe(c.stepEnd)
|
||||
expect(c.lastTurnEnd?.type === 'turn/end' && c.lastTurnEnd.data.reason).toMatchObject({ kind: 'error', step: 1, message: 'provider 500' })
|
||||
expect(c.lastTurnEnd?.type === 'turn/end' && c.lastTurnEnd.data.reason).toMatchObject({
|
||||
kind: 'error',
|
||||
step: 1,
|
||||
failure: { message: 'provider 500', code: 'SERVER' },
|
||||
})
|
||||
|
||||
// loop survives: a second turn runs to completion (invariants oracle would
|
||||
// throw on its turn/start if turn 1 had been left open).
|
||||
@@ -1328,7 +1339,7 @@ describe('turn and step boundary recovery', () => {
|
||||
|
||||
it('a throwing step/end observer cannot interrupt error finalization', async () => {
|
||||
// Observer failure after step/end commit cannot interrupt turn finalization.
|
||||
const errorStream: StreamChunk[] = [{ type: 'finish', reason: { kind: 'error', message: 'provider 500' } }]
|
||||
const errorStream: StreamChunk[] = [{ type: 'finish', reason: { kind: 'error', failure: { message: 'provider 500', code: 'SERVER' } } }]
|
||||
const adapter = new MockAdapter([errorStream, textResponse('turn 2 ok')])
|
||||
const ctx = await harness(adapter)
|
||||
const agent = ctx.agentLoop.create(SessionId('a-stependthrow'), { provider: 'mock', model: 'mock' })
|
||||
|
||||
@@ -177,7 +177,9 @@ describe('toError normalization', () => {
|
||||
// String() of { code: 500 } is '[object Object]'
|
||||
expect(errors[0]!.message).toBe('[object Object]')
|
||||
const turnEnd = agent.session.events.find(e => e.type === 'turn/end')
|
||||
expect(turnEnd?.type === 'turn/end' && turnEnd.data.reason.kind === 'error' && turnEnd.data.reason.code).toBe('UNKNOWN')
|
||||
expect(turnEnd?.type === 'turn/end' && turnEnd.data.reason.kind === 'error'
|
||||
&& ('failure' in turnEnd.data.reason ? turnEnd.data.reason.failure.code : turnEnd.data.reason.code))
|
||||
.toBe('UNKNOWN')
|
||||
})
|
||||
})
|
||||
|
||||
@@ -208,7 +210,8 @@ describe('coded error data emission', () => {
|
||||
const turnEnd = agent.session.events.find(e => e.type === 'turn/end')
|
||||
expect(turnEnd).toBeDefined()
|
||||
if (turnEnd?.type === 'turn/end' && turnEnd.data.reason.kind === 'error') {
|
||||
expect(turnEnd.data.reason.code).toBe('RATE_LIMIT')
|
||||
expect('failure' in turnEnd.data.reason ? turnEnd.data.reason.failure.code : turnEnd.data.reason.code)
|
||||
.toBe('RATE_LIMIT')
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
@@ -5,8 +5,9 @@ import LlmService, {
|
||||
CONTEXT_WINDOW_EXCEEDED_CODE,
|
||||
LlmAdapter,
|
||||
LlmError,
|
||||
ProviderRequestId,
|
||||
} from '@deepseek-ai/dsh-llm'
|
||||
import type { GenerateOptions, StreamChunk } from '@deepseek-ai/dsh-llm'
|
||||
import type { GenerateOptions, LlmFailure, StreamChunk } from '@deepseek-ai/dsh-llm'
|
||||
import SessionStore, { SessionId } from '@deepseek-ai/dsh-session'
|
||||
import SystemPrompt from '@deepseek-ai/dsh-system-prompt'
|
||||
import ToolRegistry, { defineTool } from '@deepseek-ai/dsh-tools'
|
||||
@@ -258,16 +259,17 @@ describe('agent post-step and request-error lifecycle', () => {
|
||||
|
||||
it.each([
|
||||
['thrown', contextError()],
|
||||
['in-band', [{ type: 'finish', reason: { kind: 'error', message: 'too large', code: CONTEXT_WINDOW_EXCEEDED_CODE } }] satisfies StreamChunk[]],
|
||||
['in-band', [{ type: 'finish', reason: { kind: 'error', failure: { message: 'too large', code: CONTEXT_WINDOW_EXCEEDED_CODE, status: 400 } } }] satisfies StreamChunk[]],
|
||||
] as const)('recovers a %s request failure in a new reconstructable step', async (_style, failure) => {
|
||||
const adapter = new FailureScriptAdapter([failure, textResponse('recovered')])
|
||||
const ctx = await harness(adapter)
|
||||
const agent = ctx.agentLoop.create(SessionId(`recover-${_style}`), { provider: 'mock', model: 'mock' })
|
||||
const attempts: number[] = []
|
||||
ctx.on('agent/request-error', async (subject, turn, step, error, attempt) => {
|
||||
ctx.on('agent/request-error', async (subject, turn, step, error, facts, history) => {
|
||||
expect(subject).toBe(agent)
|
||||
expect({ turn, step, code: error.code }).toEqual({ turn: 1, step: 1, code: CONTEXT_WINDOW_EXCEEDED_CODE })
|
||||
attempts.push(attempt)
|
||||
expect(facts.code).toBe(CONTEXT_WINDOW_EXCEEDED_CODE)
|
||||
attempts.push(history.length)
|
||||
subject.session.append('context/message', {
|
||||
content: [{ type: 'text', text: 'RECOVERY SURFACE MUTATION' }],
|
||||
source: { kind: 'plugin', plugin: 'test-recovery' },
|
||||
@@ -295,7 +297,7 @@ describe('agent post-step and request-error lifecycle', () => {
|
||||
const agent = ctx.agentLoop.create(SessionId(`stream-plugin-${_name.replaceAll(' ', '-')}`), { provider: 'mock', model: 'mock' })
|
||||
let recoveries = 0
|
||||
install(ctx)
|
||||
ctx.on('agent/request-error', async (_agent, _turn, _step, _error, _attempt, _signal, next) => {
|
||||
ctx.on('agent/request-error', async (_agent, _turn, _step, _error, _failure, _history, _signal, next) => {
|
||||
recoveries += 1
|
||||
return next()
|
||||
})
|
||||
@@ -326,7 +328,7 @@ describe('agent post-step and request-error lifecycle', () => {
|
||||
})
|
||||
const agent = ctx.agentLoop.create(SessionId('nested-stream-not-recoverable'), { provider: 'mock', model: 'mock' })
|
||||
let recoveries = 0
|
||||
ctx.on('agent/request-error', async (_agent, _turn, _step, _error, _attempt, _signal, next) => {
|
||||
ctx.on('agent/request-error', async (_agent, _turn, _step, _error, _failure, _history, _signal, next) => {
|
||||
recoveries += 1
|
||||
return next()
|
||||
})
|
||||
@@ -359,7 +361,7 @@ describe('agent post-step and request-error lifecycle', () => {
|
||||
}
|
||||
const agent = ctx.agentLoop.create(SessionId(`${boundary}-not-recoverable`), { provider: 'mock', model: 'mock' })
|
||||
let recoveries = 0
|
||||
ctx.on('agent/request-error', async (_agent, _turn, _step, _error, _attempt, _signal, next) => {
|
||||
ctx.on('agent/request-error', async (_agent, _turn, _step, _error, _failure, _history, _signal, next) => {
|
||||
recoveries += 1
|
||||
return next()
|
||||
})
|
||||
@@ -387,7 +389,7 @@ describe('agent post-step and request-error lifecycle', () => {
|
||||
}
|
||||
const agent = ctx.agentLoop.create(SessionId(`${failure}-not-recoverable`), { provider: 'mock', model: 'mock' })
|
||||
let recoveries = 0
|
||||
ctx.on('agent/request-error', async (_agent, _turn, _step, _error, _attempt, _signal, next) => {
|
||||
ctx.on('agent/request-error', async (_agent, _turn, _step, _error, _failure, _history, _signal, next) => {
|
||||
recoveries += 1
|
||||
return next()
|
||||
})
|
||||
@@ -406,7 +408,7 @@ describe('agent post-step and request-error lifecycle', () => {
|
||||
const ctx = await harness(makeAdapter(original))
|
||||
const agent = ctx.agentLoop.create(SessionId(`identity-${_name.replaceAll(' ', '-')}`), { provider: 'mock', model: 'mock' })
|
||||
let seen: Error | undefined
|
||||
ctx.on('agent/request-error', async (_agent, _turn, _step, error, _attempt, _signal, next) => {
|
||||
ctx.on('agent/request-error', async (_agent, _turn, _step, error, _failure, _history, _signal, next) => {
|
||||
seen = error
|
||||
return next()
|
||||
})
|
||||
@@ -417,12 +419,52 @@ describe('agent post-step and request-error lifecycle', () => {
|
||||
expect(seen).toBe(original)
|
||||
})
|
||||
|
||||
it('passes structured facts beside the original Error and records them on exhaustion', async () => {
|
||||
const original = new LlmError('provider busy', 'RATE_LIMIT', {
|
||||
status: 429,
|
||||
retryAfterMs: 2_000,
|
||||
requestId: ProviderRequestId('req-9'),
|
||||
})
|
||||
Object.freeze(original)
|
||||
const ctx = await harness(new SynchronousDispatchFailureAdapter(original))
|
||||
const agent = ctx.agentLoop.create(SessionId('structured-request-failure'), { provider: 'mock', model: 'mock' })
|
||||
let seenError: Error | undefined
|
||||
let seenFailure: LlmFailure | undefined
|
||||
let seenHistory: readonly LlmFailure[] | undefined
|
||||
ctx.on('agent/request-error', async (
|
||||
_agent, _turn, _step, error, failure, history, _signal, next,
|
||||
) => {
|
||||
seenError = error
|
||||
seenFailure = failure
|
||||
seenHistory = history
|
||||
return next()
|
||||
})
|
||||
|
||||
send(agent)
|
||||
await waitForIdle(ctx, agent)
|
||||
|
||||
expect(seenError).toBe(original)
|
||||
expect(seenFailure).toEqual({
|
||||
message: 'provider busy',
|
||||
code: 'RATE_LIMIT',
|
||||
status: 429,
|
||||
retryAfterMs: 2_000,
|
||||
requestId: ProviderRequestId('req-9'),
|
||||
})
|
||||
expect(seenHistory).toEqual([])
|
||||
expect(Object.isFrozen(seenHistory)).toBe(true)
|
||||
expect(agent.session.events.at(-1)).toMatchObject({
|
||||
type: 'turn/end',
|
||||
data: { reason: { kind: 'error', step: 1, failure: seenFailure } },
|
||||
})
|
||||
})
|
||||
|
||||
it('classifies iterator construction and explicit NO_ADAPTER as model-request failures', async () => {
|
||||
for (const scenario of ['iterator', 'no-adapter'] as const) {
|
||||
const ctx = scenario === 'iterator' ? await harness(new IteratorConstructionFailureAdapter()) : await harness()
|
||||
const agent = ctx.agentLoop.create(SessionId(`request-boundary-${scenario}`), { provider: 'mock', model: 'mock' })
|
||||
let seen = ''
|
||||
ctx.on('agent/request-error', async (_agent, _turn, _step, error, _attempt, _signal, next) => {
|
||||
ctx.on('agent/request-error', async (_agent, _turn, _step, error, _failure, _history, _signal, next) => {
|
||||
seen = error.code ?? ''
|
||||
return next()
|
||||
})
|
||||
@@ -436,14 +478,17 @@ describe('agent post-step and request-error lifecycle', () => {
|
||||
const capped = new FailureScriptAdapter([contextError('first overflow'), contextError('second overflow')])
|
||||
const cappedCtx = await harness(capped)
|
||||
const cappedAgent = cappedCtx.agentLoop.create(SessionId('retry-cap'), { provider: 'mock', model: 'mock' })
|
||||
const cappedAttempts: number[] = []
|
||||
cappedCtx.on('agent/request-error', async (_agent, _turn, _step, _error, attempt, _signal, next) => {
|
||||
cappedAttempts.push(attempt)
|
||||
return attempt < 1 ? { action: 'retry' } : next()
|
||||
const cappedHistories: string[][] = []
|
||||
cappedCtx.on('agent/request-error', async (
|
||||
_agent, _turn, _step, _error, _failure, history, _signal, next,
|
||||
) => {
|
||||
const codes = history.map(entry => entry.code)
|
||||
cappedHistories.push(codes)
|
||||
return codes.length < 1 ? { action: 'retry' } : next()
|
||||
})
|
||||
send(cappedAgent)
|
||||
await waitForIdle(cappedCtx, cappedAgent)
|
||||
expect(cappedAttempts).toEqual([0, 1])
|
||||
expect(cappedHistories).toEqual([[], [CONTEXT_WINDOW_EXCEEDED_CODE]])
|
||||
|
||||
const reset = new FailureScriptAdapter([
|
||||
contextError('first overflow'),
|
||||
@@ -458,14 +503,16 @@ describe('agent post-step and request-error lifecycle', () => {
|
||||
async execute() { return [{ type: 'text', text: 'worked' }] },
|
||||
}))
|
||||
const resetAgent = resetCtx.agentLoop.create(SessionId('retry-reset'), { provider: 'mock', model: 'mock' })
|
||||
const resetAttempts: { step: number; attempt: number }[] = []
|
||||
resetCtx.on('agent/request-error', async (_agent, _turn, step, _error, attempt, _signal, next) => {
|
||||
resetAttempts.push({ step, attempt })
|
||||
return resetAttempts.length === 1 ? { action: 'retry' } : next()
|
||||
const resetHistories: { step: number; codes: string[] }[] = []
|
||||
resetCtx.on('agent/request-error', async (
|
||||
_agent, _turn, step, _error, _failure, history, _signal, next,
|
||||
) => {
|
||||
resetHistories.push({ step, codes: history.map(entry => entry.code) })
|
||||
return resetHistories.length === 1 ? { action: 'retry' } : next()
|
||||
})
|
||||
send(resetAgent)
|
||||
await waitForIdle(resetCtx, resetAgent)
|
||||
expect(resetAttempts).toEqual([{ step: 1, attempt: 0 }, { step: 3, attempt: 0 }])
|
||||
expect(resetHistories).toEqual([{ step: 1, codes: [] }, { step: 3, codes: [] }])
|
||||
})
|
||||
|
||||
it('preserves the original provider error when recovery throws', async () => {
|
||||
@@ -479,7 +526,7 @@ describe('agent post-step and request-error lifecycle', () => {
|
||||
|
||||
expect(agent.session.events.at(-1)).toMatchObject({
|
||||
type: 'turn/end',
|
||||
data: { reason: { kind: 'error', message: 'original overflow', code: CONTEXT_WINDOW_EXCEEDED_CODE } },
|
||||
data: { reason: { kind: 'error', failure: { message: 'original overflow', code: CONTEXT_WINDOW_EXCEEDED_CODE } } },
|
||||
})
|
||||
})
|
||||
|
||||
@@ -489,7 +536,7 @@ describe('agent post-step and request-error lifecycle', () => {
|
||||
const agent = ctx.agentLoop.create(SessionId(`${action}-recovery`), { provider: 'mock', model: 'mock' })
|
||||
let entered!: () => void
|
||||
const recoveryEntered = new Promise<void>((resolve) => { entered = resolve })
|
||||
ctx.on('agent/request-error', async (_agent, _turn, _step, _error, _attempt, signal) => {
|
||||
ctx.on('agent/request-error', async (_agent, _turn, _step, _error, _failure, _history, signal) => {
|
||||
entered()
|
||||
await new Promise<void>((resolve) => {
|
||||
signal.addEventListener('abort', () => { resolve() }, { once: true })
|
||||
|
||||
@@ -44,7 +44,7 @@ Agent *creation* is provided by the plugin implementing `AgentFactory` (`dsh-age
|
||||
|
||||
The lifecycle edges have two important local caveats. `agent/created` runs after scoped setup and after both session and agent registry entries exist. Setup is trusted composition-only code; the immediately following non-vetoing `agent/session-start` notification is the first supported startup injection point. `agent/disposed` always means the exact agent has left the registry. AgentLoop emits it after its driver is quiescent, while ordered teardown may still be detaching the session and unwinding the scope; custom agents registered directly own any stronger driver-ordering contract themselves.
|
||||
|
||||
Most interception points are cooperative waterfalls returning seam-specific decisions. `agent/pre-step` and `agent/post-step` are serial checkpoints around a step's durable work, while `agent/request-error` is the failed-model-request recovery waterfall: a retry opens a new numbered step after the failed step closes. `agent/turn-stop` is the terminal serial fold: it runs after ordinary continuation and steering folding, and a returned stop remains in force through turn close and flush so later steering cannot create an extra step or turn. Ordinary queued prompts remain intact. The full rationale for scoped dispatch and terminal settlement is in the [agent-scope runtime-design Agent Note](../../../.agents/notes/implemented/architecture/2026-07-12-agent-scope-runtime-design.md#three-execution-boundaries-are-deliberately-one-way).
|
||||
Most interception points are cooperative waterfalls returning seam-specific decisions. `agent/pre-step` and `agent/post-step` are serial checkpoints around a step's durable work, while `agent/request-error` is the failed-model-request recovery waterfall: it receives the exact error, normalized failure facts, immutable prior-retried facts, and signal after the failed step closes; a retry opens a new numbered step. `agent/turn-stop` is the terminal serial fold: it runs after ordinary continuation and steering folding, and a returned stop remains in force through turn close and flush so later steering cannot create an extra step or turn. Ordinary queued prompts remain intact. The full rationale for scoped dispatch and terminal settlement is in the [agent-scope runtime-design Agent Note](../../../.agents/notes/implemented/architecture/2026-07-12-agent-scope-runtime-design.md#three-execution-boundaries-are-deliberately-one-way).
|
||||
|
||||
`PromptDecision.additionalContexts` is an array so every injected context keeps its own source, envelope, and metadata. A `ContinuationDecision` reason is narrower: it becomes a `steering/message`, not a `context/message`, and therefore carries only content and source.
|
||||
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
|
||||
import type { Context } from 'cordis'
|
||||
import type { Scoped } from '@deepseek-ai/dsh-scope'
|
||||
import type { ContentBlock, LlmCallConfig, Message, MessageSource } from '@deepseek-ai/dsh-llm'
|
||||
import type { ContentBlock, LlmCallConfig, LlmFailure, Message, MessageSource } from '@deepseek-ai/dsh-llm'
|
||||
import type { ContextEnvelope, JsonValue, Session, SessionId } from '@deepseek-ai/dsh-session'
|
||||
import type {} from '@deepseek-ai/dsh-system-prompt'
|
||||
declare module '@deepseek-ai/dsh-system-prompt' {
|
||||
@@ -270,12 +270,13 @@ declare module 'cordis' {
|
||||
* @param turn - the open turn number.
|
||||
* @param step - the failed step number.
|
||||
* @param error - the original model-request failure.
|
||||
* @param retryAttempt - zero-based number of prior recovery retries.
|
||||
* @param failure - serializable facts normalized at the final adapter boundary.
|
||||
* @param priorFailures - immutable failures that already authorized another request in this consecutive sequence.
|
||||
* @param signal - the turn abort signal.
|
||||
* Scope-filtered dispatch (`@deepseek-ai/dsh-scope`): agent-scoped listeners receive only that agent.
|
||||
* @mode waterfall
|
||||
*/
|
||||
'agent/request-error'(this: Scoped<Agent>, agent: Agent, turn: number, step: number, error: RequestError, retryAttempt: number, signal: AbortSignal, next: () => Promise<RequestErrorDecision>): Promise<RequestErrorDecision>
|
||||
'agent/request-error'(this: Scoped<Agent>, agent: Agent, turn: number, step: number, error: RequestError, failure: LlmFailure, priorFailures: readonly LlmFailure[], signal: AbortSignal, next: () => Promise<RequestErrorDecision>): Promise<RequestErrorDecision>
|
||||
/**
|
||||
* Override whether the turn continues. The default continues after tool
|
||||
* calls or steering and stops otherwise; a continue reason becomes steering.
|
||||
|
||||
@@ -60,11 +60,11 @@ Durable values need one accepted representation, not a check followed by a secon
|
||||
|
||||
### Session event vocabulary (`types.ts`)
|
||||
|
||||
The append-only log's event types, enumerated member by member — payloads, surface badges, provenance — in the generated [persistence log event catalog](../../../docs/persistence-catalog.md). Token usage and provider/model/replay provenance ride on `assistant/message`; an operational error's step is on `turn/end.reason` for `kind: 'error'`.
|
||||
The append-only log's event types, enumerated member by member — payloads, surface badges, provenance — in the generated [persistence log event catalog](../../../docs/persistence-catalog.md). Token usage and provider/model/replay provenance ride on `assistant/message`; an operational error's step is on `turn/end.reason` for `kind: 'error'`, with structured provider facts for a final model-request failure.
|
||||
|
||||
Merge-extensible via `SessionEventMap` — a plugin declaration-merges its own types (the compaction seam's `compact/*`, the hook bridges' `hook/*`); merged members appear in the same catalog.
|
||||
Merge-extensible via `SessionEventMap` — a plugin declaration-merges its own types (the compaction seam's `compact/*`, bounded recovery's non-surface `llm/retry`, the hook bridges' `hook/*`); merged members appear in the same catalog.
|
||||
|
||||
Also defines `TurnTriggerMap` and `TurnEndReasonMap` (merge-extensible sum types for typed turn boundaries — `kind`-tagged instead of strings).
|
||||
Also defines `TurnTriggerMap` and `TurnEndReasonMap` (merge-extensible sum types for typed turn boundaries — `kind`-tagged instead of strings). A final model-request error retains one structured `LlmFailure`; other turn errors retain message/code, and both identify the failed step.
|
||||
|
||||
Every `SessionEvent` carries two optional top-level fields (structural metadata):
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import type { Branded } from '@deepseek-ai/dsh-brand'
|
||||
import type { AssistantProvenance, CallId, ContentBlock, LlmCallConfig, Message, MessageSource, StreamChunk, TokenUsage, ToolSchema } from '@deepseek-ai/dsh-llm'
|
||||
import type { AssistantProvenance, CallId, ContentBlock, LlmCallConfig, LlmFailure, Message, MessageSource, StreamChunk, TokenUsage, ToolSchema } from '@deepseek-ai/dsh-llm'
|
||||
import type { JsonValue } from './json.ts'
|
||||
|
||||
/** Canonical context-tag framing, or caller-owned framing rendered verbatim. */
|
||||
@@ -102,9 +102,13 @@ export interface TurnEndReasonMap {
|
||||
* The turn failed: a step threw or the model reported a failure. `step` is the
|
||||
* step number the failure occurred on (the operational error's location — the
|
||||
* single durable record of an in-turn failure; live diagnostics also fire via
|
||||
* `agent/error`). `code` is the error's code when one was attached.
|
||||
* `agent/error`). Final model-request failures retain their normalized facts
|
||||
* as one `failure`; other turn failures retain their live Error message/code.
|
||||
*/
|
||||
error: { kind: 'error'; step: number; message: string; code?: string }
|
||||
error: { kind: 'error'; step: number } & (
|
||||
| { failure: LlmFailure; message?: never; code?: never }
|
||||
| { message: string; code?: string; failure?: never }
|
||||
)
|
||||
disposed: { kind: 'disposed' }
|
||||
/** At least one step reached its output-token ceiling, even if a plugin continued the turn. */
|
||||
'max-tokens': { kind: 'max-tokens' }
|
||||
|
||||
Reference in New Issue
Block a user