Coder/deepseek-harness
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(agent-loop): simplify observable state machine

Browse Source
This commit is contained in:
_Kerman
2026-07-24 21:18:48 +08:00
parent fb0ef82aa6
commit b73eb7663c
131 changed files with 2011 additions and 4292 deletions
Download Patch File Download Diff File Expand all files Collapse all files

83
packages/llm/llm-retry/tests/invariant.spec.ts
View File

@@ -13,32 +13,35 @@ async function setup(): Promise<Context> {
return ctx
}
const failure = { message: 'provider busy', code: 'RATE_LIMIT', status: 429 }
function closeStep(ctx: Context, id: string, turn = 1, step = 1) {
const session = ctx.sessions.create(SessionId(id))
session.append('turn/start', { turn, trigger: { kind: 'message', source: { kind: 'user' } } })
session.append('turn/start', {
turn,
trigger: turn === 1
? { kind: 'message', source: { kind: 'user' } }
: { kind: 'retry' },
})
session.append('step/start', { turn, step })
session.append('step/end', { turn, step })
return session
}
const failure = { message: 'provider busy', code: 'RATE_LIMIT', status: 429 }
describe('llm-retry invariants', () => {
it('accepts increasing retry records for successive closed steps and ignores unrelated events', async () => {
it('accepts increasing retry schedules for successive failed turns', async () => {
const ctx = await setup()
const session = closeStep(ctx, 'retry-invariant-valid')
expect(() => {
session.append('llm/retry', {
turn: 1, step: 1, retry: 1, maxRetries: 2, delayMs: 500, failure,
})
session.append('step/start', { turn: 1, step: 2 })
session.append('step/end', { turn: 1, step: 2 })
session.append('turn/end', { turn: 1, reason: { kind: 'error', step: 1, failure } })
session.append('turn/start', { turn: 2, trigger: { kind: 'retry' } })
session.append('step/start', { turn: 2, step: 1 })
session.append('step/end', { turn: 2, step: 1 })
session.append('llm/retry', {
turn: 1, step: 2, retry: 2, maxRetries: 2, delayMs: 1_000, failure,
})
const zeroDelay = closeStep(ctx, 'retry-invariant-zero-delay')
zeroDelay.append('llm/retry', {
turn: 1, step: 1, retry: 1, maxRetries: 1, delayMs: 0, failure,
turn: 2, step: 1, retry: 2, maxRetries: 2, delayMs: 0, failure,
})
}).not.toThrow()
expect(() => { ctx.emit('tools/change') }).not.toThrow()
@@ -60,7 +63,7 @@ describe('llm-retry invariants', () => {
}).toThrow(message)
})
it('rejects retry records outside the matching closed-step boundary', async () => {
it('requires an open turn and its latest closed step', async () => {
const ctx = await setup()
const absent = ctx.sessions.create(SessionId('retry-invariant-no-turn'))
expect(() => {
@@ -85,31 +88,15 @@ describe('llm-retry invariants', () => {
})
}).toThrow(/step 1 is still open/)
const noStep = ctx.sessions.create(SessionId('retry-invariant-no-step'))
noStep.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } })
expect(() => {
noStep.append('llm/retry', {
turn: 1, step: 1, retry: 1, maxRetries: 2, delayMs: 1, failure,
})
}).toThrow(/latest closed step is undefined/)
const wrongStep = closeStep(ctx, 'retry-invariant-wrong-step')
expect(() => {
wrongStep.append('llm/retry', {
turn: 1, step: 2, retry: 1, maxRetries: 2, delayMs: 1, failure,
})
}).toThrow(/latest closed step is 1/)
const closedTurn = closeStep(ctx, 'retry-invariant-closed-turn')
closedTurn.append('turn/end', { turn: 1, reason: { kind: 'aborted' } })
expect(() => {
closedTurn.append('llm/retry', {
turn: 1, step: 1, retry: 1, maxRetries: 2, delayMs: 1, failure,
})
}).toThrow(/inside an open turn/)
})
it('rejects duplicate and non-increasing retry records', async () => {
it('rejects duplicate and out-of-sequence retry schedules', async () => {
const ctx = await setup()
const duplicate = closeStep(ctx, 'retry-invariant-duplicate')
duplicate.append('llm/retry', {
@@ -119,26 +106,52 @@ describe('llm-retry invariants', () => {
duplicate.append('llm/retry', {
turn: 1, step: 1, retry: 2, maxRetries: 3, delayMs: 1, failure,
})
}).toThrow(/duplicates the retry record/)
}).toThrow(/duplicates/)
const nonIncreasing = closeStep(ctx, 'retry-invariant-non-increasing')
nonIncreasing.append('llm/retry', {
turn: 1, step: 1, retry: 1, maxRetries: 3, delayMs: 1, failure,
})
nonIncreasing.append('step/start', { turn: 1, step: 2 })
nonIncreasing.append('step/end', { turn: 1, step: 2 })
nonIncreasing.append('turn/end', { turn: 1, reason: { kind: 'error', step: 1, failure } })
nonIncreasing.append('turn/start', { turn: 2, trigger: { kind: 'retry' } })
nonIncreasing.append('step/start', { turn: 2, step: 1 })
nonIncreasing.append('step/end', { turn: 2, step: 1 })
expect(() => {
nonIncreasing.append('llm/retry', {
turn: 1, step: 2, retry: 1, maxRetries: 3, delayMs: 1, failure,
turn: 2, step: 1, retry: 1, maxRetries: 3, delayMs: 1, failure,
})
}).toThrow(/must increase/)
}).toThrow(/retry-chain position 2/)
})
it('resets retry numbering after a completed chain', async () => {
const ctx = await setup()
const session = closeStep(ctx, 'retry-invariant-reset')
session.append('llm/retry', {
turn: 1, step: 1, retry: 1, maxRetries: 2, delayMs: 1, failure,
})
session.append('turn/end', { turn: 1, reason: { kind: 'error', step: 1, failure } })
session.append('turn/start', { turn: 2, trigger: { kind: 'retry' } })
session.append('step/start', { turn: 2, step: 1 })
session.append('step/end', { turn: 2, step: 1 })
session.append('turn/end', { turn: 2, reason: { kind: 'completed' } })
session.append('turn/start', {
turn: 3,
trigger: { kind: 'message', source: { kind: 'user' } },
})
session.append('step/start', { turn: 3, step: 1 })
session.append('step/end', { turn: 3, step: 1 })
expect(() => {
session.append('llm/retry', {
turn: 3, step: 1, retry: 1, maxRetries: 2, delayMs: 1, failure,
})
}).not.toThrow()
})
it('validates existing histories on late registration', async () => {
const ctx = new Context()
await ctx.plugin(SessionStore)
const session = ctx.sessions.create(SessionId('retry-invariant-late'))
session.append('step/end', { turn: 1, step: 1 })
session.append('llm/retry', {
turn: 1, step: 1, retry: 1, maxRetries: 2, delayMs: 1, failure,
})

16
packages/llm/llm-retry/tests/loader-composition.spec.ts
View File

@@ -7,7 +7,6 @@ import { Context } from 'cordis'
import Loader from '@cordisjs/plugin-loader'
import Include from '@cordisjs/plugin-include'
import AgentRegistry from '@deepseek-ai/dsh-agent'
import type { Agent } from '@deepseek-ai/dsh-agent'
import AgentLoop from '@deepseek-ai/dsh-agent-loop'
import LlmService, { LlmAdapter, LlmError } from '@deepseek-ai/dsh-llm'
import type { GenerateOptions, StreamChunk } from '@deepseek-ai/dsh-llm'
@@ -32,17 +31,6 @@ class TransientOnceAdapter extends LlmAdapter {
}
}
function waitForIdle(ctx: Context, agent: Agent): Promise<void> {
return new Promise((resolve) => {
const dispose = ctx.on('agent/status', (subject, status) => {
if (subject === agent && status === 'idle') {
dispose()
resolve()
}
})
})
}
afterEach(async () => {
await context?.fiber.dispose()
context = undefined
@@ -113,9 +101,9 @@ describe('real Loader composition', () => {
const adapter = new TransientOnceAdapter()
loaded.llm.registerAdapter(['mock'], adapter)
const agent = loaded.agentLoop.create(SessionId('loader-retry'), { provider: 'mock', model: 'mock' })
const idle = waitForIdle(loaded, agent)
agent.followup([{ type: 'text', text: 'recover' }])
await idle
await expect.poll(() => adapter.requests).toBe(2)
await agent.whenIdle()
expect(adapter.requests).toBe(2)
expect(agent.session.events.filter(event => event.type === 'llm/retry')).toHaveLength(1)

9
packages/llm/llm-retry/tests/persistence.spec.ts
View File

@@ -43,7 +43,14 @@ describe.each(['jsonl', 'sqlite'] as const)('%s retry-event persistence', (kind)
delayMs: 750,
failure: { message: 'provider busy', code: 'RATE_LIMIT', status: 429 },
})
session.append('turn/end', { turn: 1, reason: { kind: 'aborted' } })
session.append('turn/end', {
turn: 1,
reason: {
kind: 'error',
step: 1,
failure: { message: 'provider busy', code: 'RATE_LIMIT', status: 429 },
},
})
expect(session.deriveMessages()).toEqual([])
await ctx.sessions.flush(session)

199
packages/llm/llm-retry/tests/retry.spec.ts
View File

@@ -8,9 +8,8 @@ import type { SessionEvent } from '@deepseek-ai/dsh-session'
import SystemPrompt from '@deepseek-ai/dsh-system-prompt'
import ToolRegistry, { defineContentToolFixture } from '@deepseek-ai/dsh-tools'
import AgentRegistry from '@deepseek-ai/dsh-agent'
import type { Agent, RequestErrorDecision } from '@deepseek-ai/dsh-agent'
import type { Agent } from '@deepseek-ai/dsh-agent'
import AgentLoop from '@deepseek-ai/dsh-agent-loop'
import { MAX_TIMER_DELAY_MS } from '@deepseek-ai/dsh-timeout'
import * as retry from '../src/index.ts'
type ScriptEntry = Error | Iterable<StreamChunk> | AsyncIterable<StreamChunk>
@@ -149,8 +148,8 @@ describe('bounded transient retry policy', () => {
await idle
expect(adapter.requests).toHaveLength(2)
expect(agent.session.events.filter(item => item.type === 'step/start').map(item => item.data.step))
.toEqual([1, 2])
expect(agent.session.events.filter(item => item.type === 'step/start').map(item => item.data))
.toEqual([{ turn: 1, step: 1 }, { turn: 2, step: 1 }])
expect(agent.session.deriveMessages().at(-1)).toEqual({
role: 'assistant',
content: [{ type: 'text', text: 'done' }],
@@ -185,11 +184,13 @@ describe('bounded transient retry policy', () => {
await idle
const failedChunks = agent.session.events.filter(event =>
event.type === 'assistant/chunk' && event.data.step === 1,
event.type === 'assistant/chunk' && event.data.turn === 1,
)
expect(failedChunks).toHaveLength(6)
expect(agent.session.events.filter(event => event.type === 'assistant/message').map(event => event.data.step))
.toEqual([2])
expect(agent.session.events.filter(event => event.type === 'assistant/message').map(event => ({
turn: event.data.turn,
step: event.data.step,
}))).toEqual([{ turn: 2, step: 1 }])
expect(agent.session.events.some(event => event.type === 'tool/call')).toBe(false)
expect(toolExecutions).toBe(0)
expect(agent.session.deriveMessages().at(-1)).toMatchObject({
@@ -232,6 +233,36 @@ describe('bounded transient retry policy', () => {
})
})
it('resets the retry budget for a later message', async () => {
vi.useFakeTimers()
const adapter = new ScriptedAdapter([
new LlmError('first busy', 'SERVER'),
textResponse('first done'),
new LlmError('second busy', 'SERVER'),
textResponse('second done'),
])
;({ ctx: context } = await harness(adapter, { maxTransientRetries: 1 }))
const agent = context.agentLoop.create(SessionId('retry-reset'), { provider: 'mock', model: 'mock' })
const firstRetry = waitForRetry(context, agent, 1)
agent.followup([{ type: 'text', text: 'first' }])
await firstRetry
const firstIdle = waitForIdle(context, agent)
await vi.advanceTimersByTimeAsync(500)
await firstIdle
const secondRetry = waitForRetry(context, agent, 1)
agent.followup([{ type: 'text', text: 'second' }])
await secondRetry
const secondIdle = waitForIdle(context, agent)
await vi.advanceTimersByTimeAsync(500)
await secondIdle
expect(agent.session.events.filter(event => event.type === 'llm/retry').map(event => event.data.retry))
.toEqual([1, 1])
expect(adapter.requests).toHaveLength(4)
})
it('accepts the zero-delay lower jitter bound', async () => {
vi.useFakeTimers()
const adapter = new ScriptedAdapter([
@@ -310,7 +341,6 @@ describe('bounded transient retry policy', () => {
agent.followup([{ type: 'text', text: 'go' }])
await scheduled
const idle = waitForIdle(context, agent)
await mounted.retryFiber.dispose()
await idle
await vi.advanceTimersByTimeAsync(60_000)
@@ -320,157 +350,4 @@ describe('bounded transient retry policy', () => {
expect(vi.getTimerCount()).toBe(0)
})
it('does not make plugin disposal wait for a delegated recovery policy', async () => {
const adapter = new ScriptedAdapter([new LlmError('bad key', 'AUTH')])
const mounted = await harness(adapter)
context = mounted.ctx
const downstream = Promise.withResolvers<RequestErrorDecision>()
const entered = Promise.withResolvers<undefined>()
context.on('agent/request-error', () => {
entered.resolve(undefined)
return downstream.promise
})
const agent = context.agentLoop.create(SessionId('retry-delegated-disposal'), {
provider: 'mock',
model: 'mock',
})
const idle = waitForIdle(context, agent)
agent.followup([{ type: 'text', text: 'go' }])
await entered.promise
const disposing = mounted.retryFiber.dispose()
let timer: ReturnType<typeof setTimeout> | undefined
const outcome = await Promise.race([
disposing.then(() => 'disposed' as const),
new Promise<'blocked'>((resolve) => { timer = setTimeout(() => { resolve('blocked') }, 100) }),
])
if (timer !== undefined) clearTimeout(timer)
downstream.resolve({ action: 'fail' })
await disposing
await idle
expect(outcome).toBe('disposed')
expect(adapter.requests).toHaveLength(1)
})
it('fails a captured callback after disposal without entering downstream policy', async () => {
const adapter = new ScriptedAdapter([new LlmError('bad key', 'AUTH')])
const captured = Promise.withResolvers<undefined>()
let invokeCaptured: (() => Promise<void>) | undefined
const mounted = await harness(adapter, {}, (ctx) => {
ctx.on('agent/request-error', (_agent, _turn, _step, _error, _failure, _history, _signal, next) => {
return new Promise<RequestErrorDecision>((resolve) => {
invokeCaptured = async () => { resolve(await next()) }
captured.resolve(undefined)
})
})
})
context = mounted.ctx
let downstreamCalls = 0
context.on('agent/request-error', async (_agent, _turn, _step, _error, _failure, _history, _signal, next) => {
downstreamCalls += 1
return next()
})
const agent = context.agentLoop.create(SessionId('retry-captured-disposal'), {
provider: 'mock',
model: 'mock',
})
const idle = waitForIdle(context, agent)
agent.followup([{ type: 'text', text: 'go' }])
await captured.promise
await mounted.retryFiber.dispose()
if (invokeCaptured === undefined) throw new Error('request-error waterfall did not capture retry callback')
await invokeCaptured()
await idle
expect(downstreamCalls).toBe(0)
expect(adapter.requests).toHaveLength(1)
})
it('lets turn cancellation win during backoff without opening another step', async () => {
vi.useFakeTimers()
const adapter = new ScriptedAdapter([
new LlmError('temporary', 'TIMEOUT'),
textResponse('must not run'),
])
;({ ctx: context } = await harness(adapter))
const agent = context.agentLoop.create(SessionId('retry-cancel'), { provider: 'mock', model: 'mock' })
const scheduled = waitForRetry(context, agent, 1)
agent.followup([{ type: 'text', text: 'go' }])
await scheduled
const idle = waitForIdle(context, agent)
agent.cancel({ kind: 'user' })
await idle
expect(adapter.requests).toHaveLength(1)
expect(agent.session.events.at(-1)).toMatchObject({
type: 'turn/end',
data: { reason: { kind: 'aborted' } },
})
expect(vi.getTimerCount()).toBe(0)
})
it('lets an earlier recovery listener cancel before retry policy runs', async () => {
vi.useFakeTimers()
const adapter = new ScriptedAdapter([
new LlmError('temporary', 'SERVER'),
textResponse('must not run'),
])
;({ ctx: context } = await harness(adapter, {}, (ctx) => {
ctx.on('agent/request-error', async (agent, _turn, _step, _error, _failure, _history, _signal, next) => {
agent.cancel({ kind: 'user' })
return next()
})
}))
const agent = context.agentLoop.create(SessionId('retry-pre-cancel'), { provider: 'mock', model: 'mock' })
const idle = waitForIdle(context, agent)
agent.followup([{ type: 'text', text: 'go' }])
await idle
expect(adapter.requests).toHaveLength(1)
expect(agent.session.events.some(event => event.type === 'llm/retry')).toBe(false)
expect(agent.session.events.at(-1)).toMatchObject({
type: 'turn/end',
data: { reason: { kind: 'aborted' } },
})
})
it('handles synchronous cancellation from the retry status event', async () => {
vi.useFakeTimers()
const adapter = new ScriptedAdapter([
new LlmError('temporary', 'SERVER'),
textResponse('must not run'),
])
;({ ctx: context } = await harness(adapter))
const agent = context.agentLoop.create(SessionId('retry-event-cancel'), { provider: 'mock', model: 'mock' })
context.on('session/event', (session, event) => {
if (session === agent.session && event.type === 'llm/retry') agent.cancel({ kind: 'user' })
})
const idle = waitForIdle(context, agent)
agent.followup([{ type: 'text', text: 'go' }])
await idle
expect(adapter.requests).toHaveLength(1)
expect(agent.session.events.filter(event => event.type === 'llm/retry')).toHaveLength(1)
expect(vi.getTimerCount()).toBe(0)
})
it.each([
[{ maxTransientRetries: -1 }, /maxTransientRetries/],
[{ maxTransientRetries: 1.5 }, /maxTransientRetries/],
[{ initialDelayMs: 0 }, /initialDelayMs/],
[{ maxDelayMs: Number.POSITIVE_INFINITY }, /maxDelayMs/],
[{ initialDelayMs: MAX_TIMER_DELAY_MS + 1 }, /initialDelayMs/],
[{ maxDelayMs: MAX_TIMER_DELAY_MS + 1 }, /maxDelayMs/],
[{ initialDelayMs: 20, maxDelayMs: 10 }, /less than or equal/],
[{ jitterRatio: 1.1 }, /jitterRatio/],
[{ retryableCodes: [] }, /must not be empty/],
[{ retryableCodes: ['SERVER', 'SERVER'] }, /duplicates/],
[{ retryableCodes: [''] }, /non-empty strings/],
] as const)('fails direct composition for invalid config %#', (config, message) => {
expect(() => { retry.apply(new Context(), config as retry.Config) }).toThrow(message)
})
})