Coder/deepseek-harness
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(llm): isolate retry policy histories

Browse Source
This commit is contained in:
Turtle
2026-07-25 15:38:58 +08:00
parent 38ce422f71
commit efc725b7e4
31 changed files with 666 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
packages/llm/llm-retry/README.md
View File

@@ -6,9 +6,9 @@ Each provider adapter owns an optional nested `retryPolicy`, captured when its r
Both modes use bounded exponential backoff with symmetric jitter. A valid `providerRetryAfterMs` at or below `maxDelayMs` replaces local backoff without jitter. An over-cap provider delay makes normal mode delegate, while always mode uses its configured local backoff so it cannot terminate on that instruction.
Before waiting, the plugin appends a non-surface `llm/retry` event with the provider, mode, failure, and scheduled delay. Normal events include the finite maximum; always events omit it, and UIs render `∞`. Cancellation and plugin disposal abort active backoff, drain active delegated recovery before applying the abort, and make a callback captured before disposal fail closed.
Before waiting, the plugin appends a non-surface `llm/retry` event with the provider, mode, canonical resolved-policy key, failure, and scheduled delay. The key includes every behavior-affecting field and sorts normal-mode codes because eligibility uses set membership. Retry numbers continue only across events with the same provider and complete policy key, so a route replacement with different limits, code membership, or backoff starts its own history. Normal events include the finite maximum; always events omit it, and UIs render `∞`. Cancellation and plugin disposal abort active backoff, drain active delegated recovery before applying the abort, and make a callback captured before disposal fail closed.
The separately published `./invariant` companion checks that every retry record names the current open turn and latest closed step, matches the failed request's durable provider, has a unique step record and correct provider-policy retry number, and carries a valid mode-specific budget and bounded timer delay. Full jitter may schedule zero milliseconds at its lower boundary.
The separately published `./invariant` companion checks that every retry record names the current open turn and latest closed step, matches the failed request's durable provider, carries a producer-canonical policy key consistent with its mode and finite budget, binds normal failures and every scheduled delay to that policy, has a unique step record and correct provider-policy retry number, and carries a bounded timer delay. Full jitter may schedule zero milliseconds at its lower boundary.
```yaml
- name: '@deepseek-ai/dsh-llm-deepseek'

11
packages/llm/llm-retry/src/index.ts
View File

@@ -11,6 +11,7 @@ import type { Agent, RequestError, RequestErrorDecision } from '@deepseek-ai/dsh
import type { LlmFailure, ResolvedRetryPolicy } from '@deepseek-ai/dsh-llm'
import type { SessionEvent } from '@deepseek-ai/dsh-session'
import { providerForClosedStep } from './history.ts'
import { retryPolicyKey } from './policy-key.ts'
declare module '@deepseek-ai/dsh-session' {
interface SessionEventMap {
@@ -20,6 +21,7 @@ declare module '@deepseek-ai/dsh-session' {
step: number
provider: string
mode: 'normal'
policyKey: string
retry: number
maxRetries: number
delayMs: number
@@ -29,6 +31,7 @@ declare module '@deepseek-ai/dsh-session' {
step: number
provider: string
mode: 'always'
policyKey: string
retry: number
delayMs: number
failure: LlmFailure
@@ -121,6 +124,7 @@ export function apply(ctx: Context, config: Config = {}, internals: RetryInterna
failure: LlmFailure,
provider: string,
policy: ResolvedRetryPolicy,
policyKey: string,
retry: number,
delayMs: number,
signal: AbortSignal,
@@ -133,6 +137,7 @@ export function apply(ctx: Context, config: Config = {}, internals: RetryInterna
step,
provider,
mode: policy.mode,
policyKey,
retry,
maxRetries: policy.maxRetries,
delayMs,
@@ -143,6 +148,7 @@ export function apply(ctx: Context, config: Config = {}, internals: RetryInterna
step,
provider,
mode: policy.mode,
policyKey,
retry,
delayMs,
failure,
@@ -192,6 +198,7 @@ export function apply(ctx: Context, config: Config = {}, internals: RetryInterna
return next()
}
const policyKey = retryPolicyKey(policy)
const firstPriorStep = step - priorFailures.length
const priorPolicyRetry = agent.session.events.findLast((event): event is SessionEvent<'llm/retry'> =>
event.type === 'llm/retry'
@@ -199,7 +206,7 @@ export function apply(ctx: Context, config: Config = {}, internals: RetryInterna
&& event.data.step >= firstPriorStep
&& event.data.step < step
&& event.data.provider === provider
&& event.data.mode === policy.mode,
&& event.data.policyKey === policyKey,
)
const previousRetry = priorPolicyRetry?.data.retry ?? 0
if (policy.mode === 'normal' && previousRetry >= policy.maxRetries) return next()
@@ -218,7 +225,7 @@ export function apply(ctx: Context, config: Config = {}, internals: RetryInterna
delayMs = localDelay(policy, retry, random)
}
return backoff(agent, turn, step, failure, provider, policy, retry, delayMs, signal)
return backoff(agent, turn, step, failure, provider, policy, policyKey, retry, delayMs, signal)
}
const disposeListener = ctx.on('agent/request-error', (

26
packages/llm/llm-retry/src/invariant.ts
View File

@@ -2,9 +2,9 @@
import type { Context } from 'cordis'
import type { Session, SessionEvent } from '@deepseek-ai/dsh-session'
import { MAX_TIMER_DELAY_MS } from '@deepseek-ai/dsh-timeout'
import type { InvariantFailure, InvariantInstaller } from '@deepseek-ai/dsh-invariants'
import { providerForClosedStep } from './history.ts'
import { parseRetryPolicyKey } from './policy-key.ts'
import type {} from './index.ts'
const PACKAGE_NAME = '@deepseek-ai/dsh-llm-retry'
@@ -20,30 +20,46 @@ function validateRetry(
event: SessionEvent<'llm/retry'>,
fail: InvariantFailure,
): void {
const { turn, step, provider, mode, retry, delayMs } = event.data
const { turn, step, provider, mode, policyKey, retry, delayMs } = event.data
if (!Number.isSafeInteger(retry) || retry < 1) {
fail('llm/retry retry must be a positive safe integer')
}
if (typeof provider !== 'string' || provider.length === 0) {
fail('llm/retry provider must be non-empty string')
}
const keyedPolicy = parseRetryPolicyKey(policyKey)
if (keyedPolicy === undefined) {
fail('llm/retry policyKey must encode a canonical resolved policy')
}
switch (mode) {
case 'normal': {
const { maxRetries } = event.data
if (!Number.isSafeInteger(maxRetries) || maxRetries < 1 || retry > maxRetries) {
fail(`llm/retry retry ${retry} must not exceed a positive safe maxRetries ${maxRetries}`)
}
if (keyedPolicy.mode !== 'normal') {
fail(`llm/retry mode normal must match policyKey mode ${keyedPolicy.mode}`)
}
if (keyedPolicy.maxRetries !== maxRetries) {
fail(`llm/retry maxRetries ${maxRetries} must match policyKey`)
}
if (!keyedPolicy.retryableCodes.includes(event.data.failure.code)) {
fail(`llm/retry failure code ${event.data.failure.code} must be eligible under policyKey`)
}
break
}
case 'always':
if (keyedPolicy.mode !== 'always') {
fail(`llm/retry mode always must match policyKey mode ${keyedPolicy.mode}`)
}
if ('maxRetries' in event.data) fail('llm/retry always mode must omit maxRetries')
break
default:
fail(`llm/retry mode must be normal or always, got ${String(mode)}`)
}
if (typeof delayMs !== 'number' || !Number.isFinite(delayMs)
|| delayMs < 0 || delayMs > MAX_TIMER_DELAY_MS) {
fail(`llm/retry delayMs must be a finite number within 0..${MAX_TIMER_DELAY_MS}`)
|| delayMs < 0 || delayMs > keyedPolicy.maxDelayMs) {
fail(`llm/retry delayMs must be a finite number within policyKey range 0..${keyedPolicy.maxDelayMs}`)
}
const turnStartIndex = history.findLastIndex(prior =>
@@ -86,7 +102,7 @@ function validateRetry(
index > lastSuccessIndex
&& prior.type === 'llm/retry'
&& prior.data.provider === provider
&& prior.data.mode === mode
&& prior.data.policyKey === policyKey
))
const expectedRetry = (priorPolicyRetry?.data.retry ?? 0) + 1
if (retry !== expectedRetry) {

100
packages/llm/llm-retry/src/policy-key.ts Normal file
View File

@@ -0,0 +1,100 @@
/** Canonical durable identity for resolved retry policies. @module @deepseek-ai/dsh-llm-retry/policy-key */
import type { ResolvedRetryBackoff, ResolvedRetryPolicy } from '@deepseek-ai/dsh-llm'
import { MAX_TIMER_DELAY_MS } from '@deepseek-ai/dsh-timeout'
function parseBackoff(
tuple: readonly unknown[],
offset: number,
): ResolvedRetryBackoff | undefined {
const initialDelayMs = tuple[offset]
const maxDelayMs = tuple[offset + 1]
const jitterRatio = tuple[offset + 2]
if (typeof initialDelayMs !== 'number' || !Number.isFinite(initialDelayMs)
|| initialDelayMs <= 0 || initialDelayMs > MAX_TIMER_DELAY_MS
|| typeof maxDelayMs !== 'number' || !Number.isFinite(maxDelayMs)
|| maxDelayMs <= 0 || maxDelayMs > MAX_TIMER_DELAY_MS
|| initialDelayMs > maxDelayMs
|| typeof jitterRatio !== 'number' || !Number.isFinite(jitterRatio)
|| jitterRatio < 0 || jitterRatio > 1) {
return undefined
}
return { initialDelayMs, maxDelayMs, jitterRatio }
}
/**
* Derive the canonical durable key for one fully resolved provider policy.
* Retryable-code order is normalized because eligibility uses set membership.
* @param policy - immutable policy captured from the serving registration.
* @returns canonical JSON tuple containing every behavior-affecting field.
*/
export function retryPolicyKey(policy: ResolvedRetryPolicy): string {
if (policy.mode === 'always') {
return JSON.stringify([
policy.mode,
policy.initialDelayMs,
policy.maxDelayMs,
policy.jitterRatio,
])
}
return JSON.stringify([
policy.mode,
policy.maxRetries,
[...policy.retryableCodes].sort(),
policy.initialDelayMs,
policy.maxDelayMs,
policy.jitterRatio,
])
}
/**
* Parse a producer-canonical policy key from durable input.
* @param value - untrusted persisted event field.
* @returns the resolved policy encoded by the key, or `undefined` for any non-canonical value.
*/
export function parseRetryPolicyKey(value: unknown): ResolvedRetryPolicy | undefined {
if (typeof value !== 'string' || value.length === 0) return undefined
let tuple: unknown
try {
tuple = JSON.parse(value) as unknown
} catch (_invalidPolicyKeyJson) {
return undefined
}
if (!Array.isArray(tuple)) return undefined
const items = tuple as readonly unknown[]
const mode = items[0]
let policy: ResolvedRetryPolicy
switch (mode) {
case 'always': {
if (items.length !== 4) return undefined
const backoff = parseBackoff(items, 1)
if (backoff === undefined) return undefined
policy = Object.freeze({ mode, ...backoff })
break
}
case 'normal': {
if (items.length !== 6) return undefined
const maxRetries = items[1]
const retryableCodes = items[2]
const backoff = parseBackoff(items, 3)
if (!Number.isSafeInteger(maxRetries) || (maxRetries as number) < 0
|| !Array.isArray(retryableCodes) || retryableCodes.length === 0
|| (retryableCodes as readonly unknown[])
.some(code => typeof code !== 'string' || code.length === 0)
|| new Set(retryableCodes).size !== retryableCodes.length
|| backoff === undefined) {
return undefined
}
policy = Object.freeze({
mode,
maxRetries: maxRetries as number,
retryableCodes: Object.freeze(retryableCodes as string[]),
...backoff,
})
break
}
default:
return undefined
}
return retryPolicyKey(policy) === value ? policy : undefined
}

100
packages/llm/llm-retry/tests/invariant.spec.ts
View File

@@ -27,7 +27,10 @@ function closeStep(ctx: Context, id: string, turn = 1, step = 1) {
}
const failure = { message: 'provider busy', code: 'RATE_LIMIT', status: 429 }
const normal = { provider: 'mock', mode: 'normal' as const }
const normalPolicyKey = (maxRetries: number): string =>
`["normal",${maxRetries},["RATE_LIMIT"],1,10000,0]`
const alwaysPolicyKey = '["always",1,10000,0]'
const normal = { provider: 'mock', mode: 'normal' as const, policyKey: normalPolicyKey(2) }
describe('llm-retry invariants', () => {
it('has no provider without the requested closed step', () => {
@@ -65,7 +68,8 @@ describe('llm-retry invariants', () => {
})
const zeroDelay = closeStep(ctx, 'retry-invariant-zero-delay')
zeroDelay.append('llm/retry', {
turn: 1, step: 1, ...normal, retry: 1, maxRetries: 1, delayMs: 0, failure,
turn: 1, step: 1, ...normal, policyKey: normalPolicyKey(1),
retry: 1, maxRetries: 1, delayMs: 0, failure,
})
}).not.toThrow()
expect(() => { ctx.emit('tools/change') }).not.toThrow()
@@ -80,6 +84,7 @@ describe('llm-retry invariants', () => {
step: 1,
provider: 'mock',
mode: 'always',
policyKey: alwaysPolicyKey,
retry: 1,
delayMs: 500,
failure,
@@ -91,6 +96,7 @@ describe('llm-retry invariants', () => {
step: 1,
provider: 'mock',
mode: 'always',
policyKey: alwaysPolicyKey,
retry: 1,
maxRetries: 2,
delayMs: 500,
@@ -99,6 +105,65 @@ describe('llm-retry invariants', () => {
}).toThrow(/always mode must omit maxRetries/)
})
it('binds event mode and finite budget to the canonical policy key', async () => {
const ctx = await setup()
const normalModeMismatch = closeStep(ctx, 'retry-invariant-normal-mode-key')
expect(() => {
normalModeMismatch.append('llm/retry', {
turn: 1, step: 1, ...normal, policyKey: alwaysPolicyKey,
retry: 1, maxRetries: 2, delayMs: 1, failure,
})
}).toThrow(/mode normal must match policyKey mode always/)
const alwaysModeMismatch = closeStep(ctx, 'retry-invariant-always-mode-key')
expect(() => {
alwaysModeMismatch.append('llm/retry', {
turn: 1,
step: 1,
provider: 'mock',
mode: 'always',
policyKey: normalPolicyKey(2),
retry: 1,
delayMs: 1,
failure,
})
}).toThrow(/mode always must match policyKey mode normal/)
const budgetMismatch = closeStep(ctx, 'retry-invariant-budget-key')
expect(() => {
budgetMismatch.append('llm/retry', {
turn: 1, step: 1, ...normal, policyKey: normalPolicyKey(3),
retry: 1, maxRetries: 2, delayMs: 1, failure,
})
}).toThrow(/maxRetries 2 must match policyKey/)
})
it('binds the failure code and scheduled delay to the canonical policy key', async () => {
const ctx = await setup()
const ineligibleFailure = closeStep(ctx, 'retry-invariant-failure-code-key')
expect(() => {
ineligibleFailure.append('llm/retry', {
turn: 1, step: 1, ...normal,
retry: 1, maxRetries: 2, delayMs: 1,
failure: { message: 'authentication failed', code: 'AUTH', status: 401 },
})
}).toThrow(/failure code AUTH must be eligible under policyKey/)
const overPolicyDelay = closeStep(ctx, 'retry-invariant-delay-key')
expect(() => {
overPolicyDelay.append('llm/retry', {
turn: 1,
step: 1,
provider: 'mock',
mode: 'always',
policyKey: '["always",1,1,0]',
retry: 1,
delayMs: 2,
failure,
})
}).toThrow(/within policyKey range 0\.\.1/)
})
it('rejects empty providers and unknown modes from hostile durable input', async () => {
const ctx = await setup()
const emptyProvider = closeStep(ctx, 'retry-invariant-empty-provider')
@@ -108,6 +173,7 @@ describe('llm-retry invariants', () => {
step: 1,
provider: '',
mode: 'always',
policyKey: alwaysPolicyKey,
retry: 1,
delayMs: 1,
failure,
@@ -121,11 +187,26 @@ describe('llm-retry invariants', () => {
step: 1,
provider: 'mock',
mode: 'sometimes',
policyKey: alwaysPolicyKey,
retry: 1,
delayMs: 1,
failure,
} as never)
}).toThrow(/mode must be normal or always/)
const emptyPolicyKey = closeStep(ctx, 'retry-invariant-empty-policy-key')
expect(() => {
emptyPolicyKey.append('llm/retry', {
turn: 1,
step: 1,
provider: 'mock',
mode: 'always',
policyKey: '',
retry: 1,
delayMs: 1,
failure,
})
}).toThrow(/policyKey must encode a canonical resolved policy/)
})
it.each([
@@ -206,6 +287,7 @@ describe('llm-retry invariants', () => {
step: 1,
provider: 'mock',
mode: 'always',
policyKey: alwaysPolicyKey,
retry: 1,
delayMs: 1,
failure,
@@ -219,6 +301,7 @@ describe('llm-retry invariants', () => {
step: 1,
provider: 'other',
mode: 'always',
policyKey: alwaysPolicyKey,
retry: 1,
delayMs: 1,
failure,
@@ -239,6 +322,7 @@ describe('llm-retry invariants', () => {
step: 1,
provider: 'mock',
mode: 'always',
policyKey: alwaysPolicyKey,
retry: 1,
delayMs: 1,
failure,
@@ -260,23 +344,27 @@ describe('llm-retry invariants', () => {
const ctx = await setup()
const duplicate = closeStep(ctx, 'retry-invariant-duplicate')
duplicate.append('llm/retry', {
turn: 1, step: 1, ...normal, retry: 1, maxRetries: 3, delayMs: 1, failure,
turn: 1, step: 1, ...normal, policyKey: normalPolicyKey(3),
retry: 1, maxRetries: 3, delayMs: 1, failure,
})
expect(() => {
duplicate.append('llm/retry', {
turn: 1, step: 1, ...normal, retry: 2, maxRetries: 3, delayMs: 1, failure,
turn: 1, step: 1, ...normal, policyKey: normalPolicyKey(3),
retry: 2, maxRetries: 3, delayMs: 1, failure,
})
}).toThrow(/duplicates the retry record/)
const nonIncreasing = closeStep(ctx, 'retry-invariant-non-increasing')
nonIncreasing.append('llm/retry', {
turn: 1, step: 1, ...normal, retry: 1, maxRetries: 3, delayMs: 1, failure,
turn: 1, step: 1, ...normal, policyKey: normalPolicyKey(3),
retry: 1, maxRetries: 3, delayMs: 1, failure,
})
nonIncreasing.append('step/start', { turn: 1, step: 2 })
nonIncreasing.append('step/end', { turn: 1, step: 2 })
expect(() => {
nonIncreasing.append('llm/retry', {
turn: 1, step: 2, ...normal, retry: 1, maxRetries: 3, delayMs: 1, failure,
turn: 1, step: 2, ...normal, policyKey: normalPolicyKey(3),
retry: 1, maxRetries: 3, delayMs: 1, failure,
})
}).toThrow(/must equal provider policy retry 2/)
})

1
packages/llm/llm-retry/tests/persistence.spec.ts
View File

@@ -44,6 +44,7 @@ describe.each(['jsonl', 'sqlite'] as const)('%s retry-event persistence', (kind)
step: 1,
provider: 'mock',
mode: 'always',
policyKey: '["always",500,10000,0.1]',
retry: 1,
delayMs: 750,
failure: { message: 'provider busy', code: 'RATE_LIMIT', status: 429 },

71
packages/llm/llm-retry/tests/policy-key.spec.ts Normal file
View File

@@ -0,0 +1,71 @@
import { describe, expect, it } from 'vitest'
import { resolveRetryPolicy } from '@deepseek-ai/dsh-llm'
import { MAX_TIMER_DELAY_MS } from '@deepseek-ai/dsh-timeout'
import { parseRetryPolicyKey, retryPolicyKey } from '../src/policy-key.ts'
describe('retry policy durable key', () => {
it('includes every policy field while normalizing code-set order', () => {
const first = resolveRetryPolicy({
mode: 'normal',
maxRetries: 4,
retryableCodes: ['SERVER', 'RATE_LIMIT'],
backoff: { initialDelayMs: 3, maxDelayMs: 9, jitterRatio: 0.25 },
}, 'first')
const reordered = resolveRetryPolicy({
mode: 'normal',
maxRetries: 4,
retryableCodes: ['RATE_LIMIT', 'SERVER'],
backoff: { initialDelayMs: 3, maxDelayMs: 9, jitterRatio: 0.25 },
}, 'reordered')
const key = retryPolicyKey(first)
expect(key).toBe('["normal",4,["RATE_LIMIT","SERVER"],3,9,0.25]')
expect(retryPolicyKey(reordered)).toBe(key)
expect(parseRetryPolicyKey(key)).toEqual(reordered)
})
it('round-trips always mode', () => {
const policy = resolveRetryPolicy({
mode: 'always',
backoff: { initialDelayMs: 2, maxDelayMs: 8, jitterRatio: 1 },
}, 'always')
const key = retryPolicyKey(policy)
expect(key).toBe('["always",2,8,1]')
expect(parseRetryPolicyKey(key)).toEqual(policy)
})
it.each([
undefined,
'',
'{',
'{}',
'["sometimes",1,2,0]',
'["always",1,2]',
'["always","1",2,0]',
'["always",1e400,2,0]',
'["always",0,2,0]',
`["always",${MAX_TIMER_DELAY_MS + 1},${MAX_TIMER_DELAY_MS + 1},0]`,
'["always",1,"2",0]',
'["always",1,1e400,0]',
'["always",1,0,0]',
`["always",1,${MAX_TIMER_DELAY_MS + 1},0]`,
'["always",2,1,0]',
'["always",1,2,"0"]',
'["always",1,2,1e400]',
'["always",1,2,-0.1]',
'["always",1,2,1.1]',
'["normal",2,["SERVER"],1,2]',
'["normal","2",["SERVER"],1,2,0]',
'["normal",-1,["SERVER"],1,2,0]',
'["normal",2,"SERVER",1,2,0]',
'["normal",2,[],1,2,0]',
'["normal",2,[1],1,2,0]',
'["normal",2,[""],1,2,0]',
'["normal",2,["SERVER","SERVER"],1,2,0]',
'["normal",2,["SERVER"],2,1,0]',
'["normal",2,["SERVER","RATE_LIMIT"],1,2,0]',
])('rejects non-canonical durable input %#', (value) => {
expect(parseRetryPolicyKey(value)).toBeUndefined()
})
})

105
packages/llm/llm-retry/tests/retry.spec.ts
View File

@@ -184,6 +184,7 @@ describe('provider-routed retry policy', () => {
step: 1,
provider: 'mock',
mode: 'normal',
policyKey: '["normal",2,["RATE_LIMIT","SERVER","TIMEOUT","TRANSPORT"],500,10000,0.1]',
retry: 1,
maxRetries: 2,
delayMs: 500,
@@ -561,6 +562,110 @@ describe('provider-routed retry policy', () => {
},
)
it('starts a new retry history when a same-mode route replacement changes policy', async () => {
vi.useFakeTimers()
const oldAdapter = new ScriptedAdapter([
new LlmError('old route failed', 'AUTH'),
])
const mounted = await harness(oldAdapter, { mock: alwaysConfig({
initialDelayMs: 1,
maxDelayMs: 1,
}) })
context = mounted.ctx
const agent = context.agentLoop.create(SessionId('retry-policy-replacement'), {
provider: 'mock',
model: 'mock',
})
const first = waitForRetry(context, agent, 1)
agent.followup([{ type: 'text', text: 'replace policy between attempts' }])
expect((await first).data).toMatchObject({
mode: 'always',
retry: 1,
delayMs: 1,
})
mounted.disposeAdapter()
const replacement = new ScriptedAdapter([
new LlmError('replacement failed', 'AUTH'),
textResponse('replacement recovered'),
])
replacement.configureRetryPolicies({ mock: alwaysConfig({
initialDelayMs: 3,
maxDelayMs: 9,
}) })
context.llm.registerAdapter(['mock'], replacement)
const second = waitForRetry(context, agent, 1)
await vi.advanceTimersByTimeAsync(1)
expect((await second).data).toMatchObject({
mode: 'always',
retry: 1,
delayMs: 3,
})
const idle = waitForIdle(context, agent)
await vi.advanceTimersByTimeAsync(3)
await idle
expect(oldAdapter.requests).toHaveLength(1)
expect(replacement.requests).toHaveLength(2)
expect(agent.session.events.filter(event => event.type === 'llm/retry').map(event => ({
policyKey: event.data.policyKey,
retry: event.data.retry,
}))).toEqual([
{ policyKey: '["always",1,1,0]', retry: 1 },
{ policyKey: '["always",3,9,0]', retry: 1 },
])
})
it('continues retry history when a replacement only reorders retryable codes', async () => {
vi.useFakeTimers()
const oldAdapter = new ScriptedAdapter([
new LlmError('old route failed', 'SERVER'),
])
const mounted = await harness(oldAdapter, { mock: normalConfig({
maxRetries: 2,
retryableCodes: ['SERVER', 'RATE_LIMIT'],
backoff: { initialDelayMs: 1, maxDelayMs: 4 },
}) })
context = mounted.ctx
const agent = context.agentLoop.create(SessionId('retry-policy-code-order'), {
provider: 'mock',
model: 'mock',
})
const first = waitForRetry(context, agent, 1)
agent.followup([{ type: 'text', text: 'replace equivalent policy between attempts' }])
const firstEvent = await first
expect(firstEvent.data.delayMs).toBe(1)
mounted.disposeAdapter()
const replacement = new ScriptedAdapter([
new LlmError('replacement failed', 'SERVER'),
textResponse('replacement recovered'),
])
replacement.configureRetryPolicies({ mock: normalConfig({
maxRetries: 2,
retryableCodes: ['RATE_LIMIT', 'SERVER'],
backoff: { initialDelayMs: 1, maxDelayMs: 4 },
}) })
context.llm.registerAdapter(['mock'], replacement)
const second = waitForRetry(context, agent, 2)
await vi.advanceTimersByTimeAsync(1)
const secondEvent = await second
expect(secondEvent.data).toMatchObject({ retry: 2, delayMs: 2 })
expect(secondEvent.data.policyKey).toBe(firstEvent.data.policyKey)
const idle = waitForIdle(context, agent)
await vi.advanceTimersByTimeAsync(2)
await idle
expect(oldAdapter.requests).toHaveLength(1)
expect(replacement.requests).toHaveLength(2)
})
it('keeps always mode unbounded while preserving cancellable jittered backoff', async () => {
vi.useFakeTimers()
const adapter = new ScriptedAdapter([