Coder/deepseek-harness
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(llm): isolate retry policy histories

Browse Source
This commit is contained in:
Turtle
2026-07-25 15:38:58 +08:00
parent 38ce422f71
commit efc725b7e4
31 changed files with 666 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

100
packages/llm/llm-retry/tests/invariant.spec.ts
View File

@@ -27,7 +27,10 @@ function closeStep(ctx: Context, id: string, turn = 1, step = 1) {
}
const failure = { message: 'provider busy', code: 'RATE_LIMIT', status: 429 }
const normal = { provider: 'mock', mode: 'normal' as const }
const normalPolicyKey = (maxRetries: number): string =>
`["normal",${maxRetries},["RATE_LIMIT"],1,10000,0]`
const alwaysPolicyKey = '["always",1,10000,0]'
const normal = { provider: 'mock', mode: 'normal' as const, policyKey: normalPolicyKey(2) }
describe('llm-retry invariants', () => {
it('has no provider without the requested closed step', () => {
@@ -65,7 +68,8 @@ describe('llm-retry invariants', () => {
})
const zeroDelay = closeStep(ctx, 'retry-invariant-zero-delay')
zeroDelay.append('llm/retry', {
turn: 1, step: 1, ...normal, retry: 1, maxRetries: 1, delayMs: 0, failure,
turn: 1, step: 1, ...normal, policyKey: normalPolicyKey(1),
retry: 1, maxRetries: 1, delayMs: 0, failure,
})
}).not.toThrow()
expect(() => { ctx.emit('tools/change') }).not.toThrow()
@@ -80,6 +84,7 @@ describe('llm-retry invariants', () => {
step: 1,
provider: 'mock',
mode: 'always',
policyKey: alwaysPolicyKey,
retry: 1,
delayMs: 500,
failure,
@@ -91,6 +96,7 @@ describe('llm-retry invariants', () => {
step: 1,
provider: 'mock',
mode: 'always',
policyKey: alwaysPolicyKey,
retry: 1,
maxRetries: 2,
delayMs: 500,
@@ -99,6 +105,65 @@ describe('llm-retry invariants', () => {
}).toThrow(/always mode must omit maxRetries/)
})
it('binds event mode and finite budget to the canonical policy key', async () => {
const ctx = await setup()
const normalModeMismatch = closeStep(ctx, 'retry-invariant-normal-mode-key')
expect(() => {
normalModeMismatch.append('llm/retry', {
turn: 1, step: 1, ...normal, policyKey: alwaysPolicyKey,
retry: 1, maxRetries: 2, delayMs: 1, failure,
})
}).toThrow(/mode normal must match policyKey mode always/)
const alwaysModeMismatch = closeStep(ctx, 'retry-invariant-always-mode-key')
expect(() => {
alwaysModeMismatch.append('llm/retry', {
turn: 1,
step: 1,
provider: 'mock',
mode: 'always',
policyKey: normalPolicyKey(2),
retry: 1,
delayMs: 1,
failure,
})
}).toThrow(/mode always must match policyKey mode normal/)
const budgetMismatch = closeStep(ctx, 'retry-invariant-budget-key')
expect(() => {
budgetMismatch.append('llm/retry', {
turn: 1, step: 1, ...normal, policyKey: normalPolicyKey(3),
retry: 1, maxRetries: 2, delayMs: 1, failure,
})
}).toThrow(/maxRetries 2 must match policyKey/)
})
it('binds the failure code and scheduled delay to the canonical policy key', async () => {
const ctx = await setup()
const ineligibleFailure = closeStep(ctx, 'retry-invariant-failure-code-key')
expect(() => {
ineligibleFailure.append('llm/retry', {
turn: 1, step: 1, ...normal,
retry: 1, maxRetries: 2, delayMs: 1,
failure: { message: 'authentication failed', code: 'AUTH', status: 401 },
})
}).toThrow(/failure code AUTH must be eligible under policyKey/)
const overPolicyDelay = closeStep(ctx, 'retry-invariant-delay-key')
expect(() => {
overPolicyDelay.append('llm/retry', {
turn: 1,
step: 1,
provider: 'mock',
mode: 'always',
policyKey: '["always",1,1,0]',
retry: 1,
delayMs: 2,
failure,
})
}).toThrow(/within policyKey range 0\.\.1/)
})
it('rejects empty providers and unknown modes from hostile durable input', async () => {
const ctx = await setup()
const emptyProvider = closeStep(ctx, 'retry-invariant-empty-provider')
@@ -108,6 +173,7 @@ describe('llm-retry invariants', () => {
step: 1,
provider: '',
mode: 'always',
policyKey: alwaysPolicyKey,
retry: 1,
delayMs: 1,
failure,
@@ -121,11 +187,26 @@ describe('llm-retry invariants', () => {
step: 1,
provider: 'mock',
mode: 'sometimes',
policyKey: alwaysPolicyKey,
retry: 1,
delayMs: 1,
failure,
} as never)
}).toThrow(/mode must be normal or always/)
const emptyPolicyKey = closeStep(ctx, 'retry-invariant-empty-policy-key')
expect(() => {
emptyPolicyKey.append('llm/retry', {
turn: 1,
step: 1,
provider: 'mock',
mode: 'always',
policyKey: '',
retry: 1,
delayMs: 1,
failure,
})
}).toThrow(/policyKey must encode a canonical resolved policy/)
})
it.each([
@@ -206,6 +287,7 @@ describe('llm-retry invariants', () => {
step: 1,
provider: 'mock',
mode: 'always',
policyKey: alwaysPolicyKey,
retry: 1,
delayMs: 1,
failure,
@@ -219,6 +301,7 @@ describe('llm-retry invariants', () => {
step: 1,
provider: 'other',
mode: 'always',
policyKey: alwaysPolicyKey,
retry: 1,
delayMs: 1,
failure,
@@ -239,6 +322,7 @@ describe('llm-retry invariants', () => {
step: 1,
provider: 'mock',
mode: 'always',
policyKey: alwaysPolicyKey,
retry: 1,
delayMs: 1,
failure,
@@ -260,23 +344,27 @@ describe('llm-retry invariants', () => {
const ctx = await setup()
const duplicate = closeStep(ctx, 'retry-invariant-duplicate')
duplicate.append('llm/retry', {
turn: 1, step: 1, ...normal, retry: 1, maxRetries: 3, delayMs: 1, failure,
turn: 1, step: 1, ...normal, policyKey: normalPolicyKey(3),
retry: 1, maxRetries: 3, delayMs: 1, failure,
})
expect(() => {
duplicate.append('llm/retry', {
turn: 1, step: 1, ...normal, retry: 2, maxRetries: 3, delayMs: 1, failure,
turn: 1, step: 1, ...normal, policyKey: normalPolicyKey(3),
retry: 2, maxRetries: 3, delayMs: 1, failure,
})
}).toThrow(/duplicates the retry record/)
const nonIncreasing = closeStep(ctx, 'retry-invariant-non-increasing')
nonIncreasing.append('llm/retry', {
turn: 1, step: 1, ...normal, retry: 1, maxRetries: 3, delayMs: 1, failure,
turn: 1, step: 1, ...normal, policyKey: normalPolicyKey(3),
retry: 1, maxRetries: 3, delayMs: 1, failure,
})
nonIncreasing.append('step/start', { turn: 1, step: 2 })
nonIncreasing.append('step/end', { turn: 1, step: 2 })
expect(() => {
nonIncreasing.append('llm/retry', {
turn: 1, step: 2, ...normal, retry: 1, maxRetries: 3, delayMs: 1, failure,
turn: 1, step: 2, ...normal, policyKey: normalPolicyKey(3),
retry: 1, maxRetries: 3, delayMs: 1, failure,
})
}).toThrow(/must equal provider policy retry 2/)
})

1
packages/llm/llm-retry/tests/persistence.spec.ts
View File

@@ -44,6 +44,7 @@ describe.each(['jsonl', 'sqlite'] as const)('%s retry-event persistence', (kind)
step: 1,
provider: 'mock',
mode: 'always',
policyKey: '["always",500,10000,0.1]',
retry: 1,
delayMs: 750,
failure: { message: 'provider busy', code: 'RATE_LIMIT', status: 429 },

71
packages/llm/llm-retry/tests/policy-key.spec.ts Normal file
View File

@@ -0,0 +1,71 @@
import { describe, expect, it } from 'vitest'
import { resolveRetryPolicy } from '@deepseek-ai/dsh-llm'
import { MAX_TIMER_DELAY_MS } from '@deepseek-ai/dsh-timeout'
import { parseRetryPolicyKey, retryPolicyKey } from '../src/policy-key.ts'
describe('retry policy durable key', () => {
it('includes every policy field while normalizing code-set order', () => {
const first = resolveRetryPolicy({
mode: 'normal',
maxRetries: 4,
retryableCodes: ['SERVER', 'RATE_LIMIT'],
backoff: { initialDelayMs: 3, maxDelayMs: 9, jitterRatio: 0.25 },
}, 'first')
const reordered = resolveRetryPolicy({
mode: 'normal',
maxRetries: 4,
retryableCodes: ['RATE_LIMIT', 'SERVER'],
backoff: { initialDelayMs: 3, maxDelayMs: 9, jitterRatio: 0.25 },
}, 'reordered')
const key = retryPolicyKey(first)
expect(key).toBe('["normal",4,["RATE_LIMIT","SERVER"],3,9,0.25]')
expect(retryPolicyKey(reordered)).toBe(key)
expect(parseRetryPolicyKey(key)).toEqual(reordered)
})
it('round-trips always mode', () => {
const policy = resolveRetryPolicy({
mode: 'always',
backoff: { initialDelayMs: 2, maxDelayMs: 8, jitterRatio: 1 },
}, 'always')
const key = retryPolicyKey(policy)
expect(key).toBe('["always",2,8,1]')
expect(parseRetryPolicyKey(key)).toEqual(policy)
})
it.each([
undefined,
'',
'{',
'{}',
'["sometimes",1,2,0]',
'["always",1,2]',
'["always","1",2,0]',
'["always",1e400,2,0]',
'["always",0,2,0]',
`["always",${MAX_TIMER_DELAY_MS + 1},${MAX_TIMER_DELAY_MS + 1},0]`,
'["always",1,"2",0]',
'["always",1,1e400,0]',
'["always",1,0,0]',
`["always",1,${MAX_TIMER_DELAY_MS + 1},0]`,
'["always",2,1,0]',
'["always",1,2,"0"]',
'["always",1,2,1e400]',
'["always",1,2,-0.1]',
'["always",1,2,1.1]',
'["normal",2,["SERVER"],1,2]',
'["normal","2",["SERVER"],1,2,0]',
'["normal",-1,["SERVER"],1,2,0]',
'["normal",2,"SERVER",1,2,0]',
'["normal",2,[],1,2,0]',
'["normal",2,[1],1,2,0]',
'["normal",2,[""],1,2,0]',
'["normal",2,["SERVER","SERVER"],1,2,0]',
'["normal",2,["SERVER"],2,1,0]',
'["normal",2,["SERVER","RATE_LIMIT"],1,2,0]',
])('rejects non-canonical durable input %#', (value) => {
expect(parseRetryPolicyKey(value)).toBeUndefined()
})
})

105
packages/llm/llm-retry/tests/retry.spec.ts
View File

@@ -184,6 +184,7 @@ describe('provider-routed retry policy', () => {
step: 1,
provider: 'mock',
mode: 'normal',
policyKey: '["normal",2,["RATE_LIMIT","SERVER","TIMEOUT","TRANSPORT"],500,10000,0.1]',
retry: 1,
maxRetries: 2,
delayMs: 500,
@@ -561,6 +562,110 @@ describe('provider-routed retry policy', () => {
},
)
it('starts a new retry history when a same-mode route replacement changes policy', async () => {
vi.useFakeTimers()
const oldAdapter = new ScriptedAdapter([
new LlmError('old route failed', 'AUTH'),
])
const mounted = await harness(oldAdapter, { mock: alwaysConfig({
initialDelayMs: 1,
maxDelayMs: 1,
}) })
context = mounted.ctx
const agent = context.agentLoop.create(SessionId('retry-policy-replacement'), {
provider: 'mock',
model: 'mock',
})
const first = waitForRetry(context, agent, 1)
agent.followup([{ type: 'text', text: 'replace policy between attempts' }])
expect((await first).data).toMatchObject({
mode: 'always',
retry: 1,
delayMs: 1,
})
mounted.disposeAdapter()
const replacement = new ScriptedAdapter([
new LlmError('replacement failed', 'AUTH'),
textResponse('replacement recovered'),
])
replacement.configureRetryPolicies({ mock: alwaysConfig({
initialDelayMs: 3,
maxDelayMs: 9,
}) })
context.llm.registerAdapter(['mock'], replacement)
const second = waitForRetry(context, agent, 1)
await vi.advanceTimersByTimeAsync(1)
expect((await second).data).toMatchObject({
mode: 'always',
retry: 1,
delayMs: 3,
})
const idle = waitForIdle(context, agent)
await vi.advanceTimersByTimeAsync(3)
await idle
expect(oldAdapter.requests).toHaveLength(1)
expect(replacement.requests).toHaveLength(2)
expect(agent.session.events.filter(event => event.type === 'llm/retry').map(event => ({
policyKey: event.data.policyKey,
retry: event.data.retry,
}))).toEqual([
{ policyKey: '["always",1,1,0]', retry: 1 },
{ policyKey: '["always",3,9,0]', retry: 1 },
])
})
it('continues retry history when a replacement only reorders retryable codes', async () => {
vi.useFakeTimers()
const oldAdapter = new ScriptedAdapter([
new LlmError('old route failed', 'SERVER'),
])
const mounted = await harness(oldAdapter, { mock: normalConfig({
maxRetries: 2,
retryableCodes: ['SERVER', 'RATE_LIMIT'],
backoff: { initialDelayMs: 1, maxDelayMs: 4 },
}) })
context = mounted.ctx
const agent = context.agentLoop.create(SessionId('retry-policy-code-order'), {
provider: 'mock',
model: 'mock',
})
const first = waitForRetry(context, agent, 1)
agent.followup([{ type: 'text', text: 'replace equivalent policy between attempts' }])
const firstEvent = await first
expect(firstEvent.data.delayMs).toBe(1)
mounted.disposeAdapter()
const replacement = new ScriptedAdapter([
new LlmError('replacement failed', 'SERVER'),
textResponse('replacement recovered'),
])
replacement.configureRetryPolicies({ mock: normalConfig({
maxRetries: 2,
retryableCodes: ['RATE_LIMIT', 'SERVER'],
backoff: { initialDelayMs: 1, maxDelayMs: 4 },
}) })
context.llm.registerAdapter(['mock'], replacement)
const second = waitForRetry(context, agent, 2)
await vi.advanceTimersByTimeAsync(1)
const secondEvent = await second
expect(secondEvent.data).toMatchObject({ retry: 2, delayMs: 2 })
expect(secondEvent.data.policyKey).toBe(firstEvent.data.policyKey)
const idle = waitForIdle(context, agent)
await vi.advanceTimersByTimeAsync(2)
await idle
expect(oldAdapter.requests).toHaveLength(1)
expect(replacement.requests).toHaveLength(2)
})
it('keeps always mode unbounded while preserving cancellable jittered backoff', async () => {
vi.useFakeTimers()
const adapter = new ScriptedAdapter([