fix(agent-loop): always close a started turn and any open step on error (P1-5)
After turn/start was appended, nothing guaranteed a matching turn/end: a throw from a boundary emit (agent/turn-start, agent/step-start, the normal-path agent/turn-end) escaped runTurn, and the outer runLoop backstop logged an error but never appended turn/end — leaving an unbalanced turn that replay, telemetry, and the invariants plugin all assume is impossible. runTurn is restructured around idempotent finalizers that satisfy the four traps a naive finally would hit: - closeStep()/closeTurn(emit) are guarded (stepOpen/turnEnded) so they run at most once; the agent/step-end and agent/error emits are contained so a throwing listener can't strand the turn open. - failTurn() records the single error event + reason and emits agent/error exactly once (errorReported guard) — no double-logging when the outer catch also runs (e.g. a step error followed by a throwing turn-end listener). - the catch closes an open step BEFORE turn/end (invariants reject turn/end while a step is open), and rethrows ONLY pre-turn throws (turnStarted false), where no turn/end is owed, so the backstop still nets them. - disposal precedence: reason stays disposed only when disposed AND no error was reported; otherwise the error reason wins. Tests (with the invariants plugin loaded as a balance oracle): throwing turn-start (one error, one turn/end, no step), throwing step-start (step/end before turn/end), throwing agent/error on a step-error path (balanced, loop survives), disposal mid-turn (reason disposed, no error event), a pre-turn turn/start-append throw (rethrown to the backstop, no turn/end owed), and a step error + throwing turn-end listener (error logged exactly once). Verified all six fail against a simulated finalizer bypass. dsh-invariants added as an agent-loop devDependency (test-only oracle; no package cycle).
This commit is contained in:
Tianyi Cui
@@ -6,6 +6,7 @@ import SystemPrompt from '@deepseek-ai/dsh-system-prompt'
|
||||
import ToolRegistry, { defineTool } from '@deepseek-ai/dsh-tools'
|
||||
import AgentRegistry, { AgentId } from '@deepseek-ai/dsh-agent'
|
||||
import AgentLoop, { LoopAgent } from '@deepseek-ai/dsh-agent-loop'
|
||||
import * as Invariants from '@deepseek-ai/dsh-invariants'
|
||||
import { MockAdapter, textResponse, toolCallResponse } from './mock-adapter.ts'
|
||||
|
||||
/**
|
||||
@@ -639,3 +640,250 @@ describe('P1-6: step/start is appended before agent/step-start is emitted', () =
|
||||
expect(observed[0]).toMatchObject({ turn: 1, step: 1, lastEventType: 'step/start', sawStepStart: true })
|
||||
})
|
||||
})
|
||||
|
||||
describe('P1-5: a started turn (and any open step) is always closed on a boundary throw', () => {
|
||||
// Harness with the invariants plugin loaded as an oracle: it throws on
|
||||
// append if the log goes unbalanced (turn/end while a step is open,
|
||||
// turn/start while a turn is open, etc.), so a regression surfaces as an
|
||||
// InvariantError on the NEXT turn's append rather than a silent imbalance.
|
||||
async function balancedHarness(adapter: MockAdapter) {
|
||||
const ctx = new Context()
|
||||
await ctx.plugin(LlmService)
|
||||
await ctx.plugin(SessionStore)
|
||||
await ctx.plugin(SystemPrompt)
|
||||
await ctx.plugin(ToolRegistry)
|
||||
await ctx.plugin(AgentRegistry)
|
||||
await ctx.plugin(AgentLoop, { agents: [] })
|
||||
await ctx.plugin(Invariants, { freeze: false })
|
||||
ctx.llm.registerAdapter(['mock'], adapter)
|
||||
return ctx
|
||||
}
|
||||
|
||||
/** Count turn/step boundary events for balance assertions. */
|
||||
function boundaryCounts(agent: LoopAgent) {
|
||||
const e = [...agent.session.events]
|
||||
return {
|
||||
turnStart: e.filter(x => x.type === 'turn/start').length,
|
||||
turnEnd: e.filter(x => x.type === 'turn/end').length,
|
||||
stepStart: e.filter(x => x.type === 'step/start').length,
|
||||
stepEnd: e.filter(x => x.type === 'step/end').length,
|
||||
errors: e.filter(x => x.type === 'error').length,
|
||||
lastTurnEnd: e.findLast(x => x.type === 'turn/end'),
|
||||
}
|
||||
}
|
||||
|
||||
it('a throwing agent/turn-start listener still closes the turn with exactly one error and one turn/end, no step', async () => {
|
||||
const adapter = new MockAdapter([textResponse('never reached')])
|
||||
const ctx = await balancedHarness(adapter)
|
||||
const agent = ctx.agentLoop.create('a-turnstart', { model: 'mock' })
|
||||
|
||||
let threw = false
|
||||
ctx.on('agent/turn-start', () => { if (!threw) { threw = true; throw new Error('boom turn-start') } })
|
||||
const errors: Error[] = []
|
||||
ctx.on('agent/error', (_a, _t, _s, error) => void errors.push(error))
|
||||
|
||||
send(agent, 'go')
|
||||
await waitForIdle(ctx, agent)
|
||||
|
||||
const c = boundaryCounts(agent)
|
||||
// turn opened and closed; no step ran; exactly one error logged + emitted.
|
||||
expect(c).toMatchObject({ turnStart: 1, turnEnd: 1, stepStart: 0, stepEnd: 0, errors: 1 })
|
||||
expect(errors.map(e => e.message)).toEqual(['boom turn-start'])
|
||||
expect(c.lastTurnEnd?.type === 'turn/end' && c.lastTurnEnd.data.reason).toEqual({ kind: 'error', message: 'boom turn-start' })
|
||||
// model was never called (we threw before the step's request).
|
||||
expect(adapter.requests).toHaveLength(0)
|
||||
})
|
||||
|
||||
it('a throwing agent/step-start listener closes the open step then the turn (step/end before turn/end)', async () => {
|
||||
const adapter = new MockAdapter([textResponse('never reached')])
|
||||
const ctx = await balancedHarness(adapter)
|
||||
const agent = ctx.agentLoop.create('a-stepstart', { model: 'mock' })
|
||||
|
||||
let threw = false
|
||||
ctx.on('agent/step-start', () => { if (!threw) { threw = true; throw new Error('boom step-start') } })
|
||||
const errors: Error[] = []
|
||||
ctx.on('agent/error', (_a, _t, _s, error) => void errors.push(error))
|
||||
|
||||
send(agent, 'go')
|
||||
await waitForIdle(ctx, agent)
|
||||
|
||||
const e = [...agent.session.events]
|
||||
const c = boundaryCounts(agent)
|
||||
expect(c).toMatchObject({ turnStart: 1, turnEnd: 1, stepStart: 1, stepEnd: 1, errors: 1 })
|
||||
expect(errors.map(x => x.message)).toEqual(['boom step-start'])
|
||||
// step/end must precede turn/end (the invariants oracle would reject
|
||||
// turn/end-while-step-open, but assert the order explicitly too).
|
||||
const stepEndIdx = e.findIndex(x => x.type === 'step/end')
|
||||
const turnEndIdx = e.findIndex(x => x.type === 'turn/end')
|
||||
expect(stepEndIdx).toBeGreaterThanOrEqual(0)
|
||||
expect(stepEndIdx).toBeLessThan(turnEndIdx)
|
||||
})
|
||||
|
||||
it('a throwing agent/error listener during a step-error path still balances the turn, loop survives', async () => {
|
||||
// First turn: model stream ends with a finish-error → step error path →
|
||||
// failTurn emits agent/error, whose listener throws. The turn must still
|
||||
// close balanced. Second turn proves the loop survived.
|
||||
const errorStream: StreamChunk[] = [{ type: 'finish', reason: { kind: 'error', message: 'provider 500' } }]
|
||||
const adapter = new MockAdapter([errorStream, textResponse('turn 2 ok')])
|
||||
const ctx = await balancedHarness(adapter)
|
||||
const agent = ctx.agentLoop.create('a-errorlistener', { model: 'mock' })
|
||||
|
||||
let threw = false
|
||||
ctx.on('agent/error', () => { if (!threw) { threw = true; throw new Error('boom error-listener') } })
|
||||
|
||||
send(agent, 'go')
|
||||
await waitForIdle(ctx, agent)
|
||||
|
||||
const c = boundaryCounts(agent)
|
||||
// turn 1 balanced despite the throwing agent/error listener.
|
||||
expect(c.turnStart).toBe(1)
|
||||
expect(c.turnEnd).toBe(1)
|
||||
expect(c.stepStart).toBe(c.stepEnd)
|
||||
expect(c.lastTurnEnd?.type === 'turn/end' && c.lastTurnEnd.data.reason).toMatchObject({ kind: 'error', message: 'provider 500' })
|
||||
|
||||
// loop survives: a second turn runs to completion (invariants oracle would
|
||||
// throw on its turn/start if turn 1 had been left open).
|
||||
send(agent, 'again')
|
||||
await waitForIdle(ctx, agent)
|
||||
const c2 = boundaryCounts(agent)
|
||||
expect(c2.turnStart).toBe(2)
|
||||
expect(c2.turnEnd).toBe(2)
|
||||
expect(c2.stepStart).toBe(c2.stepEnd)
|
||||
})
|
||||
|
||||
it('disposal during a running turn ends the turn with reason disposed (balanced)', async () => {
|
||||
// The 'hang' adapter blocks in stream() until the signal aborts; disposing
|
||||
// the agent's fiber mid-turn aborts the in-flight step. The turn must close
|
||||
// balanced with reason disposed (no error event for a disposal).
|
||||
const adapter = new MockAdapter(['hang'])
|
||||
const ctx = await balancedHarness(adapter)
|
||||
let agent!: LoopAgent
|
||||
const fiber = await ctx.plugin(Object.assign((inner: Context) => {
|
||||
agent = inner.agentLoop.create('a-dispose', { model: 'mock' })
|
||||
}, { inject: ['agentLoop'] }))
|
||||
|
||||
const reasons: TurnEndReason[] = []
|
||||
ctx.on('agent/turn-end', (_a, _t, reason) => void reasons.push(reason))
|
||||
|
||||
send(agent, 'go')
|
||||
await new Promise(r => setTimeout(r, 30))
|
||||
await fiber.dispose() // dispose during the hanging step
|
||||
await agent.done
|
||||
|
||||
const e = [...agent.session.events]
|
||||
const turnStarts = e.filter(x => x.type === 'turn/start').length
|
||||
const turnEnds = e.filter(x => x.type === 'turn/end').length
|
||||
expect(turnStarts).toBe(1)
|
||||
expect(turnEnds).toBe(1) // balanced — the turn was closed despite disposal
|
||||
expect(reasons).toEqual([{ kind: 'disposed' }])
|
||||
// no error event: disposal is not a failure.
|
||||
expect(e.some(x => x.type === 'error')).toBe(false)
|
||||
})
|
||||
|
||||
it('preserves reason disposed when the turn-end emit throws during disposal (outer-catch disposed branch)', async () => {
|
||||
// Dispose mid-step → the step-error branch sets reason=disposed (no error
|
||||
// reported). closeTurn(true) then emits agent/turn-end, whose listener
|
||||
// throws → control reaches the outer catch with isDisposed() && !errorReported,
|
||||
// which must PRESERVE disposed rather than overwrite it with the listener's
|
||||
// throw. This is the only path that exercises that catch sub-branch.
|
||||
const adapter = new MockAdapter(['hang'])
|
||||
const ctx = await balancedHarness(adapter)
|
||||
let agent!: LoopAgent
|
||||
const fiber = await ctx.plugin(Object.assign((inner: Context) => {
|
||||
agent = inner.agentLoop.create('a-dispose-emit-throw', { model: 'mock' })
|
||||
}, { inject: ['agentLoop'] }))
|
||||
|
||||
// The FIRST agent/turn-end emit throws (the disposal-driven turn end).
|
||||
let threw = false
|
||||
ctx.on('agent/turn-end', () => { if (!threw) { threw = true; throw new Error('boom turn-end during disposal') } })
|
||||
// Collect agent/error emissions to prove none is surfaced through that
|
||||
// channel either (the listener throw must be fully contained).
|
||||
const errorEmits: Error[] = []
|
||||
ctx.on('agent/error', (_a, _t, _s, error) => void errorEmits.push(error))
|
||||
|
||||
send(agent, 'go')
|
||||
await new Promise(r => setTimeout(r, 30))
|
||||
await fiber.dispose() // dispose during the hanging step
|
||||
await agent.done
|
||||
|
||||
// The throwing turn-end listener actually fired — proving the outer-catch
|
||||
// path was exercised, not skipped.
|
||||
expect(threw).toBe(true)
|
||||
|
||||
const e = [...agent.session.events]
|
||||
// Exactly one turn/start and one turn/end (balanced); the turn/end carries
|
||||
// the disposed reason, NOT an error reason from the throwing listener.
|
||||
expect(e.filter(x => x.type === 'turn/start')).toHaveLength(1)
|
||||
const turnEnd = e.findLast(x => x.type === 'turn/end')
|
||||
expect(turnEnd?.type === 'turn/end' && turnEnd.data.reason).toEqual({ kind: 'disposed' })
|
||||
// The throwing turn-end listener is contained: no error event is logged and
|
||||
// no agent/error is emitted (disposal is not a failure; the throw is swallowed).
|
||||
expect(e.some(x => x.type === 'error')).toBe(false)
|
||||
expect(errorEmits).toHaveLength(0)
|
||||
})
|
||||
|
||||
it('a throw at the turn/start append (before turnStarted) is rethrown to the runLoop backstop', async () => {
|
||||
// A session/event listener that throws specifically on the turn/start
|
||||
// event makes session.append('turn/start') throw while turnStarted is
|
||||
// still false. runTurn must NOT try to close a turn it never opened — it
|
||||
// rethrows, and the runLoop backstop records the error and survives.
|
||||
// (Uses the plain harness — NOT the invariants oracle — because the
|
||||
// throwing listener is itself a session/event subscriber.)
|
||||
const adapter = new MockAdapter([textResponse('turn 2')])
|
||||
const ctx = await harness(adapter)
|
||||
const agent = ctx.agentLoop.create('a-preturn', { model: 'mock' })
|
||||
|
||||
let threw = false
|
||||
ctx.on('session/event', (_session, event) => {
|
||||
if (!threw && event.type === 'turn/start') { threw = true; throw new Error('boom turn/start append') }
|
||||
})
|
||||
const errors: Error[] = []
|
||||
ctx.on('agent/error', (_a, _t, _s, error) => void errors.push(error))
|
||||
|
||||
send(agent, 'go')
|
||||
await waitForIdle(ctx, agent)
|
||||
|
||||
// The backstop logged exactly one error for the failed pre-turn append.
|
||||
expect(errors.map(e => e.message)).toEqual(['boom turn/start append'])
|
||||
// No turn/end was appended (none is owed — the turn never opened).
|
||||
expect([...agent.session.events].some(e => e.type === 'turn/end')).toBe(false)
|
||||
|
||||
// loop survives: a second turn runs normally.
|
||||
send(agent, 'second')
|
||||
await waitForIdle(ctx, agent)
|
||||
expect(adapter.requests).toHaveLength(1)
|
||||
})
|
||||
|
||||
it('a step error followed by a throwing turn-end listener logs the error exactly once (no double-report)', async () => {
|
||||
// The step fails (finish-error) → failTurn records ONE error and sets the
|
||||
// error reason. closeTurn(true) then appends turn/end and emits
|
||||
// agent/turn-end, whose listener throws → the outer catch calls failTurn
|
||||
// again, but its errorReported guard makes it a no-op. Trap #1: exactly one
|
||||
// error, the turn stays balanced.
|
||||
const errorStream: StreamChunk[] = [{ type: 'finish', reason: { kind: 'error', message: 'provider down' } }]
|
||||
const adapter = new MockAdapter([errorStream, textResponse('turn 2 ok')])
|
||||
const ctx = await balancedHarness(adapter)
|
||||
const agent = ctx.agentLoop.create('a-double', { model: 'mock' })
|
||||
|
||||
let threw = false
|
||||
ctx.on('agent/turn-end', () => { if (!threw) { threw = true; throw new Error('boom turn-end') } })
|
||||
const errors: Error[] = []
|
||||
ctx.on('agent/error', (_a, _t, _s, error) => void errors.push(error))
|
||||
|
||||
send(agent, 'go')
|
||||
await waitForIdle(ctx, agent)
|
||||
|
||||
const c = boundaryCounts(agent)
|
||||
// exactly one error event + one agent/error emit, despite two failTurn calls.
|
||||
expect(c.errors).toBe(1)
|
||||
expect(errors.map(e => e.message)).toEqual(['provider down'])
|
||||
expect(c.turnStart).toBe(1)
|
||||
expect(c.turnEnd).toBe(1) // single turn/end, balanced
|
||||
expect(c.lastTurnEnd?.type === 'turn/end' && c.lastTurnEnd.data.reason).toMatchObject({ kind: 'error', message: 'provider down' })
|
||||
|
||||
// loop survives the compound failure.
|
||||
send(agent, 'again')
|
||||
await waitForIdle(ctx, agent)
|
||||
expect(boundaryCounts(agent).turnEnd).toBe(2)
|
||||
})
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user