fix(agent): preserve lifecycle recovery boundaries (PR3 round 2)
This commit is contained in:
@@ -45,6 +45,7 @@ export function registerAutomaticCompaction(
|
|||||||
_step: number,
|
_step: number,
|
||||||
signal: AbortSignal,
|
signal: AbortSignal,
|
||||||
) => {
|
) => {
|
||||||
|
if (signal.aborted) return
|
||||||
try {
|
try {
|
||||||
const result = await service.compactIfNeeded(agent, 'pressure', signal)
|
const result = await service.compactIfNeeded(agent, 'pressure', signal)
|
||||||
if (result !== null) logResult(result, 'post-step pressure')
|
if (result !== null) logResult(result, 'post-step pressure')
|
||||||
|
|||||||
@@ -848,6 +848,21 @@ describe('automatic listener and loader composition', () => {
|
|||||||
expect(compact.calls).toHaveLength(1)
|
expect(compact.calls).toHaveLength(1)
|
||||||
})
|
})
|
||||||
|
|
||||||
|
it('skips post-step pressure when the step signal is already aborted', async () => {
|
||||||
|
const ctx = createContext()
|
||||||
|
const compact = new TestCompactService(ctx, {
|
||||||
|
models: { [MODEL]: { thresholdRatio: 0.5, retainTokens: 18 } },
|
||||||
|
})
|
||||||
|
const pressured = conversation(4)
|
||||||
|
const compactIfNeeded = vi.spyOn(compact, 'compactIfNeeded')
|
||||||
|
|
||||||
|
await expect(postStep(ctx, agent(pressured, MODEL), AbortSignal.abort('step aborted')))
|
||||||
|
.resolves.toBeUndefined()
|
||||||
|
|
||||||
|
expect(compactIfNeeded).not.toHaveBeenCalled()
|
||||||
|
expect(pressured.events.some(event => event.type === 'compact/start')).toBe(false)
|
||||||
|
})
|
||||||
|
|
||||||
it('warns and continues after operational failures, including non-Errors', async () => {
|
it('warns and continues after operational failures, including non-Errors', async () => {
|
||||||
const ctx = createContext()
|
const ctx = createContext()
|
||||||
const warnings: string[] = []
|
const warnings: string[] = []
|
||||||
|
|||||||
@@ -650,7 +650,8 @@ async function runStep(
|
|||||||
session, turn, step, message.content, assembler.usage, chunkSeqs,
|
session, turn, step, message.content, assembler.usage, chunkSeqs,
|
||||||
)
|
)
|
||||||
|
|
||||||
// Tool execution stays sequential; recheck abort around each normalized result.
|
// Tool execution stays sequential; cancellation latches synthetic results for
|
||||||
|
// every remaining call while preserving one complete result batch.
|
||||||
const toolCalls = message.content.filter(block => block.type === 'tool-call')
|
const toolCalls = message.content.filter(block => block.type === 'tool-call')
|
||||||
// Buffer context until all results are appended to preserve call/result adjacency.
|
// Buffer context until all results are appended to preserve call/result adjacency.
|
||||||
const pendingContext: HookContext[] = []
|
const pendingContext: HookContext[] = []
|
||||||
@@ -693,9 +694,6 @@ async function runStep(
|
|||||||
if (signal.aborted) aborted = true
|
if (signal.aborted) aborted = true
|
||||||
}
|
}
|
||||||
|
|
||||||
/* v8 ignore next -- signal.reason always set by cancellation or disposal. */
|
|
||||||
if (aborted) throw new Error(String(signal.reason ?? 'aborted'))
|
|
||||||
|
|
||||||
// Append buffered context after the complete result batch.
|
// Append buffered context after the complete result batch.
|
||||||
for (const context of pendingContext) {
|
for (const context of pendingContext) {
|
||||||
agent.inject(context.content, { source: context.source })
|
agent.inject(context.content, { source: context.source })
|
||||||
|
|||||||
@@ -156,7 +156,7 @@ describe('successful provider completion survives agent/step-result failure', ()
|
|||||||
})
|
})
|
||||||
|
|
||||||
describe('abort during tool execution ends the turn', () => {
|
describe('abort during tool execution ends the turn', () => {
|
||||||
it('aborting the in-flight step inside a tool prevents both remaining tools and the next model step', async () => {
|
it('balances an aborted tool batch through context, steering, and post-step before closing', async () => {
|
||||||
const adapter = new MockAdapter([
|
const adapter = new MockAdapter([
|
||||||
// model asks for two tool calls in one step
|
// model asks for two tool calls in one step
|
||||||
[
|
[
|
||||||
@@ -175,8 +175,12 @@ describe('abort during tool execution ends the turn', () => {
|
|||||||
name: 'aborter',
|
name: 'aborter',
|
||||||
description: '',
|
description: '',
|
||||||
parameters: {},
|
parameters: {},
|
||||||
async execute() {
|
async execute(_args, exec) {
|
||||||
executed.push('aborter')
|
executed.push('aborter')
|
||||||
|
exec.agent?.steer(
|
||||||
|
[{ type: 'text', text: 'steering before abort' }],
|
||||||
|
{ source: { kind: 'plugin', plugin: 'abort-test' } },
|
||||||
|
)
|
||||||
// Fire the in-flight step's AbortController directly (the loop registers
|
// Fire the in-flight step's AbortController directly (the loop registers
|
||||||
// it on the agent). This is the bare step-abort path — distinct from
|
// it on the agent). This is the bare step-abort path — distinct from
|
||||||
// cancel(), which would also clear the inbox; here the subject is the
|
// cancel(), which would also clear the inbox; here the subject is the
|
||||||
@@ -185,6 +189,13 @@ describe('abort during tool execution ends the turn', () => {
|
|||||||
return [{ type: 'text', text: 'done' }]
|
return [{ type: 'text', text: 'done' }]
|
||||||
},
|
},
|
||||||
}))
|
}))
|
||||||
|
ctx.on('tools/post-execute', async exec => ({
|
||||||
|
kind: 'accept',
|
||||||
|
additionalContext: {
|
||||||
|
content: [{ type: 'text', text: `context for ${exec.callId}` }],
|
||||||
|
source: { kind: 'plugin', plugin: 'abort-test' },
|
||||||
|
},
|
||||||
|
}))
|
||||||
ctx.tools.register(defineTool({
|
ctx.tools.register(defineTool({
|
||||||
name: 'second',
|
name: 'second',
|
||||||
description: '',
|
description: '',
|
||||||
@@ -196,13 +207,53 @@ describe('abort during tool execution ends the turn', () => {
|
|||||||
}))
|
}))
|
||||||
|
|
||||||
const reasons: TurnEndReason[] = []
|
const reasons: TurnEndReason[] = []
|
||||||
ctx.on('session/event', (_s, event) => { if (event.type === 'turn/end') reasons.push(event.data.reason) })
|
const order: string[] = []
|
||||||
|
ctx.on('session/event', (session, event) => {
|
||||||
|
if (session !== agent.session) return
|
||||||
|
switch (event.type) {
|
||||||
|
case 'assistant/message': order.push('assistant/message'); break
|
||||||
|
case 'tool/call': order.push(`tool/call:${event.data.callId}`); break
|
||||||
|
case 'tool/result': {
|
||||||
|
const outcome = event.data.error?.code === 'ABORTED' ? 'synthetic-aborted' : 'real'
|
||||||
|
order.push(`tool/result:${event.data.callId}:${outcome}`)
|
||||||
|
break
|
||||||
|
}
|
||||||
|
case 'context/message': order.push('context/message'); break
|
||||||
|
case 'steering/message': order.push('steering/message'); break
|
||||||
|
case 'step/end': order.push('step/end'); break
|
||||||
|
case 'turn/end': {
|
||||||
|
reasons.push(event.data.reason)
|
||||||
|
order.push(`turn/end:${event.data.reason.kind}`)
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
let postSteps = 0
|
||||||
|
ctx.on('agent/post-step', (subject, turn, step, signal) => {
|
||||||
|
if (subject !== agent) return
|
||||||
|
postSteps += 1
|
||||||
|
expect({ turn, step, aborted: signal.aborted }).toEqual({ turn: 1, step: 1, aborted: true })
|
||||||
|
order.push('agent/post-step')
|
||||||
|
})
|
||||||
|
|
||||||
send(agent, 'go')
|
send(agent, 'go')
|
||||||
await waitForIdle(ctx, agent)
|
await waitForIdle(ctx, agent)
|
||||||
|
|
||||||
expect(executed).toEqual(['aborter']) // second tool never ran
|
expect(executed).toEqual(['aborter']) // second tool never ran
|
||||||
expect(adapter.requests).toHaveLength(1) // no follow-up model call
|
expect(adapter.requests).toHaveLength(1) // no follow-up model call
|
||||||
|
expect(postSteps).toBe(1)
|
||||||
|
expect(order).toEqual([
|
||||||
|
'assistant/message',
|
||||||
|
'tool/call:c1',
|
||||||
|
'tool/result:c1:real',
|
||||||
|
'tool/call:c2',
|
||||||
|
'tool/result:c2:synthetic-aborted',
|
||||||
|
'context/message',
|
||||||
|
'steering/message',
|
||||||
|
'agent/post-step',
|
||||||
|
'step/end',
|
||||||
|
'turn/end:aborted',
|
||||||
|
])
|
||||||
expect(reasons).toEqual([{ kind: 'aborted', reason: 'user interrupt' }])
|
expect(reasons).toEqual([{ kind: 'aborted', reason: 'user interrupt' }])
|
||||||
const calls = agent.session.events.filter(event => event.type === 'tool/call')
|
const calls = agent.session.events.filter(event => event.type === 'tool/call')
|
||||||
const results = agent.session.events.filter(event => event.type === 'tool/result')
|
const results = agent.session.events.filter(event => event.type === 'tool/result')
|
||||||
|
|||||||
@@ -124,8 +124,9 @@ export class LlmService extends Service {
|
|||||||
* Final adapter boundary. It tags only failures from adapter selection,
|
* Final adapter boundary. It tags only failures from adapter selection,
|
||||||
* synchronous dispatch, iterator construction, or iteration while preserving
|
* synchronous dispatch, iterator construction, or iteration while preserving
|
||||||
* the original Error object. Middleware outside this generator remains
|
* the original Error object. Middleware outside this generator remains
|
||||||
* distinguishable as plugin work. Adapter cleanup is best-effort after an
|
* distinguishable as plugin work. An iteration failure skips adapter cleanup
|
||||||
* earlier failure or downstream close and never masks the winning error.
|
* so it cannot suppress the primary provider error. A downstream close awaits
|
||||||
|
* adapter cleanup, whose failures remain ordinary untagged work.
|
||||||
*/
|
*/
|
||||||
private async * adapterStream(options: GenerateOptions): AsyncGenerator<StreamChunk> {
|
private async * adapterStream(options: GenerateOptions): AsyncGenerator<StreamChunk> {
|
||||||
let iterator: AsyncIterator<StreamChunk>
|
let iterator: AsyncIterator<StreamChunk>
|
||||||
@@ -137,6 +138,7 @@ export class LlmService extends Service {
|
|||||||
}
|
}
|
||||||
|
|
||||||
let completed = false
|
let completed = false
|
||||||
|
let iterationFailed = false
|
||||||
try {
|
try {
|
||||||
while (true) {
|
while (true) {
|
||||||
let value: StreamChunk
|
let value: StreamChunk
|
||||||
@@ -148,6 +150,7 @@ export class LlmService extends Service {
|
|||||||
}
|
}
|
||||||
value = item.value
|
value = item.value
|
||||||
} catch (error: unknown) {
|
} catch (error: unknown) {
|
||||||
|
iterationFailed = true
|
||||||
throw markLlmAdapterFailure(error)
|
throw markLlmAdapterFailure(error)
|
||||||
}
|
}
|
||||||
// End the adapter-owned try before yielding: consumer/middleware
|
// End the adapter-owned try before yielding: consumer/middleware
|
||||||
@@ -155,14 +158,10 @@ export class LlmService extends Service {
|
|||||||
yield value
|
yield value
|
||||||
}
|
}
|
||||||
} finally {
|
} finally {
|
||||||
if (!completed) {
|
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition -- the iteration catch sets its latch before entering finally.
|
||||||
try {
|
if (!completed && !iterationFailed) {
|
||||||
const close = iterator.return?.bind(iterator)
|
const close = iterator.return?.bind(iterator)
|
||||||
if (close) await close()
|
if (close) await close()
|
||||||
} catch {
|
|
||||||
// Lookup and invocation are both adapter-owned cleanup following an
|
|
||||||
// existing failure/downstream close; neither can replace it.
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -72,11 +72,21 @@ describe('LlmService', () => {
|
|||||||
const original = new LlmError(`${field} getter failed`, 'RESULT_GETTER_FAILED')
|
const original = new LlmError(`${field} getter failed`, 'RESULT_GETTER_FAILED')
|
||||||
const result = field === 'done' ? {} : { done: false }
|
const result = field === 'done' ? {} : { done: false }
|
||||||
Object.defineProperty(result, field, { get: () => { throw original } })
|
Object.defineProperty(result, field, { get: () => { throw original } })
|
||||||
|
let cleanupLookups = 0
|
||||||
|
const iterator: AsyncIterator<StreamChunk> = {
|
||||||
|
next: () => Promise.resolve(result as unknown as IteratorResult<StreamChunk>),
|
||||||
|
}
|
||||||
|
Object.defineProperty(iterator, 'return', {
|
||||||
|
get: () => {
|
||||||
|
cleanupLookups += 1
|
||||||
|
throw new Error('return getter must not run after iteration fails')
|
||||||
|
},
|
||||||
|
})
|
||||||
const adapter = new class extends LlmAdapter {
|
const adapter = new class extends LlmAdapter {
|
||||||
stream(_options: GenerateOptions): AsyncIterable<StreamChunk> {
|
stream(_options: GenerateOptions): AsyncIterable<StreamChunk> {
|
||||||
return {
|
return {
|
||||||
[Symbol.asyncIterator](): AsyncIterator<StreamChunk> {
|
[Symbol.asyncIterator](): AsyncIterator<StreamChunk> {
|
||||||
return { next: () => Promise.resolve(result as unknown as IteratorResult<StreamChunk>) }
|
return iterator
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -94,6 +104,7 @@ describe('LlmService', () => {
|
|||||||
|
|
||||||
expect(caught).toBe(original)
|
expect(caught).toBe(original)
|
||||||
expect(isLlmAdapterFailure(caught)).toBe(true)
|
expect(isLlmAdapterFailure(caught)).toBe(true)
|
||||||
|
expect(cleanupLookups).toBe(0)
|
||||||
})
|
})
|
||||||
|
|
||||||
it.each(['dispatch', 'iterator'] as const)('tags synchronous adapter %s failures without replacing their Error', async (boundary) => {
|
it.each(['dispatch', 'iterator'] as const)('tags synchronous adapter %s failures without replacing their Error', async (boundary) => {
|
||||||
@@ -119,7 +130,7 @@ describe('LlmService', () => {
|
|||||||
expect(isLlmAdapterFailure(caught)).toBe(true)
|
expect(isLlmAdapterFailure(caught)).toBe(true)
|
||||||
})
|
})
|
||||||
|
|
||||||
it('tags adapter iteration failures without replacing the original Error or cleanup outcome', async () => {
|
it('propagates a rejected next promptly without awaiting a non-settling return', async () => {
|
||||||
const original = new LlmError('provider failed', 'PROVIDER_FAILED')
|
const original = new LlmError('provider failed', 'PROVIDER_FAILED')
|
||||||
let cleanupCalls = 0
|
let cleanupCalls = 0
|
||||||
const adapter = new class extends LlmAdapter {
|
const adapter = new class extends LlmAdapter {
|
||||||
@@ -130,7 +141,49 @@ describe('LlmService', () => {
|
|||||||
next: () => Promise.reject(original),
|
next: () => Promise.reject(original),
|
||||||
return: () => {
|
return: () => {
|
||||||
cleanupCalls += 1
|
cleanupCalls += 1
|
||||||
return Promise.reject(new Error('cleanup failed'))
|
return new Promise<IteratorResult<StreamChunk>>(() => {})
|
||||||
|
},
|
||||||
|
}
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
const ctx = new Context()
|
||||||
|
await ctx.plugin(LlmService)
|
||||||
|
ctx.llm.registerAdapter(['test-model'], adapter)
|
||||||
|
|
||||||
|
const failure = (async (): Promise<unknown> => {
|
||||||
|
try {
|
||||||
|
for await (const _chunk of ctx.llm.stream({ model: 'test-model', messages: [] })) { /* drain */ }
|
||||||
|
} catch (error: unknown) {
|
||||||
|
return error
|
||||||
|
}
|
||||||
|
return new Error('expected adapter iteration to fail')
|
||||||
|
})()
|
||||||
|
let timer: ReturnType<typeof setTimeout> | undefined
|
||||||
|
const timeout = new Promise<Error>((resolve) => {
|
||||||
|
timer = setTimeout(() => { resolve(new Error('adapter failure did not settle promptly')) }, 100)
|
||||||
|
})
|
||||||
|
const caught = await Promise.race([failure, timeout])
|
||||||
|
if (timer !== undefined) clearTimeout(timer)
|
||||||
|
|
||||||
|
expect(caught).toBe(original)
|
||||||
|
expect(isLlmAdapterFailure(caught)).toBe(true)
|
||||||
|
expect(cleanupCalls).toBe(0)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('awaits one adapter return on downstream close and leaves its rejection unclassified', async () => {
|
||||||
|
const cleanup = new Error('cleanup failed')
|
||||||
|
let cleanupCalls = 0
|
||||||
|
const adapter = new class extends LlmAdapter {
|
||||||
|
stream(_options: GenerateOptions): AsyncIterable<StreamChunk> {
|
||||||
|
return {
|
||||||
|
[Symbol.asyncIterator](): AsyncIterator<StreamChunk> {
|
||||||
|
return {
|
||||||
|
next: () => Promise.resolve({ done: false, value: SCRIPT[0]! }),
|
||||||
|
return: () => {
|
||||||
|
cleanupCalls += 1
|
||||||
|
return Promise.reject(cleanup)
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
@@ -143,45 +196,37 @@ describe('LlmService', () => {
|
|||||||
|
|
||||||
let caught: unknown
|
let caught: unknown
|
||||||
try {
|
try {
|
||||||
for await (const _chunk of ctx.llm.stream({ model: 'test-model', messages: [] })) { /* drain */ }
|
for await (const _chunk of ctx.llm.stream({ model: 'test-model', messages: [] })) break
|
||||||
} catch (error: unknown) {
|
} catch (error: unknown) {
|
||||||
caught = error
|
caught = error
|
||||||
}
|
}
|
||||||
|
|
||||||
expect(caught).toBe(original)
|
expect(caught).toBe(cleanup)
|
||||||
expect(isLlmAdapterFailure(caught)).toBe(true)
|
expect(isLlmAdapterFailure(caught)).toBe(false)
|
||||||
expect(cleanupCalls).toBe(1)
|
expect(cleanupCalls).toBe(1)
|
||||||
})
|
})
|
||||||
|
|
||||||
it('contains a throwing iterator.return getter after next fails without replacing the original Error', async () => {
|
it('allows downstream close when the adapter iterator has no return method', async () => {
|
||||||
const original = new LlmError('provider failed', 'PROVIDER_FAILED')
|
|
||||||
let cleanupLookups = 0
|
|
||||||
const iterator: AsyncIterator<StreamChunk> = { next: () => Promise.reject(original) }
|
|
||||||
Object.defineProperty(iterator, 'return', {
|
|
||||||
get: () => {
|
|
||||||
cleanupLookups += 1
|
|
||||||
throw new Error('return getter failed')
|
|
||||||
},
|
|
||||||
})
|
|
||||||
const adapter = new class extends LlmAdapter {
|
const adapter = new class extends LlmAdapter {
|
||||||
stream(_options: GenerateOptions): AsyncIterable<StreamChunk> {
|
stream(_options: GenerateOptions): AsyncIterable<StreamChunk> {
|
||||||
return { [Symbol.asyncIterator]: () => iterator }
|
return {
|
||||||
|
[Symbol.asyncIterator](): AsyncIterator<StreamChunk> {
|
||||||
|
return { next: () => Promise.resolve({ done: false, value: SCRIPT[0]! }) }
|
||||||
|
},
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
const ctx = new Context()
|
const ctx = new Context()
|
||||||
await ctx.plugin(LlmService)
|
await ctx.plugin(LlmService)
|
||||||
ctx.llm.registerAdapter(['test-model'], adapter)
|
ctx.llm.registerAdapter(['test-model'], adapter)
|
||||||
|
|
||||||
let caught: unknown
|
let chunks = 0
|
||||||
try {
|
for await (const _chunk of ctx.llm.stream({ model: 'test-model', messages: [] })) {
|
||||||
for await (const _chunk of ctx.llm.stream({ model: 'test-model', messages: [] })) { /* drain */ }
|
chunks += 1
|
||||||
} catch (error: unknown) {
|
break
|
||||||
caught = error
|
|
||||||
}
|
}
|
||||||
|
|
||||||
expect(caught).toBe(original)
|
expect(chunks).toBe(1)
|
||||||
expect(isLlmAdapterFailure(caught)).toBe(true)
|
|
||||||
expect(cleanupLookups).toBe(1)
|
|
||||||
})
|
})
|
||||||
|
|
||||||
it('normalizes and tags non-Error adapter failures once', async () => {
|
it('normalizes and tags non-Error adapter failures once', async () => {
|
||||||
|
|||||||
Reference in New Issue
Block a user