feat: implement bounded LLM request recovery

packages/examples/cli-demo/README.md

@@ -18,6 +18,7 @@ The package mounts no console logger, readline UI, user-interaction service, or
 | `skills` | owner defaults | skill registry, local provider, and model-facing skill tool |
 | `toolBash` | owner defaults | model-facing bash config, including this producer's background opt-in |
 | `toolTasks` | owner defaults | generic `task_output` wait bounds |
+| `llmRetry` | owner defaults | bounded transient model-request retry policy |
 | `persistenceRoot` | `./.sessions` | JSONL session root |
 | `workspaceContext` | required | workspace-instruction byte budget, or `false` to disable loading |
 
@@ -40,7 +41,7 @@ Loader configs with bare package specifiers require `node --expose-internals` or
 ### Output formats
 
 - `text` writes the last assistant message containing text, followed by one newline.
-- `json` writes one DSH-native result record: `{ type: "result", success, sessionId, turn, result, reason, usage? }`. `usage` sums every model step in the task turn.
+- `json` writes one DSH-native result record: `{ type: "result", success, sessionId, turn, result, reason, usage? }`. `usage` sums each model step in the task turn once, including billed failed retry attempts that produced usage without a committed assistant message.
 - `stream-json` writes each canonical event from the top-level session's task turn as `{ type: "session_event", sessionId, event }`, then the same result record. Child-agent activity appears only through the parent tool events and results.
 
 Only `reason.kind === "completed"` exits successfully. Other durable turn endings still emit partial text or a result record, add a stderr diagnostic, and exit nonzero. Argument and boot failures leave stdout empty. SIGINT and SIGTERM cancel active work, await disposal, and exit 130 and 143 respectively.

packages/examples/cli-demo/src/cli.ts

@@ -219,7 +219,7 @@ export async function runOneShot(ctx: Context, options: OneShotOptions): Promise
   let targetTurn: number | undefined
   let reason: TurnEndReason | undefined
   let result = ''
-  let usage: TokenUsage | undefined
+  const usageByStep = new Map<number, TokenUsage>()
   let outputError: Error | undefined
   let resolveTurn!: () => void
   let rejectTurn!: (error: Error) => void
@@ -254,9 +254,14 @@ export async function runOneShot(ctx: Context, options: OneShotOptions): Promise
       targetTurn = event.data.turn
     }
     observe(session.id, event)
+    if (event.type === 'assistant/chunk'
+      && event.data.turn === targetTurn
+      && event.data.chunk.type === 'usage') {
+      usageByStep.set(event.data.step, event.data.chunk.usage)
+    }
     if (event.type === 'assistant/message' && event.data.turn === targetTurn) {
       result = assistantText(event) ?? result
-      if (event.data.usage !== undefined) usage = addUsage(usage, event.data.usage)
+      if (event.data.usage !== undefined) usageByStep.set(event.data.step, event.data.usage)
     }
     if (event.type === 'turn/end' && event.data.turn === targetTurn) {
       reason = event.data.reason
@@ -294,6 +299,7 @@ export async function runOneShot(ctx: Context, options: OneShotOptions): Promise
   }
   await ctx.sessions.flush(agent.session)
   if (outputError !== undefined) throw outputError
+  const usage = [...usageByStep.values()].reduce<TokenUsage | undefined>(addUsage, undefined)
   return {
     type: 'result',
     success: reason.kind === 'completed',
@@ -365,7 +371,7 @@ export function formatTurnFailure(reason: TurnEndReason): string {
   switch (reason.kind) {
     case 'completed': return 'completed'
     case 'aborted': return reason.reason === undefined ? 'was aborted' : `was aborted: ${reason.reason}`
-    case 'error': return `failed at step ${reason.step}: ${reason.message}`
+    case 'error': return `failed at step ${reason.step}: ${'failure' in reason ? reason.failure.message : reason.message}`
     case 'disposed': return 'was disposed'
     case 'max-tokens': return 'reached the model output-token limit'
     case 'rejected': return `was rejected: ${reason.reason}`

packages/examples/cli-demo/src/index.ts

@@ -42,6 +42,8 @@ export interface Config {
   toolBash?: NonNullable<agentCore.Config['toolBash']>
   /** Generic background-task control-tool config forwarded through agent-spine-demo. */
   toolTasks?: NonNullable<agentCore.Config['toolTasks']>
+  /** Bounded transient model-request retry policy forwarded through agent-spine-demo. */
+  llmRetry?: NonNullable<agentCore.Config['llmRetry']>
   /** Controls automatic AGENTS.md/CLAUDE.md loading; configure a byte budget or set `false`. */
   workspaceContext: agentCore.Config['workspaceContext']
 }
@@ -62,6 +64,7 @@ export const Config: z<Config> = z.object({
   tools: ToolRegistry.Config,
   toolBash: agentCore.ToolBashConfigSchema,
   toolTasks: z.union([z.const(false), agentCore.ToolTasksConfigSchema]),
+  llmRetry: agentCore.LlmRetryConfigSchema,
   workspaceContext: z.union([z.const(false), workspaceContext.Config]).required(),
 })
 /* jscpd:ignore-end */

packages/examples/cli-demo/tests/cli.spec.ts

@@ -70,6 +70,15 @@ function toolResponse(usage: TokenUsage): StreamChunk[] {
   ]
 }
 
+function failedResponse(usage: TokenUsage): StreamChunk[] {
+  return [
+    { type: 'block-start', index: 0, blockType: 'text' },
+    { type: 'text-delta', index: 0, text: 'discarded' },
+    { type: 'usage', usage },
+    { type: 'finish', reason: { kind: 'error', failure: { message: 'temporary', code: 'SERVER' } } },
+  ]
+}
+
 function reasoningResponse(text: string): StreamChunk[] {
   return [
     { type: 'block-start', index: 0, blockType: 'reasoning' },
@@ -98,6 +107,7 @@ async function harness(script: readonly ScriptEntry[]): Promise<Harness> {
     persistenceRoot: root,
     skills: { local: { dshHome: join(skillHome, '.dsh'), agentsHome: join(skillHome, '.agents') } },
     workspaceContext: false,
+    llmRetry: { initialDelayMs: 1, maxDelayMs: 1, jitterRatio: 0 },
   })
   await new Promise(resolve => setTimeout(resolve, 80))
   ctx.llm.registerAdapter(['mock'], new ScriptedAdapter(script))
@@ -323,6 +333,21 @@ describe('runOneShot and executeCli', () => {
     })
   })
 
+  it('counts a failed retry attempt once even though it has no assistant message', async () => {
+    const failed = { inputTokens: 11, outputTokens: 2, cacheReadTokens: 3 }
+    const recovered = { inputTokens: 7, outputTokens: 5, reasoningTokens: 4 }
+    const { ctx } = await harness([failedResponse(failed), textResponse('done', recovered)])
+
+    const result = await runOneShot(ctx, { task: 'task' })
+
+    expect(result.usage).toEqual({
+      inputTokens: 18,
+      outputTokens: 7,
+      cacheReadTokens: 3,
+      reasoningTokens: 4,
+    })
+  })
+
   it('keeps the prior text when a later assistant message has no text blocks', async () => {
     const { ctx } = await harness([
       toolResponse({ inputTokens: 1, outputTokens: 1 }),
@@ -463,6 +488,7 @@ describe('formatTurnFailure', () => {
       [{ kind: 'aborted' }, 'was aborted'],
       [{ kind: 'aborted', reason: 'stop' }, 'was aborted: stop'],
       [{ kind: 'error', step: 2, message: 'bad' }, 'failed at step 2: bad'],
+      [{ kind: 'error', step: 3, failure: { message: 'provider bad', code: 'SERVER' } }, 'failed at step 3: provider bad'],
       [{ kind: 'disposed' }, 'was disposed'],
       [{ kind: 'max-tokens' }, 'output-token limit'],
       [{ kind: 'rejected', reason: 'policy' }, 'was rejected: policy'],