subagent: implement structured output for in-process backends
The seam vocabulary (SubagentStartRequest.outputSchema, SubagentResult .structured) existed but no in-process backend honored it — spawn/fork advertised outputSchema: false. This lands the missing half: - dsh-tools gains a structured-output JSON Schema subset (json-schema.ts): StructuredOutputSchema, assertSupportedOutputSchema (rejects loud outside the enforced subset, every violation listed), validateStructuredValue (path-qualified issues, total). outputSchema's seam type becomes this raw JSON-Schema subset instead of the author-facing SchemaSpec DSL — the schema travels verbatim to the model as a forced tool's parameters. - dsh-subagent-inprocess gains the shared structured runtime: one global structured_output capture tool (placeholder parameters) + a prepend:true agent/request listener doing FINAL-REQUEST enforcement (strip for plain agents, per-run schema for structured children — survives downstream request-replacing listeners) + an agent/turn-continuation veto that stops a child's turn once captured (no wasted extra model step). Lifetime is refcounted by backends (plugin lifetime) AND live runs (start→settle). - startInProcessRun drives the capture: subset asserted before the child exists, instruction appended to the child's system prompt, clean-finish nudge loop (structuredNudgeRetries, backend Config, default 1), captured value on result.structured; a clean finish without a capture settles 'error' (never a silent success with a missing field). - spawn + fork flip outputSchema: true and inject 'tools'.
This commit is contained in:
Tianyi Cui
393
packages/subagent/subagent-inprocess/tests/structured.spec.ts
Normal file
393
packages/subagent/subagent-inprocess/tests/structured.spec.ts
Normal file
@@ -0,0 +1,393 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { Context } from 'cordis'
|
||||
import LlmService, { type GenerateOptions } from '@deepseek-ai/dsh-llm'
|
||||
import SessionStore from '@deepseek-ai/dsh-session'
|
||||
import SystemPrompt from '@deepseek-ai/dsh-system-prompt'
|
||||
import ToolRegistry from '@deepseek-ai/dsh-tools'
|
||||
import AgentRegistry, { AgentId } from '@deepseek-ai/dsh-agent'
|
||||
import AgentLoop from '@deepseek-ai/dsh-agent-loop'
|
||||
import * as Invariants from '@deepseek-ai/dsh-invariants'
|
||||
import SubagentService, { type SubagentStartRequest } from '@deepseek-ai/dsh-subagent'
|
||||
import type { StructuredOutputSchema } from '@deepseek-ai/dsh-tools'
|
||||
import { MockAdapter, textResponse, toolCallResponse } from '../../../core/agent-loop/tests/mock-adapter.ts'
|
||||
import * as spawn from '../../subagent-spawn/src/index.ts'
|
||||
import * as fork from '../../subagent-fork/src/index.ts'
|
||||
import {
|
||||
acquireStructuredRuntime,
|
||||
STRUCTURED_OUTPUT_INSTRUCTION,
|
||||
STRUCTURED_OUTPUT_TOOL,
|
||||
} from '../src/structured.ts'
|
||||
|
||||
type Script = ConstructorParameters<typeof MockAdapter>[0]
|
||||
|
||||
const SCHEMA: StructuredOutputSchema = {
|
||||
type: 'object',
|
||||
properties: { answer: { type: 'number' }, note: { type: 'string' } },
|
||||
required: ['answer'],
|
||||
}
|
||||
|
||||
/**
|
||||
* Real loop + scripted mock model + the REAL spawn backend (which acquires the
|
||||
* structured runtime at apply, exactly as shipped). The mock model script
|
||||
* drives the child's structured_output calls.
|
||||
*/
|
||||
async function setup(script: Script, options?: { nudges?: number; withFork?: boolean }) {
|
||||
const ctx = new Context()
|
||||
const adapter = new MockAdapter(script)
|
||||
await ctx.plugin(LlmService)
|
||||
await ctx.plugin(SessionStore)
|
||||
await ctx.plugin(SystemPrompt)
|
||||
await ctx.plugin(ToolRegistry)
|
||||
await ctx.plugin(AgentRegistry)
|
||||
await ctx.plugin(Invariants)
|
||||
await ctx.plugin(AgentLoop, { agents: [] })
|
||||
await ctx.plugin(SubagentService)
|
||||
const fiber = await ctx.plugin(spawn, { providerName: 'spawn', structuredNudgeRetries: options?.nudges ?? 1 })
|
||||
const forkFiber = options?.withFork
|
||||
? await ctx.plugin(fork, { providerName: 'fork', structuredNudgeRetries: options?.nudges ?? 1 })
|
||||
: undefined
|
||||
ctx.llm.registerAdapter(['mock'], adapter)
|
||||
const parent = ctx.agentLoop.create(AgentId('parent'), { model: 'mock' })
|
||||
return { ctx, parent, adapter, fiber, forkFiber }
|
||||
}
|
||||
|
||||
function structuredRequest(parent: SubagentStartRequest['parent'], extra?: Partial<SubagentStartRequest>): SubagentStartRequest {
|
||||
return { prompt: [{ type: 'text', text: 'produce the answer' }], parent, outputSchema: SCHEMA, ...extra }
|
||||
}
|
||||
|
||||
/** The tool names of one recorded model request. */
|
||||
function toolNames(request: GenerateOptions): string[] {
|
||||
return (request.tools ?? []).map(tool => tool.name)
|
||||
}
|
||||
|
||||
describe('in-process structured output', () => {
|
||||
it('captures a valid structured_output call and surfaces result.structured', async () => {
|
||||
const { ctx, parent } = await setup([
|
||||
toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 42, note: 'done' }),
|
||||
])
|
||||
const run = ctx.subagents.start('spawn', structuredRequest(parent))
|
||||
const result = await run.result
|
||||
expect(result.stopReason).toBe('completed')
|
||||
expect(result.structured).toEqual({ answer: 42, note: 'done' })
|
||||
await run.dispose()
|
||||
})
|
||||
|
||||
it('stops the turn after a successful capture — no extra model step is spent', async () => {
|
||||
const { ctx, parent, adapter } = await setup([
|
||||
toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 1 }),
|
||||
textResponse('MUST NOT BE CONSUMED'),
|
||||
])
|
||||
const run = ctx.subagents.start('spawn', structuredRequest(parent))
|
||||
await run.result
|
||||
// Default continuation would run a second step after the tool call; the
|
||||
// structured runtime's turn-continuation veto stops the turn instead.
|
||||
expect(adapter.requests.length).toBe(1)
|
||||
await run.dispose()
|
||||
})
|
||||
|
||||
it('an invalid call gets an INVALID_ARGS isError result and the model retries in-turn', async () => {
|
||||
const { ctx, parent } = await setup([
|
||||
toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 'not-a-number' }),
|
||||
toolCallResponse('c2', STRUCTURED_OUTPUT_TOOL, { answer: 7 }),
|
||||
])
|
||||
const run = ctx.subagents.start('spawn', structuredRequest(parent))
|
||||
const result = await run.result
|
||||
expect(result.structured).toEqual({ answer: 7 })
|
||||
expect(result.stopReason).toBe('completed')
|
||||
// The child's log carries the isError tool/result for the invalid call.
|
||||
const child = ctx.agents.get(run.id)!
|
||||
const results = child.session.events.filter(e => e.type === 'tool/result')
|
||||
expect(results.length).toBe(2)
|
||||
expect((results[0]!.data as { isError?: boolean }).isError).toBe(true)
|
||||
await run.dispose()
|
||||
})
|
||||
|
||||
it('nudges a child that finished cleanly without calling the tool, then captures', async () => {
|
||||
const { ctx, parent } = await setup([
|
||||
textResponse('here is my answer in prose'),
|
||||
toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 3 }),
|
||||
])
|
||||
const run = ctx.subagents.start('spawn', structuredRequest(parent))
|
||||
const result = await run.result
|
||||
expect(result.structured).toEqual({ answer: 3 })
|
||||
expect(result.stopReason).toBe('completed')
|
||||
// The nudge is a real user-visible message in the child's log.
|
||||
const child = ctx.agents.get(run.id)!
|
||||
const users = child.session.events.filter(e => e.type === 'user/message')
|
||||
expect(users.length).toBe(2)
|
||||
await run.dispose()
|
||||
})
|
||||
|
||||
it('settles error when the nudges run out without a capture', async () => {
|
||||
const { ctx, parent, adapter } = await setup([
|
||||
textResponse('prose only'),
|
||||
textResponse('still prose'),
|
||||
], { nudges: 1 })
|
||||
const run = ctx.subagents.start('spawn', structuredRequest(parent))
|
||||
const result = await run.result
|
||||
expect(result.stopReason).toBe('error')
|
||||
expect(result.structured).toBeUndefined()
|
||||
expect(adapter.requests.length).toBe(2)
|
||||
await run.dispose()
|
||||
})
|
||||
|
||||
it('zero nudge retries fails immediately after the first clean prose finish', async () => {
|
||||
const { ctx, parent, adapter } = await setup([textResponse('prose')], { nudges: 0 })
|
||||
const run = ctx.subagents.start('spawn', structuredRequest(parent))
|
||||
const result = await run.result
|
||||
expect(result.stopReason).toBe('error')
|
||||
expect(adapter.requests.length).toBe(1)
|
||||
await run.dispose()
|
||||
})
|
||||
|
||||
it('a child that errored is NOT nudged (its failure is the honest result)', async () => {
|
||||
// Script exhaustion on the first call → the child turn errors.
|
||||
const { ctx, parent, adapter } = await setup([], { nudges: 3 })
|
||||
const run = ctx.subagents.start('spawn', structuredRequest(parent))
|
||||
const result = await run.result
|
||||
expect(result.stopReason).toBe('error')
|
||||
expect(adapter.requests.length).toBe(1)
|
||||
await run.dispose()
|
||||
})
|
||||
|
||||
it('rejects a schema outside the subset loud, before any child exists', async () => {
|
||||
const { ctx, parent } = await setup([])
|
||||
expect(() => ctx.subagents.start('spawn', structuredRequest(parent, {
|
||||
outputSchema: { type: 'object', oneOf: [] } as unknown as StructuredOutputSchema,
|
||||
}))).toThrow(/unsupported output schema/)
|
||||
expect(ctx.agents.get(AgentId('parent'))).toBeDefined()
|
||||
})
|
||||
|
||||
it('appends the structured instruction to the child system prompt (caller prompt preserved)', async () => {
|
||||
const { ctx, parent } = await setup([toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 1 })])
|
||||
const run = ctx.subagents.start('spawn', structuredRequest(parent, {
|
||||
agentOptions: { systemPrompt: 'You are a counter.' },
|
||||
}))
|
||||
await run.result
|
||||
const child = ctx.agents.get(run.id)!
|
||||
expect(child.options.systemPrompt).toBe(`You are a counter.\n\n${STRUCTURED_OUTPUT_INSTRUCTION}`)
|
||||
await run.dispose()
|
||||
})
|
||||
|
||||
it('a structured child WITHOUT a caller prompt gets exactly the instruction', async () => {
|
||||
const { ctx, parent } = await setup([toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 1 })])
|
||||
const run = ctx.subagents.start('spawn', structuredRequest(parent))
|
||||
await run.result
|
||||
const child = ctx.agents.get(run.id)!
|
||||
expect(child.options.systemPrompt).toBe(STRUCTURED_OUTPUT_INSTRUCTION)
|
||||
await run.dispose()
|
||||
})
|
||||
|
||||
describe('final-request enforcement (the prepend agent/request listener)', () => {
|
||||
it('a structured child sees structured_output with ITS schema; a plain agent never sees the tool', async () => {
|
||||
const { ctx, parent, adapter } = await setup([
|
||||
// Parent turn (a plain agent): must NOT see the tool.
|
||||
textResponse('parent answer'),
|
||||
// Child turn: must see it, with the run's schema.
|
||||
toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 42 }),
|
||||
])
|
||||
parent.send([{ type: 'text', text: 'hello' }])
|
||||
await parent.whenIdle()
|
||||
expect(toolNames(adapter.requests[0]!)).not.toContain(STRUCTURED_OUTPUT_TOOL)
|
||||
|
||||
const run = ctx.subagents.start('spawn', structuredRequest(parent))
|
||||
await run.result
|
||||
const childRequest = adapter.requests[1]!
|
||||
expect(toolNames(childRequest)).toContain(STRUCTURED_OUTPUT_TOOL)
|
||||
const entry = childRequest.tools!.find(tool => tool.name === STRUCTURED_OUTPUT_TOOL)!
|
||||
expect(entry.parameters).toEqual(SCHEMA)
|
||||
await run.dispose()
|
||||
})
|
||||
|
||||
it('two concurrent structured children each see their OWN schema', async () => {
|
||||
const otherSchema: StructuredOutputSchema = {
|
||||
type: 'object',
|
||||
properties: { verdict: { type: 'string', enum: ['real', 'bogus'] } },
|
||||
required: ['verdict'],
|
||||
}
|
||||
const { ctx, parent, adapter } = await setup([
|
||||
(options: GenerateOptions) => {
|
||||
// Answer with whatever schema this child was given — proves each
|
||||
// request carried the right one regardless of scheduling order.
|
||||
const entry = options.tools!.find(tool => tool.name === STRUCTURED_OUTPUT_TOOL)!
|
||||
const args = 'verdict' in (entry.parameters.properties as Record<string, unknown>)
|
||||
? { verdict: 'real' }
|
||||
: { answer: 1 }
|
||||
return toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, args)
|
||||
},
|
||||
(options: GenerateOptions) => {
|
||||
const entry = options.tools!.find(tool => tool.name === STRUCTURED_OUTPUT_TOOL)!
|
||||
const args = 'verdict' in (entry.parameters.properties as Record<string, unknown>)
|
||||
? { verdict: 'real' }
|
||||
: { answer: 1 }
|
||||
return toolCallResponse('c2', STRUCTURED_OUTPUT_TOOL, args)
|
||||
},
|
||||
])
|
||||
const runA = ctx.subagents.start('spawn', structuredRequest(parent))
|
||||
const runB = ctx.subagents.start('spawn', structuredRequest(parent, { outputSchema: otherSchema }))
|
||||
const [a, b] = await Promise.all([runA.result, runB.result])
|
||||
expect(a.structured).toEqual({ answer: 1 })
|
||||
expect(b.structured).toEqual({ verdict: 'real' })
|
||||
const schemas = adapter.requests.map(request =>
|
||||
request.tools!.find(tool => tool.name === STRUCTURED_OUTPUT_TOOL)!.parameters)
|
||||
expect(schemas).toContainEqual(SCHEMA)
|
||||
expect(schemas).toContainEqual(otherSchema)
|
||||
await runA.dispose()
|
||||
await runB.dispose()
|
||||
})
|
||||
|
||||
it('wins against a downstream listener that REPLACES the request object', async () => {
|
||||
const { ctx, parent, adapter } = await setup([
|
||||
toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 5 }),
|
||||
])
|
||||
// A downstream (non-prepend) listener that returns a brand-new request —
|
||||
// the composition caveat that erases cooperative mutations. Registered
|
||||
// AFTER the runtime's prepend listener, so it runs INSIDE it.
|
||||
ctx.on('agent/request', async (_agent, _turn, _step, _options, next) => {
|
||||
const replaced = await next()
|
||||
return { ...replaced, tools: [...(replaced.tools ?? [])] }
|
||||
})
|
||||
const run = ctx.subagents.start('spawn', structuredRequest(parent))
|
||||
const result = await run.result
|
||||
expect(result.structured).toEqual({ answer: 5 })
|
||||
const entry = adapter.requests[0]!.tools!.find(tool => tool.name === STRUCTURED_OUTPUT_TOOL)
|
||||
expect(entry).toBeDefined()
|
||||
expect(entry!.parameters).toEqual(SCHEMA)
|
||||
await run.dispose()
|
||||
})
|
||||
|
||||
it('a non-structured agent request keeps tools ABSENT when it had none (no tools: [] materialized)', async () => {
|
||||
const { parent, adapter } = await setup([
|
||||
// The registry contributes the placeholder via prompt assembly, so
|
||||
// tools is an array in the raw request — but after stripping the
|
||||
// placeholder (its ONLY entry), the field must not be re-added as a
|
||||
// different shape.
|
||||
textResponse('plain'),
|
||||
])
|
||||
parent.send([{ type: 'text', text: 'q' }])
|
||||
await parent.whenIdle()
|
||||
const request = adapter.requests[0]!
|
||||
expect(toolNames(request)).not.toContain(STRUCTURED_OUTPUT_TOOL)
|
||||
await new Promise(resolve => setTimeout(resolve, 0))
|
||||
})
|
||||
|
||||
it('handles a request with NO tools field at all, for plain and structured agents alike', async () => {
|
||||
// Drive the waterfall directly with a toolless request — the enforcement
|
||||
// listener must tolerate `tools: undefined` on both branches: leave it
|
||||
// absent for a plain agent, and create the array for a structured child.
|
||||
const { ctx, parent } = await setup([])
|
||||
const bare: GenerateOptions = { model: 'mock', messages: [] }
|
||||
const plain = await ctx.waterfall('agent/request', parent, 1, 1, bare, () => Promise.resolve(bare))
|
||||
expect(plain.tools).toBeUndefined()
|
||||
|
||||
const acquisition = acquireStructuredRuntime(ctx)
|
||||
acquisition.attach(parent, SCHEMA)
|
||||
const bare2: GenerateOptions = { model: 'mock', messages: [] }
|
||||
const shaped = await ctx.waterfall('agent/request', parent, 1, 1, bare2, () => Promise.resolve(bare2))
|
||||
expect(shaped.tools!.map(tool => tool.name)).toEqual([STRUCTURED_OUTPUT_TOOL])
|
||||
acquisition.detach(parent)
|
||||
acquisition.release()
|
||||
})
|
||||
})
|
||||
|
||||
describe('runtime lifetime (refcount: backends + live runs)', () => {
|
||||
it('registers the capture tool while a backend is loaded and unregisters when the last unloads', async () => {
|
||||
const { ctx, fiber, forkFiber } = await setup([], { withFork: true })
|
||||
expect(ctx.tools.get(STRUCTURED_OUTPUT_TOOL)).toBeDefined()
|
||||
await fiber.dispose()
|
||||
// fork still holds a reference.
|
||||
expect(ctx.tools.get(STRUCTURED_OUTPUT_TOOL)).toBeDefined()
|
||||
await forkFiber!.dispose()
|
||||
expect(ctx.tools.get(STRUCTURED_OUTPUT_TOOL)).toBeUndefined()
|
||||
})
|
||||
|
||||
it('a live run-level acquisition keeps the runtime registered after EVERY backend unloads', async () => {
|
||||
// Simulates the run-holder half of the two-level lifetime: a structured
|
||||
// run acquires at start and releases at settle, so registration ordering
|
||||
// is settle-then-unregister even if all backends unload first. (A real
|
||||
// in-process child dies WITH its backend's fiber — the acquisition's
|
||||
// observable job is this ordering, which a manual holder pins directly.)
|
||||
const { ctx, fiber, forkFiber } = await setup([], { withFork: true })
|
||||
const runHolder = acquireStructuredRuntime(ctx)
|
||||
await fiber.dispose()
|
||||
await forkFiber!.dispose()
|
||||
expect(ctx.tools.get(STRUCTURED_OUTPUT_TOOL)).toBeDefined()
|
||||
runHolder.release()
|
||||
expect(ctx.tools.get(STRUCTURED_OUTPUT_TOOL)).toBeUndefined()
|
||||
})
|
||||
|
||||
it('a structured run releases its acquisition when it settles (backend unload mid-run)', async () => {
|
||||
const { ctx, parent, fiber } = await setup(['hang'])
|
||||
const run = ctx.subagents.start('spawn', structuredRequest(parent))
|
||||
// Let the child's step start streaming, then unload the backend. The
|
||||
// backend owns the child agent, so the unload tears the child down and
|
||||
// the run settles — releasing its own acquisition on the way out.
|
||||
await new Promise(resolve => setTimeout(resolve, 30))
|
||||
await fiber.dispose()
|
||||
const result = await run.result
|
||||
expect(result.stopReason).toBe('error')
|
||||
// Both holders (backend + run) released — nothing keeps the runtime now.
|
||||
expect(ctx.tools.get(STRUCTURED_OUTPUT_TOOL)).toBeUndefined()
|
||||
await run.dispose()
|
||||
})
|
||||
|
||||
it('fork children capture structured output through the same runtime', async () => {
|
||||
const { ctx, parent } = await setup([
|
||||
toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 9 }),
|
||||
], { withFork: true })
|
||||
const run = ctx.subagents.start('fork', structuredRequest(parent))
|
||||
const result = await run.result
|
||||
expect(result.structured).toEqual({ answer: 9 })
|
||||
await run.dispose()
|
||||
})
|
||||
|
||||
it('acquisition release is idempotent (double release cannot underflow the refcount)', async () => {
|
||||
const ctx = new Context()
|
||||
await ctx.plugin(SystemPrompt)
|
||||
await ctx.plugin(ToolRegistry)
|
||||
const first = acquireStructuredRuntime(ctx)
|
||||
const second = acquireStructuredRuntime(ctx)
|
||||
first.release()
|
||||
first.release()
|
||||
// The second holder still keeps the tool registered.
|
||||
expect(ctx.tools.get(STRUCTURED_OUTPUT_TOOL)).toBeDefined()
|
||||
second.release()
|
||||
expect(ctx.tools.get(STRUCTURED_OUTPUT_TOOL)).toBeUndefined()
|
||||
})
|
||||
|
||||
it('attach/captured/detach manage per-agent state through the acquisition surface', async () => {
|
||||
const { ctx, parent } = await setup([])
|
||||
const acquisition = acquireStructuredRuntime(ctx)
|
||||
expect(acquisition.captured(parent)).toBeUndefined()
|
||||
acquisition.attach(parent, SCHEMA)
|
||||
expect(acquisition.captured(parent)).toBeUndefined()
|
||||
acquisition.detach(parent)
|
||||
acquisition.detach(parent)
|
||||
acquisition.release()
|
||||
// The backend still holds its own reference from setup().
|
||||
expect(ctx.tools.get(STRUCTURED_OUTPUT_TOOL)).toBeDefined()
|
||||
})
|
||||
})
|
||||
|
||||
it('a direct structured_output call from an agent WITHOUT a structured run is an isError', async () => {
|
||||
const { ctx, parent } = await setup([])
|
||||
const result = await ctx.tools.execute({
|
||||
callId: 'x' as never,
|
||||
name: STRUCTURED_OUTPUT_TOOL,
|
||||
arguments: { answer: 1 },
|
||||
agent: parent,
|
||||
})
|
||||
expect(result.isError).toBe(true)
|
||||
expect(result.content[0]).toMatchObject({ type: 'text' })
|
||||
})
|
||||
|
||||
it('a structured_output call with NO calling agent at all is an isError', async () => {
|
||||
const { ctx } = await setup([])
|
||||
const result = await ctx.tools.execute({
|
||||
callId: 'x' as never,
|
||||
name: STRUCTURED_OUTPUT_TOOL,
|
||||
arguments: { answer: 1 },
|
||||
})
|
||||
expect(result.isError).toBe(true)
|
||||
})
|
||||
})
|
||||
Reference in New Issue
Block a user