workflow: dynamic workflows — script-driven multi-agent orchestration
A new capability family at packages/workflow/ in the bash seam shape,
modeled on Claude Code's dynamic workflows: the model writes a JavaScript
orchestration script (export const meta = {...} + plain-JS body), a runtime
executes it, and the script — not the conversation — holds the loop, the
branching, and the intermediate results.
- dsh-workflow (ctx.workflows): abstract WorkflowService + run vocabulary
(WorkflowRun whose result NEVER rejects) + observe-only workflow/* events
carrying data snapshots (id + meta, never the live run), per-listener
contained like subagent/*.
- dsh-workflow-vm: in-process node:vm engine. Meta extraction via a
string/comment-aware scanner (template interpolation rejected; literal
evaluated alone in an empty timed context; statement blanked line-
preservingly so stacks keep script line numbers). Hooks: agent(prompt,
{label, phase, schema, model}) over ctx.subagents, parallel(), pipeline()
(no cross-stage barrier), phase(), log(), args. Fatal-vs-null discipline:
hook misuse (unknown/deferred options, bad arguments, unsupported
schemas, tripped caps, seam start failures, cancellation) throws fatal
WorkflowErrors the combinators RE-THROW — never dissolved into the
per-item null reserved for child failures. Realm boundary: inbound values
materialized by descriptor walks that never invoke accessors (defineProperty
copies, __proto__-safe); outbound values rebuilt in-realm via the
context's own JSON.parse. Determinism bans (Date.now/Math.random/argless
new Date) kept so future resume support cannot break scripts. Caps and
timeouts are validated Config. Every hook promise carries a no-op
rejection consumer (app-boot exits on unhandled rejections).
- dsh-tool-workflow: the model-facing workflow tool, synchronous like
dsh-tool-subagent (start → await → try/finally dispose; abort bridged;
non-completed → isError). Generic render card titled by a textual
meta.name sniff. The tool description carries the authoring contract.
Wired into examples/{coding-agent,acp-agent} with explicit-ask-only
guidance. Coverage at every tier: unit (meta scanner, materializer incl.
counting-getter and __proto__ regressions, combinator semantics,
concurrency ceiling, caps, cancellation, no-unhandled-rejection abandon),
integration over the real spawn stack, with-key e2e (real two-phase run +
the tool through the registry pipeline), and a recorded ACP snapshot
scenario (workflow-run, 1 child session). RFC:
docs/rfc/implemented/feature/2026-07-05-dynamic-workflows.md (deferred
work explicitly listed). AGENTS.md budget 1575 → 1590 for the new group's
layout line.
This commit is contained in:
89
packages/workflow/workflow-vm/tests/integration.spec.ts
Normal file
89
packages/workflow/workflow-vm/tests/integration.spec.ts
Normal file
@@ -0,0 +1,89 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { Context } from 'cordis'
|
||||
import LlmService from '@deepseek-ai/dsh-llm'
|
||||
import SessionStore from '@deepseek-ai/dsh-session'
|
||||
import SystemPrompt from '@deepseek-ai/dsh-system-prompt'
|
||||
import ToolRegistry from '@deepseek-ai/dsh-tools'
|
||||
import AgentRegistry, { AgentId } from '@deepseek-ai/dsh-agent'
|
||||
import AgentLoop from '@deepseek-ai/dsh-agent-loop'
|
||||
import * as Invariants from '@deepseek-ai/dsh-invariants'
|
||||
import SubagentService from '@deepseek-ai/dsh-subagent'
|
||||
import * as spawn from '@deepseek-ai/dsh-subagent-spawn'
|
||||
import { STRUCTURED_OUTPUT_TOOL } from '@deepseek-ai/dsh-subagent-inprocess'
|
||||
import { MockAdapter, textResponse, toolCallResponse } from '../../../core/agent-loop/tests/mock-adapter.ts'
|
||||
import VmWorkflowEngine from '../src/index.ts'
|
||||
|
||||
type Script = ConstructorParameters<typeof MockAdapter>[0]
|
||||
|
||||
/**
|
||||
* The whole in-process stack, keyless: the vm engine drives the REAL spawn
|
||||
* backend (with its structured runtime) on a real agent loop; the scripted
|
||||
* mock MODEL is the only mocked boundary. This is the integration guard the
|
||||
* per-hook unit tests (which stub the subagent seam) structurally cannot give.
|
||||
*/
|
||||
async function setup(script: Script) {
|
||||
const ctx = new Context()
|
||||
const adapter = new MockAdapter(script)
|
||||
await ctx.plugin(LlmService)
|
||||
await ctx.plugin(SessionStore)
|
||||
await ctx.plugin(SystemPrompt)
|
||||
await ctx.plugin(ToolRegistry)
|
||||
await ctx.plugin(AgentRegistry)
|
||||
await ctx.plugin(Invariants)
|
||||
await ctx.plugin(AgentLoop, { agents: [] })
|
||||
await ctx.plugin(SubagentService)
|
||||
await ctx.plugin(spawn, { providerName: 'spawn', structuredNudgeRetries: 1 })
|
||||
await ctx.plugin(VmWorkflowEngine, {})
|
||||
ctx.llm.registerAdapter(['mock'], adapter)
|
||||
const parent = ctx.agentLoop.create(AgentId('parent'), { model: 'mock' })
|
||||
return { ctx, parent, adapter }
|
||||
}
|
||||
|
||||
describe('dsh-workflow-vm over the real in-process stack', () => {
|
||||
it('runs a two-stage workflow: a plain child, then a schema child through the structured runtime', async () => {
|
||||
const { ctx, parent } = await setup([
|
||||
textResponse('the file list is a.ts'),
|
||||
toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { verdict: 'real', confidence: 0.9 }),
|
||||
])
|
||||
const childIds: string[] = []
|
||||
ctx.on('workflow/agent-start', (_info, agent) => { childIds.push(agent.childId) })
|
||||
const run = ctx.workflows.start({
|
||||
script: `export const meta = { name: 'integration', description: 'plain + structured children' }
|
||||
phase('Read')
|
||||
const prose = await agent('read the repo')
|
||||
phase('Judge')
|
||||
const judged = await agent('judge: ' + prose, {
|
||||
schema: { type: 'object', properties: { verdict: { type: 'string', enum: ['real', 'bogus'] }, confidence: { type: 'number' } }, required: ['verdict'] },
|
||||
})
|
||||
return { prose, verdict: judged.verdict, confidence: judged.confidence }`,
|
||||
parent,
|
||||
})
|
||||
const result = await run.result
|
||||
expect(result.stopReason).toBe('completed')
|
||||
expect(result.value).toEqual({ prose: 'the file list is a.ts', verdict: 'real', confidence: 0.9 })
|
||||
expect(result.agentsStarted).toBe(2)
|
||||
await run.dispose()
|
||||
// Both children were disposed to quiescence — no live child agents remain.
|
||||
expect(childIds.length).toBe(2)
|
||||
for (const childId of childIds) {
|
||||
expect(ctx.agents.get(AgentId(childId))).toBeUndefined()
|
||||
}
|
||||
})
|
||||
|
||||
it('a child that fails against its schema (nudges exhausted) reaches the script as null', async () => {
|
||||
const { ctx, parent } = await setup([
|
||||
textResponse('prose only'),
|
||||
textResponse('still prose after the nudge'),
|
||||
])
|
||||
const run = ctx.workflows.start({
|
||||
script: `export const meta = { name: 'null-path', description: 'schema failure maps to null' }
|
||||
const judged = await agent('judge it', { schema: { type: 'object', properties: { v: { type: 'string' } } } })
|
||||
return { got: judged === null ? 'null' : 'value' }`,
|
||||
parent,
|
||||
})
|
||||
const result = await run.result
|
||||
expect(result.stopReason).toBe('completed')
|
||||
expect(result.value).toEqual({ got: 'null' })
|
||||
await run.dispose()
|
||||
})
|
||||
})
|
||||
143
packages/workflow/workflow-vm/tests/meta.spec.ts
Normal file
143
packages/workflow/workflow-vm/tests/meta.spec.ts
Normal file
@@ -0,0 +1,143 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { WorkflowError } from '@deepseek-ai/dsh-workflow'
|
||||
import { extractMeta } from '../src/meta.ts'
|
||||
|
||||
const TIMEOUT = 1000
|
||||
|
||||
/** Extract and expect success. */
|
||||
function ok(script: string) {
|
||||
return extractMeta(script, TIMEOUT)
|
||||
}
|
||||
|
||||
/** The WorkflowError a bad script produces (throws if it extracts cleanly). */
|
||||
function bad(script: string): WorkflowError {
|
||||
try {
|
||||
extractMeta(script, TIMEOUT)
|
||||
} catch (error: unknown) {
|
||||
if (error instanceof WorkflowError) return error
|
||||
throw error
|
||||
}
|
||||
throw new Error('expected extraction to fail')
|
||||
}
|
||||
|
||||
describe('extractMeta', () => {
|
||||
it('extracts a full meta block and blanks the statement line-preservingly', () => {
|
||||
const script = `export const meta = {
|
||||
name: 'audit-routes',
|
||||
description: 'Audit every route',
|
||||
whenToUse: 'when auditing',
|
||||
phases: [{ title: 'Scan', detail: 'find files' }, { title: 'Fix', model: 'deepseek-v4-pro' }],
|
||||
}
|
||||
const x = 1
|
||||
return x`
|
||||
const { meta, body } = ok(script)
|
||||
expect(meta).toEqual({
|
||||
name: 'audit-routes',
|
||||
description: 'Audit every route',
|
||||
whenToUse: 'when auditing',
|
||||
phases: [{ title: 'Scan', detail: 'find files' }, { title: 'Fix', model: 'deepseek-v4-pro' }],
|
||||
})
|
||||
// Same line count; the statement's characters blanked; the body intact.
|
||||
expect(body.split('\n').length).toBe(script.split('\n').length)
|
||||
expect(body.split('\n')[6]).toBe('const x = 1')
|
||||
expect(body).not.toContain('export')
|
||||
})
|
||||
|
||||
it('allows leading line and block comments before the meta statement', () => {
|
||||
const script = `// a workflow
|
||||
/* multi
|
||||
line */
|
||||
export const meta = { name: 'x', description: 'y' }
|
||||
return 1`
|
||||
expect(ok(script).meta.name).toBe('x')
|
||||
})
|
||||
|
||||
it('handles braces inside strings and comments while scanning', () => {
|
||||
const script = `export const meta = {
|
||||
name: 'tricky', // } not a close {
|
||||
/* } also not } */
|
||||
description: "has { braces } and 'quotes'",
|
||||
}
|
||||
return 2`
|
||||
expect(ok(script).meta.description).toBe("has { braces } and 'quotes'")
|
||||
})
|
||||
|
||||
it('tolerates template-quoted strings WITHOUT interpolation, escapes included', () => {
|
||||
const script = 'export const meta = { name: `plain`, description: `esc \\` tick` }\nreturn 1'
|
||||
expect(ok(script).meta.name).toBe('plain')
|
||||
})
|
||||
|
||||
it('consumes a trailing semicolon after the literal, spaces included', () => {
|
||||
const { body } = ok("export const meta = { name: 'x', description: 'y' };\nreturn 1")
|
||||
expect(body).not.toContain(';')
|
||||
expect(body.split('\n')[1]).toBe('return 1')
|
||||
const spaced = ok("export const meta = { name: 'x', description: 'y' } ;\nreturn 1")
|
||||
expect(spaced.body).not.toContain(';')
|
||||
})
|
||||
|
||||
it('rejects a script that does not begin with the meta statement (SCRIPT_PARSE)', () => {
|
||||
expect(bad('const a = 1').code).toBe('SCRIPT_PARSE')
|
||||
expect(bad('').code).toBe('SCRIPT_PARSE')
|
||||
expect(bad('export const meta = [1]').code).toBe('SCRIPT_PARSE')
|
||||
})
|
||||
|
||||
it('rejects template interpolation in the meta block as impure (SCRIPT_PARSE)', () => {
|
||||
const error = bad('export const meta = { name: `w-${1}`, description: "d" }\nreturn 1')
|
||||
expect(error.code).toBe('SCRIPT_PARSE')
|
||||
expect(error.message).toContain('pure literal')
|
||||
})
|
||||
|
||||
it('rejects unbalanced literals, unterminated strings, and unterminated comments (SCRIPT_PARSE)', () => {
|
||||
expect(bad('export const meta = { name: "x", description: "y"').code).toBe('SCRIPT_PARSE')
|
||||
expect(bad('export const meta = { name: "x').code).toBe('SCRIPT_PARSE')
|
||||
expect(bad('export const meta = { /* open').code).toBe('SCRIPT_PARSE')
|
||||
// A line comment running to EOF (no newline) leaves the literal unbalanced.
|
||||
expect(bad('export const meta = { name: "x" // eof comment').code).toBe('SCRIPT_PARSE')
|
||||
})
|
||||
|
||||
it('rejects a literal referencing variables or calls (META_INVALID via the empty realm)', () => {
|
||||
const error = bad('export const meta = { name: someVariable, description: "d" }\nreturn 1')
|
||||
expect(error.code).toBe('META_INVALID')
|
||||
expect(error.message).toContain('pure literal')
|
||||
expect(bad('export const meta = { name: compute(), description: "d" }').code).toBe('META_INVALID')
|
||||
})
|
||||
|
||||
it('rejects a literal evaluating to non-JSON data (META_INVALID via materialization)', () => {
|
||||
const error = bad('export const meta = { name: "x", description: "d", phases: [{ get title() { return "t" } }] }')
|
||||
expect(error.code).toBe('META_INVALID')
|
||||
expect(error.message).toContain('JSON data')
|
||||
})
|
||||
|
||||
it('rejects shape violations with EVERY violation listed (META_INVALID)', () => {
|
||||
const error = bad('export const meta = { description: 7, bogus: 1 }\nreturn 1')
|
||||
expect(error.code).toBe('META_INVALID')
|
||||
expect(error.message).toContain('meta.name must be a non-empty string')
|
||||
expect(error.message).toContain('meta.description must be a non-empty string')
|
||||
expect(error.message).toContain('meta.bogus is not a recognized field')
|
||||
})
|
||||
|
||||
it('rejects malformed whenToUse and phases shapes precisely', () => {
|
||||
expect(bad('export const meta = { name: "x", description: "d", whenToUse: 3 }').message)
|
||||
.toContain('meta.whenToUse must be a string')
|
||||
expect(bad('export const meta = { name: "x", description: "d", phases: "no" }').message)
|
||||
.toContain('meta.phases must be an array')
|
||||
expect(bad('export const meta = { name: "x", description: "d", phases: [3] }').message)
|
||||
.toContain('meta.phases[0] must be an object')
|
||||
expect(bad('export const meta = { name: "x", description: "d", phases: [{}] }').message)
|
||||
.toContain('meta.phases[0].title must be a non-empty string')
|
||||
expect(bad('export const meta = { name: "x", description: "d", phases: [{ title: "t", extra: 1 }] }').message)
|
||||
.toContain('meta.phases[0].extra is not a recognized field')
|
||||
expect(bad('export const meta = { name: "x", description: "d", phases: [{ title: "t", detail: 1 }] }').message)
|
||||
.toContain('meta.phases[0].detail must be a string')
|
||||
expect(bad('export const meta = { name: "x", description: "d", phases: [{ title: "t", model: 1 }] }').message)
|
||||
.toContain('meta.phases[0].model must be a string')
|
||||
})
|
||||
|
||||
it('stops scanning at the balanced literal — trailing expression text stays in the body', () => {
|
||||
// The scanner extracts exactly `{ valueOf: null }`; the ` && 3` is body
|
||||
// text (which would fail compilation later, but extraction sees only the
|
||||
// literal and reports its unknown field).
|
||||
expect(bad('export const meta = { valueOf: null } && 3').message)
|
||||
.toContain('meta.valueOf is not a recognized field')
|
||||
})
|
||||
})
|
||||
114
packages/workflow/workflow-vm/tests/realm.spec.ts
Normal file
114
packages/workflow/workflow-vm/tests/realm.spec.ts
Normal file
@@ -0,0 +1,114 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import * as vm from 'node:vm'
|
||||
import { materializeFromRealm, MaterializeError } from '../src/realm.ts'
|
||||
|
||||
/** Evaluate an expression inside a fresh vm realm and hand back the raw realm value. */
|
||||
function inRealm(expression: string): unknown {
|
||||
return vm.runInNewContext(`(${expression})`)
|
||||
}
|
||||
|
||||
/** The MaterializeError message for a value that must be rejected (throws if accepted). */
|
||||
function rejection(value: unknown): string {
|
||||
try {
|
||||
materializeFromRealm(value)
|
||||
} catch (error: unknown) {
|
||||
if (error instanceof MaterializeError) return error.message
|
||||
throw error
|
||||
}
|
||||
throw new Error('expected the value to be rejected')
|
||||
}
|
||||
|
||||
describe('materializeFromRealm', () => {
|
||||
it('copies realm objects/arrays/scalars into host plain data', () => {
|
||||
const value = inRealm("{ a: 1, b: 'x', c: true, d: null, list: [1, [2, { deep: 'y' }]] }")
|
||||
const out = materializeFromRealm(value) as Record<string, unknown>
|
||||
expect(out).toEqual({ a: 1, b: 'x', c: true, d: null, list: [1, [2, { deep: 'y' }]] })
|
||||
// The copy is HOST data: prototypes are the host intrinsics.
|
||||
expect(Object.getPrototypeOf(out)).toBe(Object.prototype)
|
||||
expect(Array.isArray(out.list)).toBe(true)
|
||||
// And it round-trips through JSON byte-identically (the whole point).
|
||||
expect(JSON.parse(JSON.stringify(out))).toEqual(out)
|
||||
})
|
||||
|
||||
it('accepts undefined ONLY at the root (a valueless script return)', () => {
|
||||
expect(materializeFromRealm(undefined)).toBeUndefined()
|
||||
expect(rejection(inRealm('{ a: undefined }'))).toContain('value.a')
|
||||
})
|
||||
|
||||
it('never invokes accessors: a counting getter is rejected, not read', () => {
|
||||
const counter = inRealm(`
|
||||
(() => {
|
||||
globalThis.reads = 0
|
||||
return { get x() { globalThis.reads += 1; return 1 } }
|
||||
})()
|
||||
`)
|
||||
expect(rejection(counter)).toContain('accessor properties cannot cross')
|
||||
// The getter body never ran — descriptor inspection only.
|
||||
expect((counter as { x?: unknown }).x).toBe(1) // sanity: reading DOES run it…
|
||||
expect(rejection(counter)).toContain('accessor') // …but materialization still never did
|
||||
})
|
||||
|
||||
it('a "__proto__" key becomes an OWN data property of the copy, never a prototype mutation', () => {
|
||||
const value: unknown = vm.runInNewContext('JSON.parse(\'{"__proto__": {"polluted": 1}, "ok": 2}\')')
|
||||
const out = materializeFromRealm(value) as Record<string, unknown>
|
||||
expect(Object.getPrototypeOf(out)).toBe(Object.prototype)
|
||||
expect(Object.prototype.hasOwnProperty.call(out, '__proto__')).toBe(true)
|
||||
expect(out.ok).toBe(2)
|
||||
// The host Object.prototype was NOT touched.
|
||||
expect(({} as Record<string, unknown>).polluted).toBeUndefined()
|
||||
})
|
||||
|
||||
it('rejects functions, symbols (keys and values), and bigints with path-qualified messages', () => {
|
||||
expect(rejection(inRealm('{ fn: () => 1 }'))).toContain('value.fn')
|
||||
expect(rejection(inRealm("{ [Symbol('k')]: 1 }"))).toContain('symbol-keyed')
|
||||
expect(rejection(inRealm("{ s: Symbol('v') }"))).toContain('value.s')
|
||||
expect(rejection(inRealm('{ big: 1n }'))).toContain('value.big')
|
||||
expect(rejection(inRealm("[Symbol('x')]"))).toContain('value[0]')
|
||||
const taggedArray = inRealm("(() => { const a = [1]; a[Symbol('t')] = 1; return a })()")
|
||||
expect(rejection(taggedArray)).toContain('symbol-keyed')
|
||||
})
|
||||
|
||||
it('rejects non-finite numbers and undefined values inside containers', () => {
|
||||
expect(rejection(inRealm('{ n: NaN }'))).toContain('non-finite')
|
||||
expect(rejection(inRealm('[Infinity]'))).toContain('non-finite')
|
||||
})
|
||||
|
||||
it('rejects exotic prototypes (Date, Map, class instances) but accepts null-prototype data', () => {
|
||||
expect(rejection(inRealm('{ d: new Date(0) }'))).toContain('exotic prototype')
|
||||
expect(rejection(inRealm('new Map()'))).toContain('exotic prototype')
|
||||
expect(rejection(inRealm('(() => { class C { constructor() { this.x = 1 } } return new C() })()')))
|
||||
.toContain('exotic prototype')
|
||||
expect(materializeFromRealm(inRealm('Object.assign(Object.create(null), { a: 1 })'))).toEqual({ a: 1 })
|
||||
})
|
||||
|
||||
it('rejects cycles and accepts the same object reused as a sibling (a DAG)', () => {
|
||||
expect(rejection(inRealm('(() => { const o = {}; o.self = o; return o })()'))).toContain('circular')
|
||||
const dag = inRealm('(() => { const leaf = { v: 1 }; return { a: leaf, b: leaf } })()')
|
||||
expect(materializeFromRealm(dag)).toEqual({ a: { v: 1 }, b: { v: 1 } })
|
||||
})
|
||||
|
||||
it('rejects sparse arrays, accessor elements, and non-index array properties', () => {
|
||||
expect(rejection(inRealm('[1, , 3]'))).toContain('sparse')
|
||||
expect(rejection(inRealm('(() => { const a = [1]; Object.defineProperty(a, 0, { get: () => 1 }); return a })()')))
|
||||
.toContain('accessor')
|
||||
expect(rejection(inRealm('(() => { const a = [1]; a.total = 3; return a })()')))
|
||||
.toContain('non-index')
|
||||
})
|
||||
|
||||
it('skips non-enumerable own properties (matching JSON.stringify exactly)', () => {
|
||||
const value = inRealm(`(() => {
|
||||
const o = { visible: 1 }
|
||||
Object.defineProperty(o, 'hidden', { value: () => 1, enumerable: false })
|
||||
return o
|
||||
})()`)
|
||||
expect(materializeFromRealm(value)).toEqual({ visible: 1 })
|
||||
})
|
||||
|
||||
it('works on plain host values too (the boundary is realm-agnostic)', () => {
|
||||
expect(materializeFromRealm({ a: [1, 'x'] })).toEqual({ a: [1, 'x'] })
|
||||
expect(materializeFromRealm('str')).toBe('str')
|
||||
expect(materializeFromRealm(3)).toBe(3)
|
||||
expect(materializeFromRealm(false)).toBe(false)
|
||||
expect(materializeFromRealm(null)).toBeNull()
|
||||
})
|
||||
})
|
||||
614
packages/workflow/workflow-vm/tests/workflow-vm.spec.ts
Normal file
614
packages/workflow/workflow-vm/tests/workflow-vm.spec.ts
Normal file
@@ -0,0 +1,614 @@
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import { Context } from 'cordis'
|
||||
import Loader from '@cordisjs/plugin-loader'
|
||||
import { AgentId } from '@deepseek-ai/dsh-agent'
|
||||
import type { Agent } from '@deepseek-ai/dsh-agent'
|
||||
import SubagentService from '@deepseek-ai/dsh-subagent'
|
||||
import type { SubagentCapabilities, SubagentProvider, SubagentResult, SubagentRun, SubagentStartRequest } from '@deepseek-ai/dsh-subagent'
|
||||
import type { WorkflowResult, WorkflowRunInfo } from '@deepseek-ai/dsh-workflow'
|
||||
import * as vmEngineModule from '../src/index.ts'
|
||||
import VmWorkflowEngine, { type Config } from '../src/index.ts'
|
||||
|
||||
/** A minimal parent stand-in: the engine only threads it through to the provider. */
|
||||
function fakeParent(): Agent {
|
||||
return { id: AgentId('workflow-parent'), options: {} } as unknown as Agent
|
||||
}
|
||||
|
||||
/** One controllable child run: the test (or auto mode) settles it. */
|
||||
interface ControlledRun {
|
||||
request: SubagentStartRequest
|
||||
settle(result: SubagentResult): void
|
||||
cancelled: string | undefined
|
||||
disposed: boolean
|
||||
}
|
||||
|
||||
/**
|
||||
* A scripted in-test provider over the REAL SubagentService registry: `auto`
|
||||
* settles each run via the reply function on a microtask; `manual` piles runs
|
||||
* up in `runs` for the test to settle (concurrency/cancellation tests). A run
|
||||
* aborts (settles `aborted`) when the request signal fires, like the real
|
||||
* in-process backends.
|
||||
*/
|
||||
class StubProvider implements SubagentProvider {
|
||||
readonly capabilities: SubagentCapabilities = { outputSchema: true, depthLimit: true, toolFilter: true }
|
||||
readonly runs: ControlledRun[] = []
|
||||
|
||||
constructor(
|
||||
readonly name: string,
|
||||
private readonly reply?: (request: SubagentStartRequest, index: number) => SubagentResult,
|
||||
) {}
|
||||
|
||||
start(request: SubagentStartRequest): SubagentRun {
|
||||
let settle!: (result: SubagentResult) => void
|
||||
const result = new Promise<SubagentResult>((resolve) => { settle = resolve })
|
||||
const controlled: ControlledRun = { request, settle, cancelled: undefined, disposed: false }
|
||||
this.runs.push(controlled)
|
||||
const index = this.runs.length - 1
|
||||
request.signal?.addEventListener('abort', () => { settle({ output: [], stopReason: 'aborted' }) }, { once: true })
|
||||
if (this.reply) {
|
||||
const reply = this.reply
|
||||
queueMicrotask(() => { settle(reply(request, index)) })
|
||||
}
|
||||
return {
|
||||
id: AgentId(`stub-child-${index}`),
|
||||
result,
|
||||
cancel: (reason?: string) => {
|
||||
controlled.cancelled = reason ?? 'cancelled'
|
||||
settle({ output: [], stopReason: 'aborted' })
|
||||
},
|
||||
dispose: () => {
|
||||
controlled.disposed = true
|
||||
return Promise.resolve()
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Text-reply helper for auto providers. */
|
||||
function text(reply: string): SubagentResult {
|
||||
return { output: [{ type: 'text', text: reply }], stopReason: 'completed' }
|
||||
}
|
||||
|
||||
interface SetupOptions {
|
||||
config?: Config
|
||||
reply?: (request: SubagentStartRequest, index: number) => SubagentResult
|
||||
manual?: boolean
|
||||
}
|
||||
|
||||
async function setup(options?: SetupOptions) {
|
||||
const ctx = new Context()
|
||||
await ctx.plugin(SubagentService)
|
||||
const provider = new StubProvider('stub', options?.manual ? undefined : options?.reply ?? (() => text('stub reply')))
|
||||
ctx.subagents.registerProvider(provider)
|
||||
await ctx.plugin(VmWorkflowEngine, { provider: 'stub', ...options?.config })
|
||||
return { ctx, provider, parent: fakeParent() }
|
||||
}
|
||||
|
||||
/** Wrap a body in the minimal valid meta header. */
|
||||
function script(body: string, metaExtra = ''): string {
|
||||
return `export const meta = { name: 'test-flow', description: 'a test workflow'${metaExtra} }\n${body}`
|
||||
}
|
||||
|
||||
/** Start + await one run, disposing on the way out. */
|
||||
async function run(ctx: Context, parent: Agent, source: string, args?: unknown): Promise<WorkflowResult> {
|
||||
const handle = ctx.workflows.start({ script: source, parent, ...args !== undefined ? { args } : {} })
|
||||
try {
|
||||
return await handle.result
|
||||
} finally {
|
||||
await handle.dispose()
|
||||
}
|
||||
}
|
||||
|
||||
describe('dsh-workflow-vm', () => {
|
||||
describe('script execution', () => {
|
||||
it('runs a script end-to-end: agent() text results, phases, log, args, return value', async () => {
|
||||
const { ctx, parent, provider } = await setup({ reply: (_request, index) => text(`answer-${index}`) })
|
||||
const events: [string, unknown[]][] = []
|
||||
for (const name of ['workflow/start', 'workflow/phase', 'workflow/log', 'workflow/agent-start', 'workflow/agent-end', 'workflow/end'] as const) {
|
||||
ctx.on(name, (...payload: unknown[]) => { events.push([name, payload]) })
|
||||
}
|
||||
const result = await run(ctx, parent, script(`
|
||||
phase('Scan')
|
||||
log('starting with ' + args.files.length + ' files')
|
||||
const answers = await pipeline(args.files, (prev, item) => agent('read ' + item))
|
||||
phase('Report')
|
||||
return { answers, count: args.files.length }
|
||||
`, ", phases: [{ title: 'Scan' }, { title: 'Report' }]"), { files: ['a.ts', 'b.ts'] })
|
||||
|
||||
expect(result.stopReason).toBe('completed')
|
||||
expect(result.agentsStarted).toBe(2)
|
||||
expect(result.value).toEqual({ answers: ['answer-0', 'answer-1'], count: 2 })
|
||||
expect(provider.runs.every(r => r.disposed)).toBe(true)
|
||||
|
||||
const names = events.map(([name]) => name)
|
||||
expect(names[0]).toBe('workflow/start')
|
||||
expect(names).toContain('workflow/phase')
|
||||
expect(names).toContain('workflow/log')
|
||||
expect(names.at(-1)).toBe('workflow/end')
|
||||
const info = events[0]![1][0] as WorkflowRunInfo
|
||||
expect(info.meta.name).toBe('test-flow')
|
||||
const end = events.at(-1)![1][1] as Record<string, unknown>
|
||||
expect(end).toEqual({ stopReason: 'completed', agentsStarted: 2 })
|
||||
expect('value' in end).toBe(false)
|
||||
})
|
||||
|
||||
it('agent-start/end events carry seq, label (defaulted from the prompt), phase, and outcome', async () => {
|
||||
const { ctx, parent } = await setup()
|
||||
const starts: unknown[] = []
|
||||
const ends: unknown[] = []
|
||||
ctx.on('workflow/agent-start', (_info, agent) => starts.push(agent))
|
||||
ctx.on('workflow/agent-end', (_info, agent) => ends.push(agent))
|
||||
await run(ctx, parent, script(`
|
||||
phase('Find')
|
||||
await agent('a prompt that is quite long and will surely get truncated down to a display label\\n'
|
||||
+ 'with a second line the label must not include')
|
||||
await agent('short', { label: 'named', phase: 'Custom' })
|
||||
return null
|
||||
`))
|
||||
expect(starts[0]).toMatchObject({ seq: 1, phase: 'Find', childId: 'stub-child-0' })
|
||||
expect((starts[0] as { label: string }).label.length).toBeLessThanOrEqual(48)
|
||||
expect((starts[0] as { label: string }).label).not.toContain('second line')
|
||||
expect(starts[1]).toMatchObject({ seq: 2, label: 'named', phase: 'Custom' })
|
||||
expect(ends[0]).toMatchObject({ seq: 1, outcome: 'completed' })
|
||||
})
|
||||
|
||||
it('agent({schema}) forwards outputSchema to the provider and returns the structured value into the realm', async () => {
|
||||
const { ctx, parent, provider } = await setup({
|
||||
reply: () => ({ output: [], structured: { files: ['x.ts', 'y.ts'] }, stopReason: 'completed' }),
|
||||
})
|
||||
const result = await run(ctx, parent, script(`
|
||||
const found = await agent('list files', { schema: { type: 'object', properties: { files: { type: 'array', items: { type: 'string' } } }, required: ['files'] } })
|
||||
return { first: found.files[0], count: found.files.length }
|
||||
`))
|
||||
expect(result.value).toEqual({ first: 'x.ts', count: 2 })
|
||||
expect(provider.runs[0]!.request.outputSchema).toEqual({
|
||||
type: 'object',
|
||||
properties: { files: { type: 'array', items: { type: 'string' } } },
|
||||
required: ['files'],
|
||||
})
|
||||
})
|
||||
|
||||
it('model option maps to agentOptions.model on the start request', async () => {
|
||||
const { ctx, parent, provider } = await setup()
|
||||
await run(ctx, parent, script("return await agent('p', { model: 'deepseek-v4-pro' })"))
|
||||
expect(provider.runs[0]!.request.agentOptions).toEqual({ model: 'deepseek-v4-pro' })
|
||||
})
|
||||
|
||||
it('a failed child resolves null (scripts filter), never throwing into the script', async () => {
|
||||
const { ctx, parent } = await setup({
|
||||
reply: (_request, index) => index === 0 ? { output: [], stopReason: 'error' } : text('ok'),
|
||||
})
|
||||
const result = await run(ctx, parent, script(`
|
||||
const results = await parallel([() => agent('one'), () => agent('two')])
|
||||
return results
|
||||
`))
|
||||
expect(result.value).toEqual([null, 'ok'])
|
||||
})
|
||||
|
||||
it('a schema run that completes WITHOUT a structured value is a child failure (null + failed outcome)', async () => {
|
||||
const { ctx, parent } = await setup({ reply: () => text('prose, no structure') })
|
||||
const ends: unknown[] = []
|
||||
ctx.on('workflow/agent-end', (_info, agent) => ends.push(agent))
|
||||
const result = await run(ctx, parent, script(`
|
||||
return await agent('p', { schema: { type: 'object' } })
|
||||
`))
|
||||
expect(result.value).toBeNull()
|
||||
expect(ends[0]).toMatchObject({ outcome: 'failed' })
|
||||
})
|
||||
|
||||
it('a script with no return value resolves value: null', async () => {
|
||||
const { ctx, parent } = await setup()
|
||||
const result = await run(ctx, parent, script("await agent('p')"))
|
||||
expect(result.stopReason).toBe('completed')
|
||||
expect(result.value).toBeNull()
|
||||
})
|
||||
})
|
||||
|
||||
describe('combinator semantics', () => {
|
||||
it('pipeline has NO cross-stage barrier: a fast item finishes stage 2 while a slow item holds stage 1', async () => {
|
||||
const { ctx, parent, provider } = await setup({ manual: true })
|
||||
const handle = ctx.workflows.start({
|
||||
script: script(`
|
||||
const out = await pipeline(['slow', 'fast'],
|
||||
(prev, item) => agent('s1 ' + item),
|
||||
(prev, item) => agent('s2 ' + item + ' after ' + prev),
|
||||
)
|
||||
return out
|
||||
`),
|
||||
parent: fakeParent(),
|
||||
})
|
||||
// Both items enter stage 1 concurrently.
|
||||
await vi.waitFor(() => { expect(provider.runs.length).toBe(2) })
|
||||
// Settle only the FAST item's stage 1 → its stage 2 starts with no barrier.
|
||||
provider.runs[1]!.settle(text('fast-1'))
|
||||
await vi.waitFor(() => { expect(provider.runs.length).toBe(3) })
|
||||
expect((provider.runs[2]!.request.prompt[0] as { text: string }).text).toBe('s2 fast after fast-1')
|
||||
// The slow item is still sitting in stage 1.
|
||||
provider.runs[2]!.settle(text('fast-2'))
|
||||
provider.runs[0]!.settle(text('slow-1'))
|
||||
await vi.waitFor(() => { expect(provider.runs.length).toBe(4) })
|
||||
provider.runs[3]!.settle(text('slow-2'))
|
||||
const result = await handle.result
|
||||
expect(result.value).toEqual(['slow-2', 'fast-2'])
|
||||
await handle.dispose()
|
||||
void parent
|
||||
})
|
||||
|
||||
it('pipeline stage callbacks receive (prev, item, index); an ordinary stage throw nulls the ITEM and skips its remaining stages', async () => {
|
||||
const { ctx, parent, provider } = await setup({ reply: request => text(`ok:${(request.prompt[0] as { text: string }).text}`) })
|
||||
const result = await run(ctx, parent, script(`
|
||||
const out = await pipeline([10, 20],
|
||||
(prev, item, index) => {
|
||||
if (item === 10) throw new Error('ordinary failure')
|
||||
return agent('stage1-' + item + '-' + index)
|
||||
},
|
||||
(prev) => agent('stage2 saw ' + prev),
|
||||
)
|
||||
return out
|
||||
`))
|
||||
expect(result.stopReason).toBe('completed')
|
||||
const prompts = provider.runs.map(r => (r.request.prompt[0] as { text: string }).text)
|
||||
// Item 10 never reached stage 1's agent nor stage 2.
|
||||
expect(prompts).toEqual(['stage1-20-1', 'stage2 saw ok:stage1-20-1'])
|
||||
expect(result.value).toEqual([null, 'ok:stage2 saw ok:stage1-20-1'])
|
||||
})
|
||||
|
||||
it('parallel maps a throwing thunk to null and never rejects for ordinary errors', async () => {
|
||||
const { ctx, parent } = await setup()
|
||||
const result = await run(ctx, parent, script(`
|
||||
return await parallel([
|
||||
() => { throw new Error('boom') },
|
||||
() => agent('fine'),
|
||||
() => 'plain value',
|
||||
])
|
||||
`))
|
||||
expect(result.value).toEqual([null, 'stub reply', 'plain value'])
|
||||
})
|
||||
|
||||
it('FATAL errors propagate through parallel AND pipeline instead of dissolving into null', async () => {
|
||||
const { ctx, parent } = await setup()
|
||||
const viaParallel = await run(ctx, parent, script(`
|
||||
return await parallel([() => agent('x', { isolation: 'worktree' })])
|
||||
`))
|
||||
expect(viaParallel.stopReason).toBe('error')
|
||||
expect(viaParallel.error).toContain('"isolation" is deferred')
|
||||
|
||||
const viaPipeline = await run(ctx, parent, script(`
|
||||
return await pipeline([1], () => agent('x', { bogus: true }))
|
||||
`))
|
||||
expect(viaPipeline.stopReason).toBe('error')
|
||||
expect(viaPipeline.error).toContain('"bogus" is not recognized')
|
||||
})
|
||||
|
||||
it('validates combinator arguments loudly (non-array, non-function, missing stages)', async () => {
|
||||
const { ctx, parent } = await setup()
|
||||
expect((await run(ctx, parent, script("return await parallel('no')"))).error).toContain('parallel() requires an array')
|
||||
expect((await run(ctx, parent, script('return await parallel([3])'))).error).toContain('item 0 is not a function')
|
||||
expect((await run(ctx, parent, script("return await pipeline('no', () => 1)"))).error).toContain('pipeline() requires an items array')
|
||||
expect((await run(ctx, parent, script('return await pipeline([1])'))).error).toContain('at least one stage')
|
||||
expect((await run(ctx, parent, script("return await pipeline([1], 'x')"))).error).toContain('stage 0 is not a function')
|
||||
})
|
||||
})
|
||||
|
||||
describe('caps and option validation', () => {
|
||||
it('trips the total-agent cap with a message naming the config knob', async () => {
|
||||
const { ctx, parent } = await setup({ config: { provider: 'stub', maxTotalAgents: 2 } })
|
||||
const result = await run(ctx, parent, script(`
|
||||
await agent('1'); await agent('2'); await agent('3')
|
||||
return 'unreachable'
|
||||
`))
|
||||
expect(result.stopReason).toBe('error')
|
||||
expect(result.error).toContain('total agent cap (2)')
|
||||
expect(result.error).toContain('maxTotalAgents')
|
||||
expect(result.agentsStarted).toBe(2)
|
||||
})
|
||||
|
||||
it('trips the per-call item cap for parallel and pipeline', async () => {
|
||||
const { ctx, parent } = await setup({ config: { provider: 'stub', maxItemsPerCall: 2 } })
|
||||
expect((await run(ctx, parent, script('return await parallel([() => 1, () => 2, () => 3])'))).error)
|
||||
.toContain('over the per-call cap (2)')
|
||||
expect((await run(ctx, parent, script('return await pipeline([1, 2, 3], (x) => x)'))).error)
|
||||
.toContain('maxItemsPerCall')
|
||||
})
|
||||
|
||||
it('enforces the concurrency ceiling: never more than maxConcurrentAgents children in flight', async () => {
|
||||
const { ctx, parent, provider } = await setup({ manual: true, config: { provider: 'stub', maxConcurrentAgents: 2 } })
|
||||
const handle = ctx.workflows.start({
|
||||
script: script("return await parallel([1, 2, 3, 4, 5].map((n) => () => agent('job ' + n)))"),
|
||||
parent,
|
||||
})
|
||||
// Only 2 children may exist until one settles.
|
||||
await vi.waitFor(() => { expect(provider.runs.length).toBe(2) })
|
||||
await new Promise(resolve => setTimeout(resolve, 20))
|
||||
expect(provider.runs.length).toBe(2)
|
||||
// Settle children in arrival order; after each settle at most ONE more
|
||||
// child may enter — the window never exceeds the ceiling.
|
||||
for (let index = 0; index < 5; index++) {
|
||||
await vi.waitFor(() => { expect(provider.runs.length).toBeGreaterThan(index) })
|
||||
expect(provider.runs.length).toBeLessThanOrEqual(Math.min(index + 2, 5))
|
||||
provider.runs[index]!.settle(text(`r${index}`))
|
||||
}
|
||||
const result = await handle.result
|
||||
expect(result.stopReason).toBe('completed')
|
||||
expect(result.agentsStarted).toBe(5)
|
||||
expect(result.value).toEqual(['r0', 'r1', 'r2', 'r3', 'r4'])
|
||||
await handle.dispose()
|
||||
})
|
||||
|
||||
it('rejects malformed agent() arguments and option types loudly', async () => {
|
||||
const { ctx, parent } = await setup()
|
||||
expect((await run(ctx, parent, script('return await agent(42)'))).error).toContain('non-empty prompt string')
|
||||
expect((await run(ctx, parent, script("return await agent('')"))).error).toContain('non-empty prompt string')
|
||||
expect((await run(ctx, parent, script("return await agent('p', 'opts')"))).error).toContain('options must be an object')
|
||||
expect((await run(ctx, parent, script("return await agent('p', { label: 3 })"))).error).toContain('"label" must be a string')
|
||||
expect((await run(ctx, parent, script("return await agent('p', { effort: 'high' })"))).error).toContain('"effort" is deferred')
|
||||
})
|
||||
|
||||
it('rejects options that are not plain JSON data (an accessor smuggled into opts)', async () => {
|
||||
const { ctx, parent } = await setup()
|
||||
const result = await run(ctx, parent, script("return await agent('p', { get label() { return 'x' } })"))
|
||||
expect(result.stopReason).toBe('error')
|
||||
expect(result.error).toContain('options must be plain JSON data')
|
||||
})
|
||||
|
||||
it('validates phase() and log() arguments loudly', async () => {
|
||||
const { ctx, parent } = await setup()
|
||||
expect((await run(ctx, parent, script('phase(3)'))).error).toContain('phase() requires a non-empty title string')
|
||||
expect((await run(ctx, parent, script("phase('')"))).error).toContain('phase() requires a non-empty title string')
|
||||
expect((await run(ctx, parent, script('log(3)'))).error).toContain('log() requires a message string')
|
||||
})
|
||||
|
||||
it('rejects an unsupported schema via the shared subset assertion (UNSUPPORTED_SCHEMA)', async () => {
|
||||
const { ctx, parent } = await setup()
|
||||
const result = await run(ctx, parent, script("return await agent('p', { schema: { type: 'object', oneOf: [] } })"))
|
||||
expect(result.stopReason).toBe('error')
|
||||
expect(result.error).toContain('outside the supported subset')
|
||||
expect(result.error).toContain('oneOf')
|
||||
})
|
||||
|
||||
it('wraps a provider start failure as a fatal AGENT_START error (a missing provider cannot dissolve into null)', async () => {
|
||||
const { ctx, parent } = await setup({ config: { provider: 'nonexistent' } })
|
||||
const result = await run(ctx, parent, script("return await pipeline([1], () => agent('p'))"))
|
||||
expect(result.stopReason).toBe('error')
|
||||
expect(result.error).toContain('could not start a child on provider "nonexistent"')
|
||||
})
|
||||
})
|
||||
|
||||
describe('determinism bans and realm isolation', () => {
|
||||
it('Date.now, Math.random, and argless new Date throw; parameterized Date stays usable', async () => {
|
||||
const { ctx, parent } = await setup()
|
||||
expect((await run(ctx, parent, script('return Date.now()'))).error).toContain('Date.now() is not available')
|
||||
expect((await run(ctx, parent, script('return Math.random()'))).error).toContain('Math.random() is not available')
|
||||
expect((await run(ctx, parent, script('return new Date().toISOString()'))).error).toContain('argless new Date()')
|
||||
const ok = await run(ctx, parent, script('return new Date(0).getTime()'))
|
||||
expect(ok.value).toBe(0)
|
||||
})
|
||||
|
||||
it('args cross into the realm as data: mutating them (or their prototype chain) cannot reach host intrinsics', async () => {
|
||||
const { ctx, parent } = await setup()
|
||||
const hostArgs = { files: ['a.ts'], nested: { deep: [1, 2] } }
|
||||
const result = await run(ctx, parent, script(`
|
||||
args.files.push('b.ts')
|
||||
Object.getPrototypeOf(args).polluted = 'realm-only'
|
||||
return { count: args.files.length, deep: args.nested.deep[1] }
|
||||
`), hostArgs)
|
||||
expect(result.value).toEqual({ count: 2, deep: 2 })
|
||||
// The host copy is untouched, and the HOST Object.prototype was never reachable.
|
||||
expect(hostArgs.files).toEqual(['a.ts'])
|
||||
expect(({} as Record<string, unknown>).polluted).toBeUndefined()
|
||||
})
|
||||
|
||||
it('scalar/null args pass through directly; absent args leave the global undefined', async () => {
|
||||
const { ctx, parent } = await setup()
|
||||
expect((await run(ctx, parent, script('return args * 2'), 21)).value).toBe(42)
|
||||
expect((await run(ctx, parent, script('return args === null'), null)).value).toBe(true)
|
||||
expect((await run(ctx, parent, script('return typeof args'))).value).toBe('undefined')
|
||||
})
|
||||
|
||||
it('a non-JSON return value fails loud as RESULT_UNSERIALIZABLE', async () => {
|
||||
const { ctx, parent } = await setup()
|
||||
const withDate = await run(ctx, parent, script('return { when: new Date(0) }'))
|
||||
expect(withDate.stopReason).toBe('error')
|
||||
expect(withDate.error).toContain('not plain JSON data')
|
||||
const withFn = await run(ctx, parent, script('return { fn: () => 1 }'))
|
||||
expect(withFn.error).toContain('not plain JSON data')
|
||||
})
|
||||
|
||||
it('kills a synchronous spin in the initial slice via the vm timeout', async () => {
|
||||
const { ctx, parent } = await setup({ config: { provider: 'stub', syncTimeoutMs: 50 } })
|
||||
const result = await run(ctx, parent, script('while (true) {}'))
|
||||
expect(result.stopReason).toBe('error')
|
||||
expect(result.error?.toLowerCase()).toContain('timed out')
|
||||
})
|
||||
})
|
||||
|
||||
describe('lifecycle: parse errors, cancellation, disposal', () => {
|
||||
it('start() throws synchronously for an unparseable script or invalid meta', async () => {
|
||||
const { ctx, parent } = await setup()
|
||||
expect(() => ctx.workflows.start({ script: 'const x = 1', parent })).toThrow(/must begin with/)
|
||||
expect(() => ctx.workflows.start({ script: script('return ((('), parent })).toThrow(/does not parse/)
|
||||
})
|
||||
|
||||
it('cancel() aborts in-flight children and settles the run cancelled', async () => {
|
||||
const { ctx, parent, provider } = await setup({ manual: true })
|
||||
const handle = ctx.workflows.start({ script: script("return await agent('long job')"), parent })
|
||||
await vi.waitFor(() => { expect(provider.runs.length).toBe(1) })
|
||||
handle.cancel('user stopped it')
|
||||
const result = await handle.result
|
||||
expect(result.stopReason).toBe('cancelled')
|
||||
expect(result.error).toContain('user stopped it')
|
||||
expect(provider.runs[0]!.disposed).toBe(true)
|
||||
await handle.dispose()
|
||||
})
|
||||
|
||||
it('an already-aborted request signal cancels before any child starts', async () => {
|
||||
const { ctx, parent, provider } = await setup({ manual: true })
|
||||
const controller = new AbortController()
|
||||
controller.abort()
|
||||
const handle = ctx.workflows.start({ script: script("return await agent('never')"), parent, signal: controller.signal })
|
||||
const result = await handle.result
|
||||
expect(result.stopReason).toBe('cancelled')
|
||||
expect(provider.runs.length).toBe(0)
|
||||
await handle.dispose()
|
||||
})
|
||||
|
||||
it('the signal aborting mid-run cancels like cancel()', async () => {
|
||||
const { ctx, parent, provider } = await setup({ manual: true })
|
||||
const controller = new AbortController()
|
||||
const handle = ctx.workflows.start({ script: script("return await agent('job')"), parent, signal: controller.signal })
|
||||
await vi.waitFor(() => { expect(provider.runs.length).toBe(1) })
|
||||
controller.abort()
|
||||
const result = await handle.result
|
||||
expect(result.stopReason).toBe('cancelled')
|
||||
await handle.dispose()
|
||||
})
|
||||
|
||||
it('reports a non-Error script throw (a thrown string) faithfully', async () => {
|
||||
const { ctx, parent } = await setup()
|
||||
const result = await run(ctx, parent, script("throw 'plain string failure'"))
|
||||
expect(result.stopReason).toBe('error')
|
||||
expect(result.error).toContain('plain string failure')
|
||||
})
|
||||
|
||||
it('a script Error surfaces its stack, carrying the script line numbers (lineOffset)', async () => {
|
||||
const { ctx, parent } = await setup()
|
||||
const result = await run(ctx, parent, script("throw new Error('with stack')"))
|
||||
expect(result.stopReason).toBe('error')
|
||||
// Line 1 is the blanked meta statement; the throw sits on line 2.
|
||||
expect(result.error).toContain('workflow:test-flow:2')
|
||||
})
|
||||
|
||||
it('an object throw with neither stack nor message stringifies', async () => {
|
||||
const { ctx, parent } = await setup()
|
||||
const result = await run(ctx, parent, script('throw { code: 42 }'))
|
||||
expect(result.stopReason).toBe('error')
|
||||
expect(result.error).toBe('[object Object]')
|
||||
})
|
||||
|
||||
it('falls back to the message for an Error whose stack was stripped', async () => {
|
||||
const { ctx, parent } = await setup()
|
||||
const result = await run(ctx, parent, script(`
|
||||
const e = new Error('stackless failure')
|
||||
e.stack = undefined
|
||||
throw e
|
||||
`))
|
||||
expect(result.stopReason).toBe('error')
|
||||
expect(result.error).toBe('stackless failure')
|
||||
})
|
||||
|
||||
it('a waiter resumed by a release RACING a cancel still dies at the post-acquire check', async () => {
|
||||
const { ctx, parent, provider } = await setup({ manual: true, config: { provider: 'stub', maxConcurrentAgents: 1 } })
|
||||
const handle = ctx.workflows.start({
|
||||
script: script("return await parallel([() => agent('a'), () => agent('b')])"),
|
||||
parent,
|
||||
})
|
||||
await vi.waitFor(() => { expect(provider.runs.length).toBe(1) })
|
||||
// Same synchronous block: the release resolves b's waiter, then the
|
||||
// cancel lands BEFORE b's continuation runs — b must not start a child.
|
||||
provider.runs[0]!.settle(text('a-done'))
|
||||
handle.cancel('raced')
|
||||
const result = await handle.result
|
||||
expect(result.stopReason).toBe('cancelled')
|
||||
expect(provider.runs.length).toBe(1)
|
||||
await handle.dispose()
|
||||
})
|
||||
|
||||
it('a dropped agent() promise cannot become an unhandled rejection when cancellation lands', async () => {
|
||||
const unhandled: unknown[] = []
|
||||
const onUnhandled = (reason: unknown): void => { unhandled.push(reason) }
|
||||
process.on('unhandledRejection', onUnhandled)
|
||||
try {
|
||||
const { ctx, parent, provider } = await setup({ manual: true })
|
||||
const handle = ctx.workflows.start({
|
||||
script: script(`
|
||||
agent('dropped, never awaited')
|
||||
return await agent('awaited')
|
||||
`),
|
||||
parent,
|
||||
})
|
||||
await vi.waitFor(() => { expect(provider.runs.length).toBe(2) })
|
||||
handle.cancel()
|
||||
await handle.result
|
||||
await handle.dispose()
|
||||
// Let any stray rejection reach the process hook before asserting.
|
||||
await new Promise(resolve => setTimeout(resolve, 20))
|
||||
expect(unhandled).toEqual([])
|
||||
} finally {
|
||||
process.off('unhandledRejection', onUnhandled)
|
||||
}
|
||||
})
|
||||
|
||||
it('dispose() abandons a stuck script after the grace instead of hanging (result stays pending)', async () => {
|
||||
const { ctx, parent } = await setup({ config: { provider: 'stub', disposeGraceMs: 30 } })
|
||||
const handle = ctx.workflows.start({
|
||||
// No hooks involved: an unsettleable await the engine cannot reject.
|
||||
script: script("await new Promise(() => {})\nreturn 'unreachable'"),
|
||||
parent,
|
||||
})
|
||||
const before = Date.now()
|
||||
await handle.dispose()
|
||||
expect(Date.now() - before).toBeLessThan(1000)
|
||||
const settled = await Promise.race([handle.result.then(() => 'settled'), Promise.resolve('pending')])
|
||||
expect(settled).toBe('pending')
|
||||
})
|
||||
|
||||
it('dispose() is idempotent and settles cleanly after a completed run', async () => {
|
||||
const { ctx, parent } = await setup()
|
||||
const handle = ctx.workflows.start({ script: script('return 1'), parent })
|
||||
await handle.result
|
||||
await handle.dispose()
|
||||
await handle.dispose()
|
||||
})
|
||||
|
||||
it('strays: children fired without await are aborted once the script settles', async () => {
|
||||
const { ctx, parent, provider } = await setup({ manual: true })
|
||||
const handle = ctx.workflows.start({
|
||||
script: script(`
|
||||
agent('stray')
|
||||
return 'done without awaiting'
|
||||
`),
|
||||
parent,
|
||||
})
|
||||
const result = await handle.result
|
||||
expect(result.stopReason).toBe('completed')
|
||||
await vi.waitFor(() => {
|
||||
expect(provider.runs.length).toBe(1)
|
||||
expect(provider.runs[0]!.disposed).toBe(true)
|
||||
})
|
||||
await handle.dispose()
|
||||
})
|
||||
})
|
||||
|
||||
describe('service surface', () => {
|
||||
it('run ids are unique per start; the run handle and event payloads hold SEPARATE meta clones', async () => {
|
||||
const { ctx, parent } = await setup()
|
||||
let eventMeta: WorkflowRunInfo | undefined
|
||||
ctx.on('workflow/start', (info) => { eventMeta = info })
|
||||
const first = ctx.workflows.start({ script: script('return 1'), parent })
|
||||
const second = ctx.workflows.start({ script: script('return 2'), parent })
|
||||
expect(first.id).not.toBe(second.id)
|
||||
// Mutating a listener's snapshot cannot corrupt the holder's view.
|
||||
eventMeta!.meta.name = 'corrupted'
|
||||
expect(second.meta.name).toBe('test-flow')
|
||||
await Promise.all([first.result, second.result])
|
||||
await first.dispose()
|
||||
await second.dispose()
|
||||
})
|
||||
|
||||
it('unregisters ctx.workflows when the engine fiber is disposed (HMR safety)', async () => {
|
||||
const ctx = new Context()
|
||||
await ctx.plugin(SubagentService)
|
||||
const fiber = await ctx.plugin(VmWorkflowEngine, {})
|
||||
expect(ctx.get('workflows')).toBeDefined()
|
||||
await fiber.dispose()
|
||||
expect(ctx.get('workflows')).toBeUndefined()
|
||||
})
|
||||
|
||||
it('has the class-plugin export shape (default = the engine service class)', () => {
|
||||
expect(vmEngineModule.default).toBe(VmWorkflowEngine)
|
||||
const loader = Object.create(Loader.prototype) as Loader
|
||||
const unwrapped: unknown = loader.unwrapExports(vmEngineModule)
|
||||
expect(unwrapped).toBe(VmWorkflowEngine)
|
||||
})
|
||||
})
|
||||
})
|
||||
131
packages/workflow/workflow-vm/tests/workflow.e2e.ts
Normal file
131
packages/workflow/workflow-vm/tests/workflow.e2e.ts
Normal file
@@ -0,0 +1,131 @@
|
||||
import { afterEach, describe, expect, it } from 'vitest'
|
||||
import { Context } from 'cordis'
|
||||
import LlmService from '@deepseek-ai/dsh-llm'
|
||||
import SessionStore from '@deepseek-ai/dsh-session'
|
||||
import SystemPrompt from '@deepseek-ai/dsh-system-prompt'
|
||||
import ToolRegistry from '@deepseek-ai/dsh-tools'
|
||||
import AgentRegistry, { AgentId } from '@deepseek-ai/dsh-agent'
|
||||
import AgentLoop from '@deepseek-ai/dsh-agent-loop'
|
||||
import * as LlmDeepSeek from '@deepseek-ai/dsh-llm-deepseek'
|
||||
import SubagentService from '@deepseek-ai/dsh-subagent'
|
||||
import * as Spawn from '@deepseek-ai/dsh-subagent-spawn'
|
||||
import * as ToolWorkflow from '@deepseek-ai/dsh-tool-workflow'
|
||||
import { CallId } from '@deepseek-ai/dsh-llm'
|
||||
import VmWorkflowEngine from '../src/index.ts'
|
||||
|
||||
/**
|
||||
* With-key e2e for the workflow engine: a REAL script drives REAL spawn
|
||||
* children against the live DeepSeek API — one plain child and one schema'd
|
||||
* child through the real structured-output runtime — and the run's value,
|
||||
* events, and child sessions are asserted from the outside (never the
|
||||
* script's self-report alone). Key-gated (self-skips without
|
||||
* DEEPSEEK_API_KEY).
|
||||
*/
|
||||
|
||||
let ctx: Context | undefined
|
||||
|
||||
afterEach(async () => {
|
||||
await ctx?.fiber.dispose()
|
||||
ctx = undefined
|
||||
})
|
||||
|
||||
async function harness(): Promise<Context> {
|
||||
const built = new Context()
|
||||
await built.plugin(LlmService)
|
||||
await built.plugin(SessionStore)
|
||||
await built.plugin(SystemPrompt)
|
||||
await built.plugin(ToolRegistry)
|
||||
await built.plugin(AgentRegistry)
|
||||
await built.plugin(AgentLoop, { agents: [] })
|
||||
await built.plugin(LlmDeepSeek, { models: ['deepseek-v4-flash'] })
|
||||
await built.plugin(SubagentService)
|
||||
await built.plugin(Spawn, { providerName: 'spawn', structuredNudgeRetries: 1 })
|
||||
await built.plugin(VmWorkflowEngine, { provider: 'spawn' })
|
||||
await built.plugin(ToolWorkflow, {})
|
||||
return built
|
||||
}
|
||||
|
||||
const SCRIPT = `export const meta = {
|
||||
name: 'e2e-arithmetic',
|
||||
description: 'two real children: one prose, one structured',
|
||||
phases: [{ title: 'Ask' }, { title: 'Judge' }],
|
||||
}
|
||||
phase('Ask')
|
||||
log('asking the prose child')
|
||||
const prose = await agent('Reply with exactly one short sentence: what is 2 + 2?')
|
||||
phase('Judge')
|
||||
const judged = await agent(
|
||||
'Here is an answer to the question "what is 2+2": ' + prose
|
||||
+ ' — report whether it contains the number 4 and your confidence between 0 and 1.',
|
||||
{ schema: { type: 'object', properties: { containsFour: { type: 'boolean' }, confidence: { type: 'number' } }, required: ['containsFour'] } },
|
||||
)
|
||||
return { prose, containsFour: judged === null ? null : judged.containsFour }`
|
||||
|
||||
describe.skipIf(!process.env.DEEPSEEK_API_KEY)('workflow engine with-key e2e', () => {
|
||||
it('runs a two-phase script over real children, one through the structured runtime', async () => {
|
||||
ctx = await harness()
|
||||
const parentHandle = ctx.agents.create({
|
||||
agentId: AgentId('wf-e2e-parent'),
|
||||
sessionId: 'wf-e2e-session' as never,
|
||||
agentOptions: { model: 'deepseek-v4-flash' },
|
||||
})
|
||||
|
||||
const events: string[] = []
|
||||
const childIds: string[] = []
|
||||
for (const name of ['workflow/start', 'workflow/phase', 'workflow/log', 'workflow/agent-start', 'workflow/agent-end', 'workflow/end'] as const) {
|
||||
ctx.on(name, (...payload: unknown[]) => {
|
||||
events.push(name)
|
||||
if (name === 'workflow/agent-start') childIds.push((payload[1] as { childId: string }).childId)
|
||||
})
|
||||
}
|
||||
|
||||
const run = ctx.workflows.start({ script: SCRIPT, parent: parentHandle.agent })
|
||||
const result = await run.result
|
||||
await run.dispose()
|
||||
|
||||
expect(result.stopReason).toBe('completed')
|
||||
expect(result.agentsStarted).toBe(2)
|
||||
const value = result.value as { prose: string; containsFour: boolean | null }
|
||||
// World checks: the prose child really answered (a real completion), and
|
||||
// the structured child judged it against the REAL schema-forced tool.
|
||||
expect(value.prose.length).toBeGreaterThan(0)
|
||||
expect(value.containsFour).toBe(true)
|
||||
|
||||
expect(events[0]).toBe('workflow/start')
|
||||
expect(events.at(-1)).toBe('workflow/end')
|
||||
expect(events.filter(name => name === 'workflow/phase').length).toBe(2)
|
||||
expect(events.filter(name => name === 'workflow/agent-start').length).toBe(2)
|
||||
expect(childIds.length).toBe(2)
|
||||
// The children were disposed to quiescence after collection.
|
||||
for (const childId of childIds) {
|
||||
expect(ctx.agents.get(AgentId(childId))).toBeUndefined()
|
||||
}
|
||||
await parentHandle.dispose()
|
||||
}, 240_000)
|
||||
|
||||
it('the workflow TOOL runs the same path through the real registry pipeline', async () => {
|
||||
ctx = await harness()
|
||||
const parentHandle = ctx.agents.create({
|
||||
agentId: AgentId('wf-e2e-tool-parent'),
|
||||
sessionId: 'wf-e2e-tool-session' as never,
|
||||
agentOptions: { model: 'deepseek-v4-flash' },
|
||||
})
|
||||
|
||||
const result = await ctx.tools.execute({
|
||||
callId: CallId('wf-e2e-call'),
|
||||
name: 'workflow',
|
||||
arguments: {
|
||||
script: `export const meta = { name: 'e2e-tool', description: 'one real child via the tool' }
|
||||
const answer = await agent('Reply with exactly one word: the capital of France.')
|
||||
return { answer }`,
|
||||
},
|
||||
agent: parentHandle.agent,
|
||||
})
|
||||
|
||||
expect(result.isError).toBe(false)
|
||||
const text = (result.content[0] as { text: string }).text
|
||||
expect(text).toContain('workflow "e2e-tool" completed (1 agent)')
|
||||
expect(text.toLowerCase()).toContain('paris')
|
||||
await parentHandle.dispose()
|
||||
}, 240_000)
|
||||
})
|
||||
Reference in New Issue
Block a user