test: address Codex review of property tests (PR 3)
- llm: generator now emits finish chunks (the finish-defaults property was vacuously green); add a property asserting streaming and one-shot assembly agree on usage and finish - agent-loop: assert the synchronous burst batches into exactly one turn; add a mixed-schedule property (send/settle interleavings); recordStatus returns its disposer; per-run timeouts so a hang loses no seed - session: randomize the noise/message interleaving (was a fixed alternation) - tools: exclude non-finite doubles from generated numeric args (JSON-real)
This commit is contained in:
@@ -58,13 +58,14 @@ function nextIdle(ctx: Context, agent: LoopAgent): Promise<void> {
|
||||
})
|
||||
}
|
||||
|
||||
/** Record every status transition for the legal-machine assertion. */
|
||||
function recordStatus(ctx: Context, agent: LoopAgent): string[] {
|
||||
/** Record every status transition for the legal-machine assertion. Returns
|
||||
* the seen list plus a disposer for the listener (per the registry convention). */
|
||||
function recordStatus(ctx: Context, agent: LoopAgent): { seen: string[]; dispose: () => void } {
|
||||
const seen: string[] = []
|
||||
ctx.on('agent/status', (subject, status) => {
|
||||
const dispose = ctx.on('agent/status', (subject, status) => {
|
||||
if (subject === agent) seen.push(status)
|
||||
})
|
||||
return seen
|
||||
return { seen, dispose }
|
||||
}
|
||||
|
||||
function userMessageTexts(agent: LoopAgent): string[] {
|
||||
@@ -95,7 +96,7 @@ describe('agent loop scheduling properties', () => {
|
||||
const ctx = await harness()
|
||||
try {
|
||||
const agent = ctx.agentLoop.create('a', { model: 'mock' })
|
||||
const trace = recordStatus(ctx, agent)
|
||||
const { seen: trace } = recordStatus(ctx, agent)
|
||||
const idle = nextIdle(ctx, agent)
|
||||
// Send all in one synchronous tick: they queue before the loop wakes.
|
||||
for (const text of texts) agent.send([{ type: 'text', text }])
|
||||
@@ -103,15 +104,14 @@ describe('agent loop scheduling properties', () => {
|
||||
|
||||
// No message lost: every send appears as a user/message, in order.
|
||||
expect(userMessageTexts(agent)).toEqual(texts)
|
||||
// Turn numbers strictly increase.
|
||||
const turns = turnNumbers(agent)
|
||||
for (let i = 1; i < turns.length; i++) expect(turns[i]!).toBeGreaterThan(turns[i - 1]!)
|
||||
// A synchronous burst batches into exactly one turn.
|
||||
expect(turnNumbers(agent)).toEqual([1])
|
||||
assertLegalStatusTrace(trace)
|
||||
} finally {
|
||||
await ctx.fiber.dispose()
|
||||
}
|
||||
},
|
||||
), { numRuns: 25 })
|
||||
), { numRuns: 25, timeout: 2000 })
|
||||
})
|
||||
|
||||
it('sequential sends each get their own turn with increasing numbers', async () => {
|
||||
@@ -133,6 +133,44 @@ describe('agent loop scheduling properties', () => {
|
||||
await ctx.fiber.dispose()
|
||||
}
|
||||
},
|
||||
), { numRuns: 20 })
|
||||
), { numRuns: 20, timeout: 2000 })
|
||||
})
|
||||
|
||||
it('mixed schedule (send, optionally settle) loses no message and orders turns', async () => {
|
||||
// Each step is a (text, settle?) pair: settle=true awaits idle before the
|
||||
// next send (own turn); settle=false sends in the same tick (batches).
|
||||
const stepArb = fc.record({ text: fc.string({ minLength: 1 }), settle: fc.boolean() })
|
||||
await fc.assert(fc.asyncProperty(
|
||||
fc.array(stepArb, { minLength: 1, maxLength: 6 }),
|
||||
async (steps) => {
|
||||
const ctx = await harness()
|
||||
try {
|
||||
const agent = ctx.agentLoop.create('a', { model: 'mock' })
|
||||
// Capture an idle waiter before EACH send; the last one is guaranteed
|
||||
// to resolve because the final send always triggers (or joins) a turn
|
||||
// that ends idle. Awaiting an already-resolved waiter is a no-op, so a
|
||||
// trailing settle step can't cause a hang.
|
||||
let lastIdle: Promise<void> | undefined
|
||||
for (const step of steps) {
|
||||
const idle = nextIdle(ctx, agent)
|
||||
lastIdle = idle
|
||||
agent.send([{ type: 'text', text: step.text }])
|
||||
if (step.settle) await idle
|
||||
}
|
||||
await lastIdle
|
||||
|
||||
// No message lost or reordered, regardless of batching.
|
||||
expect(userMessageTexts(agent)).toEqual(steps.map(s => s.text))
|
||||
// Turn numbers are a strictly increasing 1..N prefix (N = turn count).
|
||||
const turns = turnNumbers(agent)
|
||||
expect(turns).toEqual(turns.map((_, i) => i + 1))
|
||||
// Every message landed in some turn; turns never exceed messages.
|
||||
expect(turns.length).toBeLessThanOrEqual(steps.length)
|
||||
expect(turns.length).toBeGreaterThanOrEqual(1)
|
||||
} finally {
|
||||
await ctx.fiber.dispose()
|
||||
}
|
||||
},
|
||||
), { numRuns: 25, timeout: 3000 })
|
||||
})
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user