fix(llm-mock-server): harden fault boundaries
This commit is contained in:
@@ -140,8 +140,11 @@ describe('bounded retry through the real DeepSeek HTTP/SSE adapter', () => {
|
||||
expect(finalAssistantText(agent)).toBe('recovered response')
|
||||
})
|
||||
|
||||
it('treats a wire-valid content-less completion as success without retrying', async () => {
|
||||
const server = await start(['empty', 'success'], { apiKey: 'mock-key' })
|
||||
it('retries a wire-valid content-less completion without committing an empty message', async () => {
|
||||
const server = await start(['empty', 'success'], {
|
||||
apiKey: 'mock-key',
|
||||
successText: 'recovered from empty',
|
||||
})
|
||||
context = await harness(server.baseURL)
|
||||
const agent = context.agentLoop.create(SessionId('wire-empty'), {
|
||||
provider: 'deepseek',
|
||||
@@ -150,16 +153,17 @@ describe('bounded retry through the real DeepSeek HTTP/SSE adapter', () => {
|
||||
|
||||
await sendAndWait(context, agent)
|
||||
|
||||
expect(server.requests).toHaveLength(1)
|
||||
expect(agent.session.events.some(event => event.type === 'llm/retry')).toBe(false)
|
||||
expect(agent.session.events.find(event => event.type === 'assistant/message')).toMatchObject({
|
||||
data: { turn: 1, step: 1, content: [] },
|
||||
})
|
||||
expect(server.requests).toHaveLength(2)
|
||||
expect(server.requests[0]?.body).toEqual(server.requests[1]?.body)
|
||||
expect(agent.session.events.filter(event => event.type === 'llm/retry').map(event => event.data.failure.code))
|
||||
.toEqual(['EMPTY_RESPONSE'])
|
||||
expect(agent.session.events.filter(event => event.type === 'assistant/message').map(event => event.data.step))
|
||||
.toEqual([2])
|
||||
expect(agent.session.events.at(-1)).toMatchObject({
|
||||
type: 'turn/end',
|
||||
data: { reason: { kind: 'completed' } },
|
||||
})
|
||||
expect(finalAssistantText(agent)).toBeUndefined()
|
||||
expect(finalAssistantText(agent)).toBe('recovered from empty')
|
||||
})
|
||||
|
||||
it('exposes a clean partial EOF as non-default-retryable STREAM_CLOSED', async () => {
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
A scriptable OpenAI-compatible HTTP/SSE server for exercising real LLM adapters, the agent loop, and recovery policy without a provider key. It accepts `POST /chat/completions` and `POST /v1/chat/completions`; each accepted request consumes one configured behavior in arrival order. Invalid methods, paths, bearer tokens, and JSON do not consume the script.
|
||||
|
||||
The library entry exports `startMockLlmServer(options)`, behavior and telemetry types, the default random stress weights, and a running handle with the bound `baseURL`, generated or configured `randomSeed`, captured requests, and idempotent `close()`. Closing force-terminates stalled connections.
|
||||
The library entry exports `startMockLlmServer(options)`, behavior and telemetry types, the default random stress weights, the accepted Node timer bound, and a running handle with the bound `baseURL`, generated or configured `randomSeed`, captured requests, and idempotent `close()`. Closing force-terminates stalled connections.
|
||||
|
||||
## Standalone use
|
||||
|
||||
@@ -67,7 +67,7 @@ When random weights include `stall`, configure the client under test with a shor
|
||||
|
||||
## Timing and content controls
|
||||
|
||||
The CLI exposes `--success-text`, `--partial-text`, `--reasoning-text`, `--chunk-size`, `--chunk-delay-ms`, `--disconnect-delay-ms`, `--retry-after-ms`, `--request-id`, `--tool-name`, and `--tool-arguments`. The library accepts the same camel-case options. An optional exact `apiKey` validates `Authorization: Bearer <token>`; omission accepts any token.
|
||||
The CLI exposes `--success-text`, `--partial-text`, `--reasoning-text`, `--chunk-size`, `--chunk-delay-ms`, `--disconnect-delay-ms`, `--retry-after-ms`, `--request-id`, `--tool-name`, and `--tool-arguments`. Millisecond delays are bounded integers within Node's timer range; `retryAfterMs` must also be positive. The library accepts the same camel-case options. An optional exact `apiKey` validates `Authorization: Bearer <token>`; omission accepts any token.
|
||||
|
||||
## Model Experience
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
* @module @deepseek-ai/dsh-llm-mock-server/cli
|
||||
*/
|
||||
|
||||
import { MOCK_LLM_BEHAVIORS } from './index.ts'
|
||||
import { MAX_MOCK_LLM_TIMER_DELAY_MS, MOCK_LLM_BEHAVIORS } from './index.ts'
|
||||
import type {
|
||||
ConcreteMockLlmBehavior,
|
||||
MockLlmBehavior,
|
||||
@@ -18,7 +18,7 @@ export const CONNECTION_REFUSED_BEHAVIOR = 'connection_refused'
|
||||
export interface MockLlmCliConfig {
|
||||
/** Server options after removing the lifecycle-only `connection_refused` entry. */
|
||||
readonly server: MockLlmServerOptions
|
||||
/** Delay before binding the model port; zero starts immediately. */
|
||||
/** Delay before binding the model port; an integer from zero through the Node timer maximum. */
|
||||
readonly listenDelayMs: number
|
||||
/** Whether the original sequence requested a true pre-listen refusal phase. */
|
||||
readonly startsUnavailable: boolean
|
||||
@@ -77,6 +77,14 @@ function numberValue(option: string, value: string): number {
|
||||
return parsed
|
||||
}
|
||||
|
||||
function boundedIntegerValue(option: string, value: string, min: number, max: number): number {
|
||||
const parsed = numberValue(option, value)
|
||||
if (!Number.isInteger(parsed) || parsed < min || parsed > max) {
|
||||
throw new Error(`dsh-llm-mock-server: ${option} must be an integer between ${min} and ${max}`)
|
||||
}
|
||||
return parsed
|
||||
}
|
||||
|
||||
function parseSequence(raw: string): { startsUnavailable: boolean; sequence: MockLlmBehavior[] } {
|
||||
const entries = raw.split(',').map(entry => entry.trim())
|
||||
if (entries.some(entry => entry.length === 0)) {
|
||||
@@ -154,7 +162,9 @@ export function parseMockLlmCliArgs(argv: readonly string[]): MockLlmCliParseRes
|
||||
case '--host': host = value; break
|
||||
case '--port': port = numberValue(option, value); break
|
||||
case '--api-key': apiKey = value; break
|
||||
case '--listen-delay-ms': listenDelayMs = numberValue(option, value); break
|
||||
case '--listen-delay-ms':
|
||||
listenDelayMs = boundedIntegerValue(option, value, 0, MAX_MOCK_LLM_TIMER_DELAY_MS)
|
||||
break
|
||||
case '--seed': randomSeed = numberValue(option, value); break
|
||||
case '--random-weights': randomWeights = parseRandomWeights(value); break
|
||||
case '--success-text': successText = value; break
|
||||
|
||||
@@ -9,7 +9,7 @@
|
||||
import { createServer } from 'node:http'
|
||||
import type { IncomingHttpHeaders, IncomingMessage, ServerResponse } from 'node:http'
|
||||
import { randomBytes } from 'node:crypto'
|
||||
import type { AddressInfo } from 'node:net'
|
||||
import { isIP, type AddressInfo } from 'node:net'
|
||||
import { setTimeout as delay } from 'node:timers/promises'
|
||||
|
||||
/** Request-scoped behaviors accepted by {@link startMockLlmServer}. */
|
||||
@@ -69,6 +69,9 @@ export const DEFAULT_MOCK_LLM_RANDOM_WEIGHTS: Readonly<MockLlmRandomWeights> = O
|
||||
malformed_json: 1,
|
||||
})
|
||||
|
||||
/** Largest millisecond delay accepted by Node timers without truncation. */
|
||||
export const MAX_MOCK_LLM_TIMER_DELAY_MS = 2_147_483_647
|
||||
|
||||
/** How one accepted request ended at the mock boundary. */
|
||||
export type MockLlmRequestOutcome = 'completed' | 'reset' | 'stalled' | 'client_closed' | 'server_error'
|
||||
|
||||
@@ -186,7 +189,6 @@ interface ResolvedOptions {
|
||||
readonly onEvent?: (event: MockLlmServerEvent) => void
|
||||
}
|
||||
|
||||
const MAX_TIMER_DELAY_MS = 2_147_483_647
|
||||
const DEFAULT_SUCCESS_TEXT = 'mock response recovered'
|
||||
const DEFAULT_PARTIAL_TEXT = 'discarded partial response'
|
||||
const DEFAULT_REASONING_TEXT = 'mock reasoning'
|
||||
@@ -203,14 +205,24 @@ function resolveOptions(options: MockLlmServerOptions): ResolvedOptions {
|
||||
const host = options.host ?? '127.0.0.1'
|
||||
const port = boundedInteger('port', options.port ?? 0, 0, 65_535)
|
||||
const chunkSize = boundedInteger('chunkSize', options.chunkSize ?? 8, 1, Number.MAX_SAFE_INTEGER)
|
||||
const chunkDelayMs = boundedInteger('chunkDelayMs', options.chunkDelayMs ?? 25, 0, MAX_TIMER_DELAY_MS)
|
||||
const chunkDelayMs = boundedInteger(
|
||||
'chunkDelayMs',
|
||||
options.chunkDelayMs ?? 25,
|
||||
0,
|
||||
MAX_MOCK_LLM_TIMER_DELAY_MS,
|
||||
)
|
||||
const disconnectDelayMs = boundedInteger(
|
||||
'disconnectDelayMs',
|
||||
options.disconnectDelayMs ?? 10,
|
||||
0,
|
||||
MAX_TIMER_DELAY_MS,
|
||||
MAX_MOCK_LLM_TIMER_DELAY_MS,
|
||||
)
|
||||
const retryAfterMs = boundedInteger(
|
||||
'retryAfterMs',
|
||||
options.retryAfterMs ?? 1_000,
|
||||
1,
|
||||
MAX_MOCK_LLM_TIMER_DELAY_MS,
|
||||
)
|
||||
const retryAfterMs = boundedInteger('retryAfterMs', options.retryAfterMs ?? 1_000, 1, MAX_TIMER_DELAY_MS)
|
||||
const randomSeed = boundedInteger(
|
||||
'randomSeed',
|
||||
options.randomSeed ?? randomBytes(4).readUInt32LE(0),
|
||||
@@ -285,8 +297,9 @@ function emit(options: ResolvedOptions, event: MockLlmServerEvent): void {
|
||||
}
|
||||
|
||||
async function readJsonBody(request: IncomingMessage): Promise<unknown> {
|
||||
let body = ''
|
||||
for await (const chunk of request) body += Buffer.from(chunk).toString('utf8')
|
||||
const chunks: Buffer[] = []
|
||||
for await (const chunk of request) chunks.push(Buffer.from(chunk as Uint8Array))
|
||||
const body = Buffer.concat(chunks).toString('utf8')
|
||||
return body.length === 0 ? undefined : JSON.parse(body)
|
||||
}
|
||||
|
||||
@@ -320,6 +333,7 @@ function finishRecord(
|
||||
record: MockLlmRequestRecord,
|
||||
outcome: MockLlmRequestOutcome,
|
||||
): void {
|
||||
if (record.outcome !== undefined) return
|
||||
record.outcome = outcome
|
||||
emit(options, {
|
||||
type: 'result',
|
||||
@@ -713,8 +727,9 @@ export async function startMockLlmServer(options: MockLlmServerOptions): Promise
|
||||
})
|
||||
|
||||
const address = server.address() as AddressInfo
|
||||
const advertisedHost = isIP(resolved.host) === 6 ? `[${resolved.host}]` : resolved.host
|
||||
return {
|
||||
baseURL: `http://${resolved.host}:${address.port}`,
|
||||
baseURL: `http://${advertisedHost}:${address.port}`,
|
||||
port: address.port,
|
||||
randomSeed: resolved.randomSeed,
|
||||
requests,
|
||||
|
||||
@@ -110,6 +110,9 @@ describe('mock LLM server CLI parser', () => {
|
||||
[['--sequence', 'unknown'], /unknown behavior/],
|
||||
[['--sequence', 'connection_refused,success', '--port', '0'], /nonzero/],
|
||||
[['--sequence', 'success', '--listen-delay-ms', '5'], /requires connection_refused/],
|
||||
[['--sequence', 'connection_refused,success', '--listen-delay-ms', '-1'], /integer between 0 and 2147483647/],
|
||||
[['--sequence', 'connection_refused,success', '--listen-delay-ms', '1.5'], /integer between 0 and 2147483647/],
|
||||
[['--sequence', 'connection_refused,success', '--listen-delay-ms', '2147483648'], /integer between 0 and 2147483647/],
|
||||
[['--sequence', 'success', '--seed', '1'], /require random/],
|
||||
[['--sequence', 'random', '--random-weights', 'success'], /expects behavior=weight/],
|
||||
[['--sequence', 'random', '--random-weights', 'random=1'], /concrete behavior/],
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { request } from 'node:http'
|
||||
import { afterEach, describe, expect, it } from 'vitest'
|
||||
import type { MockLlmBehavior, MockLlmServer, MockLlmServerEvent } from '../src/index.ts'
|
||||
import { startMockLlmServer } from '../src/index.ts'
|
||||
@@ -32,6 +33,22 @@ function chat(
|
||||
})
|
||||
}
|
||||
|
||||
function rawChat(server: MockLlmServer, chunks: readonly Buffer[]): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const outgoing = request(`${server.baseURL}/v1/chat/completions`, {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
}, (response) => {
|
||||
response.once('error', reject)
|
||||
response.once('end', resolve)
|
||||
response.resume()
|
||||
})
|
||||
outgoing.once('error', reject)
|
||||
for (const chunk of chunks) outgoing.write(chunk)
|
||||
outgoing.end()
|
||||
})
|
||||
}
|
||||
|
||||
describe('mock LLM server wire behaviors', () => {
|
||||
it('streams a complete text response and captures the request', async () => {
|
||||
const events: MockLlmServerEvent[] = []
|
||||
@@ -151,10 +168,12 @@ describe('mock LLM server wire behaviors', () => {
|
||||
['stream_disconnect', 100] as const,
|
||||
['partial_disconnect', 100] as const,
|
||||
])('records a client that closes during %s', async (behavior, delayMs) => {
|
||||
const events: MockLlmServerEvent[] = []
|
||||
const server = await start([behavior], {
|
||||
chunkDelayMs: delayMs,
|
||||
disconnectDelayMs: delayMs,
|
||||
chunkSize: 1,
|
||||
onEvent: (event) => { events.push(event) },
|
||||
})
|
||||
const controller = new AbortController()
|
||||
const response = await chat(server, { signal: controller.signal })
|
||||
@@ -163,6 +182,30 @@ describe('mock LLM server wire behaviors', () => {
|
||||
await new Promise((resolve) => { setTimeout(resolve, 5) })
|
||||
|
||||
expect(server.requests[0]).toMatchObject({ behavior, outcome: 'client_closed' })
|
||||
expect(events.filter(event => event.type === 'result')).toEqual([
|
||||
expect.objectContaining({ behavior, outcome: 'client_closed' }),
|
||||
])
|
||||
})
|
||||
|
||||
it('preserves UTF-8 code points split across request chunks', async () => {
|
||||
const server = await start(['success'])
|
||||
const encoded = Buffer.from(JSON.stringify({ messages: [{ role: 'user', content: '你好' }] }))
|
||||
const characterOffset = encoded.indexOf(Buffer.from('你'))
|
||||
expect(characterOffset).toBeGreaterThanOrEqual(0)
|
||||
|
||||
await rawChat(server, [
|
||||
encoded.subarray(0, characterOffset + 1),
|
||||
encoded.subarray(characterOffset + 1),
|
||||
])
|
||||
|
||||
expect(server.requests[0]?.body).toEqual({ messages: [{ role: 'user', content: '你好' }] })
|
||||
})
|
||||
|
||||
it('formats an IPv6 listener as a valid base URL', async () => {
|
||||
const server = await start(['success'], { host: '::1' })
|
||||
|
||||
expect(server.baseURL).toMatch(/^http:\/\/\[::1\]:\d+$/)
|
||||
expect((await chat(server)).status).toBe(200)
|
||||
})
|
||||
|
||||
it('emits reasoning, tool calls, max-token finishes, slow chunks, and a wrong content type', async () => {
|
||||
|
||||
Reference in New Issue
Block a user