fix(llm-mock-server): harden fault boundaries

This commit is contained in:
Yichen Jiang
2026-07-25 22:45:58 +08:00
parent 5446714177
commit badf7d1c63
10 changed files with 109 additions and 31 deletions

View File

@@ -140,8 +140,11 @@ describe('bounded retry through the real DeepSeek HTTP/SSE adapter', () => {
expect(finalAssistantText(agent)).toBe('recovered response')
})
it('treats a wire-valid content-less completion as success without retrying', async () => {
const server = await start(['empty', 'success'], { apiKey: 'mock-key' })
it('retries a wire-valid content-less completion without committing an empty message', async () => {
const server = await start(['empty', 'success'], {
apiKey: 'mock-key',
successText: 'recovered from empty',
})
context = await harness(server.baseURL)
const agent = context.agentLoop.create(SessionId('wire-empty'), {
provider: 'deepseek',
@@ -150,16 +153,17 @@ describe('bounded retry through the real DeepSeek HTTP/SSE adapter', () => {
await sendAndWait(context, agent)
expect(server.requests).toHaveLength(1)
expect(agent.session.events.some(event => event.type === 'llm/retry')).toBe(false)
expect(agent.session.events.find(event => event.type === 'assistant/message')).toMatchObject({
data: { turn: 1, step: 1, content: [] },
})
expect(server.requests).toHaveLength(2)
expect(server.requests[0]?.body).toEqual(server.requests[1]?.body)
expect(agent.session.events.filter(event => event.type === 'llm/retry').map(event => event.data.failure.code))
.toEqual(['EMPTY_RESPONSE'])
expect(agent.session.events.filter(event => event.type === 'assistant/message').map(event => event.data.step))
.toEqual([2])
expect(agent.session.events.at(-1)).toMatchObject({
type: 'turn/end',
data: { reason: { kind: 'completed' } },
})
expect(finalAssistantText(agent)).toBeUndefined()
expect(finalAssistantText(agent)).toBe('recovered from empty')
})
it('exposes a clean partial EOF as non-default-retryable STREAM_CLOSED', async () => {

View File

@@ -2,7 +2,7 @@
A scriptable OpenAI-compatible HTTP/SSE server for exercising real LLM adapters, the agent loop, and recovery policy without a provider key. It accepts `POST /chat/completions` and `POST /v1/chat/completions`; each accepted request consumes one configured behavior in arrival order. Invalid methods, paths, bearer tokens, and JSON do not consume the script.
The library entry exports `startMockLlmServer(options)`, behavior and telemetry types, the default random stress weights, and a running handle with the bound `baseURL`, generated or configured `randomSeed`, captured requests, and idempotent `close()`. Closing force-terminates stalled connections.
The library entry exports `startMockLlmServer(options)`, behavior and telemetry types, the default random stress weights, the accepted Node timer bound, and a running handle with the bound `baseURL`, generated or configured `randomSeed`, captured requests, and idempotent `close()`. Closing force-terminates stalled connections.
## Standalone use
@@ -67,7 +67,7 @@ When random weights include `stall`, configure the client under test with a shor
## Timing and content controls
The CLI exposes `--success-text`, `--partial-text`, `--reasoning-text`, `--chunk-size`, `--chunk-delay-ms`, `--disconnect-delay-ms`, `--retry-after-ms`, `--request-id`, `--tool-name`, and `--tool-arguments`. The library accepts the same camel-case options. An optional exact `apiKey` validates `Authorization: Bearer <token>`; omission accepts any token.
The CLI exposes `--success-text`, `--partial-text`, `--reasoning-text`, `--chunk-size`, `--chunk-delay-ms`, `--disconnect-delay-ms`, `--retry-after-ms`, `--request-id`, `--tool-name`, and `--tool-arguments`. Millisecond delays are bounded integers within Node's timer range; `retryAfterMs` must also be positive. The library accepts the same camel-case options. An optional exact `apiKey` validates `Authorization: Bearer <token>`; omission accepts any token.
## Model Experience

View File

@@ -3,7 +3,7 @@
* @module @deepseek-ai/dsh-llm-mock-server/cli
*/
import { MOCK_LLM_BEHAVIORS } from './index.ts'
import { MAX_MOCK_LLM_TIMER_DELAY_MS, MOCK_LLM_BEHAVIORS } from './index.ts'
import type {
ConcreteMockLlmBehavior,
MockLlmBehavior,
@@ -18,7 +18,7 @@ export const CONNECTION_REFUSED_BEHAVIOR = 'connection_refused'
export interface MockLlmCliConfig {
/** Server options after removing the lifecycle-only `connection_refused` entry. */
readonly server: MockLlmServerOptions
/** Delay before binding the model port; zero starts immediately. */
/** Delay before binding the model port; an integer from zero through the Node timer maximum. */
readonly listenDelayMs: number
/** Whether the original sequence requested a true pre-listen refusal phase. */
readonly startsUnavailable: boolean
@@ -77,6 +77,14 @@ function numberValue(option: string, value: string): number {
return parsed
}
function boundedIntegerValue(option: string, value: string, min: number, max: number): number {
const parsed = numberValue(option, value)
if (!Number.isInteger(parsed) || parsed < min || parsed > max) {
throw new Error(`dsh-llm-mock-server: ${option} must be an integer between ${min} and ${max}`)
}
return parsed
}
function parseSequence(raw: string): { startsUnavailable: boolean; sequence: MockLlmBehavior[] } {
const entries = raw.split(',').map(entry => entry.trim())
if (entries.some(entry => entry.length === 0)) {
@@ -154,7 +162,9 @@ export function parseMockLlmCliArgs(argv: readonly string[]): MockLlmCliParseRes
case '--host': host = value; break
case '--port': port = numberValue(option, value); break
case '--api-key': apiKey = value; break
case '--listen-delay-ms': listenDelayMs = numberValue(option, value); break
case '--listen-delay-ms':
listenDelayMs = boundedIntegerValue(option, value, 0, MAX_MOCK_LLM_TIMER_DELAY_MS)
break
case '--seed': randomSeed = numberValue(option, value); break
case '--random-weights': randomWeights = parseRandomWeights(value); break
case '--success-text': successText = value; break

View File

@@ -9,7 +9,7 @@
import { createServer } from 'node:http'
import type { IncomingHttpHeaders, IncomingMessage, ServerResponse } from 'node:http'
import { randomBytes } from 'node:crypto'
import type { AddressInfo } from 'node:net'
import { isIP, type AddressInfo } from 'node:net'
import { setTimeout as delay } from 'node:timers/promises'
/** Request-scoped behaviors accepted by {@link startMockLlmServer}. */
@@ -69,6 +69,9 @@ export const DEFAULT_MOCK_LLM_RANDOM_WEIGHTS: Readonly<MockLlmRandomWeights> = O
malformed_json: 1,
})
/** Largest millisecond delay accepted by Node timers without truncation. */
export const MAX_MOCK_LLM_TIMER_DELAY_MS = 2_147_483_647
/** How one accepted request ended at the mock boundary. */
export type MockLlmRequestOutcome = 'completed' | 'reset' | 'stalled' | 'client_closed' | 'server_error'
@@ -186,7 +189,6 @@ interface ResolvedOptions {
readonly onEvent?: (event: MockLlmServerEvent) => void
}
const MAX_TIMER_DELAY_MS = 2_147_483_647
const DEFAULT_SUCCESS_TEXT = 'mock response recovered'
const DEFAULT_PARTIAL_TEXT = 'discarded partial response'
const DEFAULT_REASONING_TEXT = 'mock reasoning'
@@ -203,14 +205,24 @@ function resolveOptions(options: MockLlmServerOptions): ResolvedOptions {
const host = options.host ?? '127.0.0.1'
const port = boundedInteger('port', options.port ?? 0, 0, 65_535)
const chunkSize = boundedInteger('chunkSize', options.chunkSize ?? 8, 1, Number.MAX_SAFE_INTEGER)
const chunkDelayMs = boundedInteger('chunkDelayMs', options.chunkDelayMs ?? 25, 0, MAX_TIMER_DELAY_MS)
const chunkDelayMs = boundedInteger(
'chunkDelayMs',
options.chunkDelayMs ?? 25,
0,
MAX_MOCK_LLM_TIMER_DELAY_MS,
)
const disconnectDelayMs = boundedInteger(
'disconnectDelayMs',
options.disconnectDelayMs ?? 10,
0,
MAX_TIMER_DELAY_MS,
MAX_MOCK_LLM_TIMER_DELAY_MS,
)
const retryAfterMs = boundedInteger(
'retryAfterMs',
options.retryAfterMs ?? 1_000,
1,
MAX_MOCK_LLM_TIMER_DELAY_MS,
)
const retryAfterMs = boundedInteger('retryAfterMs', options.retryAfterMs ?? 1_000, 1, MAX_TIMER_DELAY_MS)
const randomSeed = boundedInteger(
'randomSeed',
options.randomSeed ?? randomBytes(4).readUInt32LE(0),
@@ -285,8 +297,9 @@ function emit(options: ResolvedOptions, event: MockLlmServerEvent): void {
}
async function readJsonBody(request: IncomingMessage): Promise<unknown> {
let body = ''
for await (const chunk of request) body += Buffer.from(chunk).toString('utf8')
const chunks: Buffer[] = []
for await (const chunk of request) chunks.push(Buffer.from(chunk as Uint8Array))
const body = Buffer.concat(chunks).toString('utf8')
return body.length === 0 ? undefined : JSON.parse(body)
}
@@ -320,6 +333,7 @@ function finishRecord(
record: MockLlmRequestRecord,
outcome: MockLlmRequestOutcome,
): void {
if (record.outcome !== undefined) return
record.outcome = outcome
emit(options, {
type: 'result',
@@ -713,8 +727,9 @@ export async function startMockLlmServer(options: MockLlmServerOptions): Promise
})
const address = server.address() as AddressInfo
const advertisedHost = isIP(resolved.host) === 6 ? `[${resolved.host}]` : resolved.host
return {
baseURL: `http://${resolved.host}:${address.port}`,
baseURL: `http://${advertisedHost}:${address.port}`,
port: address.port,
randomSeed: resolved.randomSeed,
requests,

View File

@@ -110,6 +110,9 @@ describe('mock LLM server CLI parser', () => {
[['--sequence', 'unknown'], /unknown behavior/],
[['--sequence', 'connection_refused,success', '--port', '0'], /nonzero/],
[['--sequence', 'success', '--listen-delay-ms', '5'], /requires connection_refused/],
[['--sequence', 'connection_refused,success', '--listen-delay-ms', '-1'], /integer between 0 and 2147483647/],
[['--sequence', 'connection_refused,success', '--listen-delay-ms', '1.5'], /integer between 0 and 2147483647/],
[['--sequence', 'connection_refused,success', '--listen-delay-ms', '2147483648'], /integer between 0 and 2147483647/],
[['--sequence', 'success', '--seed', '1'], /require random/],
[['--sequence', 'random', '--random-weights', 'success'], /expects behavior=weight/],
[['--sequence', 'random', '--random-weights', 'random=1'], /concrete behavior/],

View File

@@ -1,3 +1,4 @@
import { request } from 'node:http'
import { afterEach, describe, expect, it } from 'vitest'
import type { MockLlmBehavior, MockLlmServer, MockLlmServerEvent } from '../src/index.ts'
import { startMockLlmServer } from '../src/index.ts'
@@ -32,6 +33,22 @@ function chat(
})
}
function rawChat(server: MockLlmServer, chunks: readonly Buffer[]): Promise<void> {
return new Promise((resolve, reject) => {
const outgoing = request(`${server.baseURL}/v1/chat/completions`, {
method: 'POST',
headers: { 'content-type': 'application/json' },
}, (response) => {
response.once('error', reject)
response.once('end', resolve)
response.resume()
})
outgoing.once('error', reject)
for (const chunk of chunks) outgoing.write(chunk)
outgoing.end()
})
}
describe('mock LLM server wire behaviors', () => {
it('streams a complete text response and captures the request', async () => {
const events: MockLlmServerEvent[] = []
@@ -151,10 +168,12 @@ describe('mock LLM server wire behaviors', () => {
['stream_disconnect', 100] as const,
['partial_disconnect', 100] as const,
])('records a client that closes during %s', async (behavior, delayMs) => {
const events: MockLlmServerEvent[] = []
const server = await start([behavior], {
chunkDelayMs: delayMs,
disconnectDelayMs: delayMs,
chunkSize: 1,
onEvent: (event) => { events.push(event) },
})
const controller = new AbortController()
const response = await chat(server, { signal: controller.signal })
@@ -163,6 +182,30 @@ describe('mock LLM server wire behaviors', () => {
await new Promise((resolve) => { setTimeout(resolve, 5) })
expect(server.requests[0]).toMatchObject({ behavior, outcome: 'client_closed' })
expect(events.filter(event => event.type === 'result')).toEqual([
expect.objectContaining({ behavior, outcome: 'client_closed' }),
])
})
it('preserves UTF-8 code points split across request chunks', async () => {
const server = await start(['success'])
const encoded = Buffer.from(JSON.stringify({ messages: [{ role: 'user', content: '你好' }] }))
const characterOffset = encoded.indexOf(Buffer.from('你'))
expect(characterOffset).toBeGreaterThanOrEqual(0)
await rawChat(server, [
encoded.subarray(0, characterOffset + 1),
encoded.subarray(characterOffset + 1),
])
expect(server.requests[0]?.body).toEqual({ messages: [{ role: 'user', content: '你好' }] })
})
it('formats an IPv6 listener as a valid base URL', async () => {
const server = await start(['success'], { host: '::1' })
expect(server.baseURL).toMatch(/^http:\/\/\[::1\]:\d+$/)
expect((await chat(server)).status).toBe(200)
})
it('emits reasoning, tool calls, max-token finishes, slow chunks, and a wrong content type', async () => {