import { describe, expect, it } from 'vitest' import { Context } from 'cordis' import LlmService, { errorChain, GenerateOptions, HarnessError, isContextWindowExceededError, isQuotaExceededError, isLlmAdapterFailure, LlmAdapter, LlmError, llmFailureOf, llmRetryPolicyOf, ProviderRequestId, ReasoningEffortId, resolveRetryPolicy, StreamChunk, } from '@deepseek-ai/dsh-llm' import type { LlmModelContext, LlmModelInfo, LlmModelReasoningInfo, LlmProviderInfo, LlmResolvedModelInfo, } from '@deepseek-ai/dsh-llm' class ScriptedAdapter extends LlmAdapter { constructor(private script: StreamChunk[]) { super() } async * stream(_options: GenerateOptions): AsyncIterable { yield * this.script } } class RecordingAdapter extends ScriptedAdapter { lastOptions: GenerateOptions | undefined override async * stream(options: GenerateOptions): AsyncIterable { this.lastOptions = options yield * super.stream(options) } } class ThrowingAdapter extends LlmAdapter { constructor(private readonly failure: Error) { super() } stream(_options: GenerateOptions): AsyncIterable { throw this.failure } } class CatalogAdapter extends ScriptedAdapter { constructor( private readonly provider: LlmProviderInfo, private readonly models: readonly LlmModelInfo[], private readonly contexts: Readonly> = {}, private readonly reasoning: Readonly> = {}, ) { super(SCRIPT) } override providerInfo(_provider: string): LlmProviderInfo { return this.provider } override listModels(_provider: string): Promise { return Promise.resolve(this.models) } override resolveModel( provider: string, model: string, ): Promise { return Promise.resolve({ provider, id: model, name: model, ...this.contexts[model] === undefined ? {} : { context: this.contexts[model] }, ...this.reasoning[model] === undefined ? {} : { reasoning: this.reasoning[model] }, }) } } const SCRIPT: StreamChunk[] = [ { type: 'block-start', index: 0, blockType: 'text' }, { type: 'text-delta', index: 0, text: 'hi' }, { type: 'block-end', index: 0, block: { type: 'text', text: 'hi' } }, { type: 'finish', reason: { kind: 'stop' } }, ] describe('LlmService', () => { it('recognizes structured and model-capacity context-window overflow details', () => { expect(isContextWindowExceededError('context_length_exceeded maximum context length')).toBe(true) expect(isContextWindowExceededError('context-window-overflowed')).toBe(true) expect(isContextWindowExceededError('This model maximum context length is 128000 tokens')).toBe(true) expect(isContextWindowExceededError('input is too long for this model')).toBe(true) expect(isContextWindowExceededError('request too large for model context')).toBe(true) expect(isContextWindowExceededError('input exceeds the model context window limit')).toBe(true) }) it('does not mistake unrelated input validation for context-window overflow', () => { expect(isContextWindowExceededError('invalid request: malformed tool arguments')).toBe(false) expect(isContextWindowExceededError('invalid input: temperature exceeds maximum allowed value')).toBe(false) expect(isContextWindowExceededError('input exceeds maximum allowed value')).toBe(false) expect(isContextWindowExceededError('context window size must be positive')).toBe(false) }) it('distinguishes exhausted account quota from transient rate limiting', () => { for (const detail of [ 'insufficient_quota', 'account balance depleted', 'usage-limit-exceeded', 'out of credits', 'OpenAI API error (429): You exceeded your current quota, please check your plan and billing details.', ]) expect(isQuotaExceededError(detail)).toBe(true) expect(isQuotaExceededError('HTTP 429: rate limit reached')).toBe(false) expect(isQuotaExceededError('quota resets in one minute')).toBe(false) }) it('errorChain renders the full cause chain of a wrapped transport failure', () => { const chain = new TypeError('fetch failed', { cause: new Error('connect ECONNREFUSED 127.0.0.1:443') }) expect(errorChain(chain)).toBe('fetch failed: connect ECONNREFUSED 127.0.0.1:443') }) it('errorChain renders AggregateError members (Happy Eyeballs multi-address failures)', () => { const aggregate = new AggregateError( [new Error('connect ECONNREFUSED ::1:443'), new Error('connect ECONNREFUSED 127.0.0.1:443')], '', ) const wrapped = new TypeError('fetch failed', { cause: aggregate }) expect(errorChain(wrapped)).toBe( 'fetch failed: AggregateError [connect ECONNREFUSED ::1:443; connect ECONNREFUSED 127.0.0.1:443]', ) }) it('errorChain survives non-Error values, hostile coercion, and circular causes', () => { expect(errorChain('plain string')).toBe('plain string') expect(errorChain({ toString: () => { throw new Error('hostile') } })).toBe('') const circular = new Error('outer') circular.cause = circular expect(errorChain(circular)).toBe('outer: ') // A hostile accessor collapses only its own node, not the whole chain. const hostileNode = new Error('node') Object.defineProperty(hostileNode, 'message', { get() { throw new Error('hostile getter') } }) expect(errorChain(new Error('outer', { cause: hostileNode }))).toBe('outer: ') // A diamond-shared (non-cyclic) cause renders in full on both paths. const shared = new Error('shared') const diamond = new AggregateError([new Error('a', { cause: shared }), new Error('b', { cause: shared })], 'agg') expect(errorChain(diamond)).toBe('agg [a: shared; b: shared]') }) it('errorChain falls back to the error name, skips empty aggregates, and stops at null causes', () => { expect(errorChain(new TypeError('', { cause: null }))).toBe('TypeError') expect(errorChain(new AggregateError([], 'all failed'))).toBe('all failed') }) it('errorChain collapses a cause that repeats the wrapper message verbatim', () => { // The `new HarnessError(String(value), code, { cause: value })` normalization // pattern repeats its cause; rendering it twice would only add noise. const wrapped = new HarnessError('boom', 'UNKNOWN', { cause: 'boom' }) expect(errorChain(wrapped)).toBe('boom') }) it('routes stream() to the registered adapter', async () => { const ctx = new Context() await ctx.plugin(LlmService) ctx.llm.registerAdapter(['test-provider'], new ScriptedAdapter(SCRIPT)) const chunks: StreamChunk[] = [] for await (const chunk of ctx.llm.stream({ provider: 'test-provider', model: 'test-model', messages: [] })) chunks.push(chunk) expect(chunks).toEqual(SCRIPT) }) it('captures provider-owned retry policy at registration and defaults omission', async () => { const configured = resolveRetryPolicy({ mode: 'always' }, 'test retryPolicy') const adapter = new class extends ScriptedAdapter { override providerRetryPolicy(provider: string) { return provider === 'configured' ? configured : undefined } }(SCRIPT) const ctx = new Context() await ctx.plugin(LlmService) ctx.llm.registerAdapter(['configured', 'defaulted'], adapter) expect(ctx.llm.providerRetryPolicy('configured')).toBe(configured) expect(ctx.llm.providerRetryPolicy('defaulted')).toMatchObject({ mode: 'normal', maxRetries: 2, }) expect(() => ctx.llm.providerRetryPolicy('missing')).toThrow( expect.objectContaining({ code: 'NO_ADAPTER' }), ) }) it('keeps the serving registration policy on an in-flight call after route replacement', async () => { const oldPolicy = resolveRetryPolicy({ mode: 'always' }, 'old retryPolicy') const newPolicy = resolveRetryPolicy({ mode: 'normal', maxRetries: 0 }, 'new retryPolicy') const entered = Promise.withResolvers() const release = Promise.withResolvers() const failure = new LlmError('old route failed', 'AUTH') const oldAdapter = new class extends LlmAdapter { override providerRetryPolicy(): typeof oldPolicy { return oldPolicy } async * stream(_options: GenerateOptions): AsyncIterable { entered.resolve(undefined) await release.promise throw failure } }() const newAdapter = new class extends ScriptedAdapter { override providerRetryPolicy(): typeof newPolicy { return newPolicy } }(SCRIPT) const ctx = new Context() await ctx.plugin(LlmService) const disposeOld = ctx.llm.registerAdapter(['route'], oldAdapter) const stream = ctx.llm.stream({ provider: 'route', model: 'model', messages: [] }) const outcome = (async (): Promise => { try { for await (const _chunk of stream) { /* drain */ } } catch (error: unknown) { return error } return undefined })() await entered.promise disposeOld() ctx.llm.registerAdapter(['route'], newAdapter) release.resolve(undefined) expect(await outcome).toBe(failure) expect(llmRetryPolicyOf(stream)).toBe(oldPolicy) expect(ctx.llm.providerRetryPolicy('route')).toBe(newPolicy) }) it('throws NO_ADAPTER for unregistered providers', async () => { const ctx = new Context() await ctx.plugin(LlmService) const stream = ctx.llm.stream({ provider: 'nope', model: 'any-model', messages: [] }) let caught: unknown try { for await (const _ of stream) { /* drain */ } } catch (error: unknown) { caught = error } expect(caught).toBeInstanceOf(LlmError) expect((caught as LlmError).code).toBe('NO_ADAPTER') expect((caught as LlmError).message).toContain('no adapter registered') expect(isLlmAdapterFailure(stream, caught)).toBe(true) expect(llmRetryPolicyOf(stream)).toBeUndefined() }) it.each(['done', 'value'] as const)('tags a throwing IteratorResult.%s getter without replacing its Error', async (field) => { const original = new LlmError(`${field} getter failed`, 'RESULT_GETTER_FAILED') const result = field === 'done' ? {} : { done: false } Object.defineProperty(result, field, { get: () => { throw original } }) let cleanupLookups = 0 const iterator: AsyncIterator = { next: () => Promise.resolve(result as unknown as IteratorResult), } Object.defineProperty(iterator, 'return', { get: () => { cleanupLookups += 1 throw new Error('return getter must not run after iteration fails') }, }) const adapter = new class extends LlmAdapter { stream(_options: GenerateOptions): AsyncIterable { return { [Symbol.asyncIterator](): AsyncIterator { return iterator }, } } }() const ctx = new Context() await ctx.plugin(LlmService) ctx.llm.registerAdapter(['test-model'], adapter) const stream = ctx.llm.stream({ provider: 'test-model', model: 'test-model', messages: [] }) let caught: unknown try { for await (const _chunk of stream) { /* drain */ } } catch (error: unknown) { caught = error } expect(caught).toBe(original) expect(isLlmAdapterFailure(stream, caught)).toBe(true) expect(cleanupLookups).toBe(0) }) it.each(['dispatch', 'iterator'] as const)('tags synchronous adapter %s failures without replacing their Error', async (boundary) => { const original = new LlmError(`${boundary} failed`, 'BOUNDARY_FAILED') const adapter = new class extends LlmAdapter { stream(_options: GenerateOptions): AsyncIterable { if (boundary === 'dispatch') throw original return { [Symbol.asyncIterator]: () => { throw original } } } }() const ctx = new Context() await ctx.plugin(LlmService) ctx.llm.registerAdapter(['test-model'], adapter) const stream = ctx.llm.stream({ provider: 'test-model', model: 'test-model', messages: [] }) let caught: unknown try { for await (const _chunk of stream) { /* drain */ } } catch (error: unknown) { caught = error } expect(caught).toBe(original) expect(isLlmAdapterFailure(stream, caught)).toBe(true) expect(llmFailureOf(stream, caught)).toEqual({ message: `${boundary} failed`, code: 'BOUNDARY_FAILED', }) }) it('keeps structured provider facts beside a frozen third-party Error', async () => { const original = new LlmError('provider busy', 'RATE_LIMIT', { status: 429, providerRetryAfterMs: 1_500, requestId: ProviderRequestId('req-7'), }) Object.freeze(original) const ctx = new Context() await ctx.plugin(LlmService) ctx.llm.registerAdapter(['test-provider'], new ThrowingAdapter(original)) const stream = ctx.llm.stream({ provider: 'test-provider', model: 'test-model', messages: [] }) let caught: unknown try { for await (const _chunk of stream) { /* drain */ } } catch (error: unknown) { caught = error } expect(caught).toBe(original) expect(llmFailureOf(stream, caught)).toEqual({ message: 'provider busy', code: 'RATE_LIMIT', status: 429, providerRetryAfterMs: 1_500, requestId: ProviderRequestId('req-7'), }) }) it('does not trust retry facts carried by an unknown third-party Error', async () => { const carried = { message: 'busy', code: 'SERVER', status: 503 } const original = Object.assign(new Error('busy'), { failure: carried }) const ctx = new Context() await ctx.plugin(LlmService) ctx.llm.registerAdapter(['test-provider'], new ThrowingAdapter(original)) const stream = ctx.llm.stream({ provider: 'test-provider', model: 'test-model', messages: [] }) await expect((async () => { for await (const _chunk of stream) { /* drain */ } })()).rejects.toBe(original) const facts = llmFailureOf(stream, original) carried.status = 500 expect(facts).toEqual({ message: 'busy', code: 'UNKNOWN' }) expect(Object.isFrozen(facts)).toBe(true) expect(facts).not.toBe(carried) }) it('keeps validated failure facts across package copies with matching own codes', async () => { const original = Object.assign(new Error('provider busy'), { code: 'RATE_LIMIT', failure: { message: 'provider busy', code: 'RATE_LIMIT', status: 429, providerRetryAfterMs: 1_500, requestId: 'req-cross-copy', }, }) const ctx = new Context() await ctx.plugin(LlmService) ctx.llm.registerAdapter(['test-provider'], new ThrowingAdapter(original)) const stream = ctx.llm.stream({ provider: 'test-provider', model: 'test-model', messages: [] }) await expect((async () => { for await (const _chunk of stream) { /* drain */ } })()).rejects.toBe(original) expect(llmFailureOf(stream, original)).toEqual({ message: 'provider busy', code: 'RATE_LIMIT', status: 429, providerRetryAfterMs: 1_500, requestId: 'req-cross-copy', }) }) it('keeps an unknown SDK Error exact without trusting its private code or accessors', async () => { const original = Object.assign(new Error('socket closed'), { code: 'ECONNRESET' }) Object.defineProperty(original, 'failure', { get() { throw new Error('SDK failure accessor must not run') }, }) const ctx = new Context() await ctx.plugin(LlmService) ctx.llm.registerAdapter(['test-provider'], new ThrowingAdapter(original)) const stream = ctx.llm.stream({ provider: 'test-provider', model: 'test-model', messages: [] }) await expect((async () => { for await (const _chunk of stream) { /* drain */ } })()).rejects.toBe(original) expect(original.code).toBe('ECONNRESET') expect(llmFailureOf(stream, original)).toEqual({ message: 'socket closed', code: 'UNKNOWN' }) }) it('keeps an SDK Error exact when its message accessor is hostile', async () => { const original = Object.defineProperty(new Error(), 'message', { get() { throw new Error('SDK message accessor trap') }, }) const ctx = new Context() await ctx.plugin(LlmService) ctx.llm.registerAdapter(['test-provider'], new ThrowingAdapter(original)) const stream = ctx.llm.stream({ provider: 'test-provider', model: 'test-model', messages: [] }) await expect((async () => { for await (const _chunk of stream) { /* drain */ } })()).rejects.toBe(original) expect(llmFailureOf(stream, original)).toEqual({ message: 'LLM adapter failed', code: 'UNKNOWN' }) }) it('keeps an SDK Error exact without trusting accessor-backed carried facts', async () => { const original = Object.assign(new Error('busy'), { failure: { message: 'busy', code: 'SERVER', status: 503 }, }) Object.defineProperty(original, 'code', { get() { throw new Error('SDK code accessor must not escape') }, }) const ctx = new Context() await ctx.plugin(LlmService) ctx.llm.registerAdapter(['test-provider'], new ThrowingAdapter(original)) const stream = ctx.llm.stream({ provider: 'test-provider', model: 'test-model', messages: [] }) await expect((async () => { for await (const _chunk of stream) { /* drain */ } })()).rejects.toBe(original) expect(llmFailureOf(stream, original)).toEqual({ message: 'busy', code: 'UNKNOWN' }) }) it('does not trust carried facts matched only by an inherited code', async () => { class InheritedCodeError extends Error { get code(): string { return 'SERVER' } } const original = Object.assign(new InheritedCodeError('busy'), { failure: { message: 'busy', code: 'SERVER', status: 503 }, }) const ctx = new Context() await ctx.plugin(LlmService) ctx.llm.registerAdapter(['test-provider'], new ThrowingAdapter(original)) const stream = ctx.llm.stream({ provider: 'test-provider', model: 'test-model', messages: [] }) await expect((async () => { for await (const _chunk of stream) { /* drain */ } })()).rejects.toBe(original) expect(llmFailureOf(stream, original)).toEqual({ message: 'busy', code: 'UNKNOWN' }) }) it('keeps an SDK Error exact when code descriptor inspection is trapped', async () => { const target = Object.assign(new Error('busy'), { code: 'SERVER', failure: { message: 'busy', code: 'SERVER', status: 503 }, }) const original = new Proxy(target, { getOwnPropertyDescriptor(value, property) { if (property === 'code') throw new Error('SDK code descriptor trap') return Reflect.getOwnPropertyDescriptor(value, property) }, }) const ctx = new Context() await ctx.plugin(LlmService) ctx.llm.registerAdapter(['test-provider'], new ThrowingAdapter(original)) const stream = ctx.llm.stream({ provider: 'test-provider', model: 'test-model', messages: [] }) await expect((async () => { for await (const _chunk of stream) { /* drain */ } })()).rejects.toBe(original) expect(llmFailureOf(stream, original)).toEqual({ message: 'busy', code: 'UNKNOWN' }) }) it('falls back safely when SDK objects trap failure inspection or expose malformed facts', async () => { const propertyTrap = new Proxy(new HarnessError('descriptor trapped', 'SERVER'), { getOwnPropertyDescriptor(target, property) { if (property === 'failure') throw new Error('SDK descriptor trap') return Reflect.getOwnPropertyDescriptor(target, property) }, }) const throwingFacts = Object.create(null) as Record Object.defineProperty(throwingFacts, 'message', { get() { throw new Error('SDK fact getter trap') }, }) const carrying = (message: string, failure: unknown): HarnessError => Object.defineProperty( new HarnessError(message, 'SERVER'), 'failure', { value: failure }, ) const factGetter = carrying('fact getter failed', throwingFacts) const malformed = carrying('malformed facts', { message: 'provider busy', code: 'SERVER', requestId: 1 }) const primitive = carrying('primitive facts', 1) const nullFacts = carrying('null facts', null) const mismatched = carrying('mismatched facts', { message: 'busy', code: 'RATE_LIMIT' }) for (const [original, expectedMessage] of [ [propertyTrap, 'descriptor trapped'], [factGetter, 'fact getter failed'], [malformed, 'malformed facts'], [primitive, 'primitive facts'], [nullFacts, 'null facts'], [mismatched, 'mismatched facts'], ] as const) { const ctx = new Context() await ctx.plugin(LlmService) ctx.llm.registerAdapter(['test-provider'], new ThrowingAdapter(original)) const stream = ctx.llm.stream({ provider: 'test-provider', model: 'test-model', messages: [] }) await expect((async () => { for await (const _chunk of stream) { /* drain */ } })()).rejects.toBe(original) expect(llmFailureOf(stream, original)).toEqual({ message: expectedMessage, code: 'SERVER' }) } }) it('retains a stable code from a HarnessError without requiring LlmError facts', async () => { const original = new HarnessError('stable adapter failure', 'ADAPTER_STABLE') const ctx = new Context() await ctx.plugin(LlmService) ctx.llm.registerAdapter(['test-provider'], new ThrowingAdapter(original)) const stream = ctx.llm.stream({ provider: 'test-provider', model: 'test-model', messages: [] }) await expect((async () => { for await (const _chunk of stream) { /* drain */ } })()).rejects.toBe(original) expect(llmFailureOf(stream, original)).toEqual({ message: 'stable adapter failure', code: 'ADAPTER_STABLE', }) expect(llmFailureOf(stream, 'not an Error')).toBeUndefined() expect(llmFailureOf({ [Symbol.asyncIterator]: () => stream[Symbol.asyncIterator]() }, original)).toBeUndefined() }) it('keeps a nested adapter failure scoped to the nested model call', async () => { const original = new LlmError('nested provider failed', 'NESTED_FAILED') const outer = new RecordingAdapter(SCRIPT) const nested = new ThrowingAdapter(original) const ctx = new Context() await ctx.plugin(LlmService) ctx.llm.registerAdapter(['outer'], outer) ctx.llm.registerAdapter(['nested'], nested) let nestedStream: AsyncIterable | undefined ctx.on('llm/stream', (options, next) => { if (options.provider !== 'outer') return next() return (async function* () { nestedStream = ctx.llm.stream({ provider: 'nested', model: 'nested', messages: [] }) yield * nestedStream })() }) const outerStream = ctx.llm.stream({ provider: 'outer', model: 'outer', messages: [] }) let caught: unknown try { for await (const _chunk of outerStream) { /* drain */ } } catch (error: unknown) { caught = error } expect(caught).toBe(original) expect(nestedStream).toBeDefined() expect(isLlmAdapterFailure(nestedStream!, caught)).toBe(true) expect(isLlmAdapterFailure(outerStream, caught)).toBe(false) expect(outer.lastOptions).toBeUndefined() }) it('keeps call scopes distinct when middleware reuses an iterable', async () => { const firstFailure = new LlmError('first provider failed', 'FIRST_FAILED') const secondFailure = new LlmError('second provider failed', 'SECOND_FAILED') const delegates: AsyncIterable[] = [] const shared: AsyncIterable = { [Symbol.asyncIterator](): AsyncIterator { const delegate = delegates.shift() if (delegate === undefined) throw new Error('shared stream has no call delegate') return delegate[Symbol.asyncIterator]() }, } const ctx = new Context() await ctx.plugin(LlmService) ctx.llm.registerAdapter(['first'], new ThrowingAdapter(firstFailure)) ctx.llm.registerAdapter(['second'], new ThrowingAdapter(secondFailure)) ctx.on('llm/stream', (_options, next) => { delegates.push(next()) return shared }) const firstStream = ctx.llm.stream({ provider: 'first', model: 'first', messages: [] }) const secondStream = ctx.llm.stream({ provider: 'second', model: 'second', messages: [] }) const catchFailure = async (stream: AsyncIterable): Promise => { try { for await (const _chunk of stream) { /* drain */ } } catch (error: unknown) { return error } return new Error('expected adapter to fail') } expect(firstStream).not.toBe(secondStream) const firstCaught = await catchFailure(firstStream) expect(firstCaught).toBe(firstFailure) expect(isLlmAdapterFailure(firstStream, firstCaught)).toBe(true) expect(isLlmAdapterFailure(secondStream, firstCaught)).toBe(false) const secondCaught = await catchFailure(secondStream) expect(secondCaught).toBe(secondFailure) expect(isLlmAdapterFailure(secondStream, secondCaught)).toBe(true) expect(isLlmAdapterFailure(firstStream, secondCaught)).toBe(false) expect(delegates).toHaveLength(0) }) it('propagates a rejected next promptly without awaiting a non-settling return', async () => { const original = new LlmError('provider failed', 'PROVIDER_FAILED') let cleanupCalls = 0 const adapter = new class extends LlmAdapter { stream(_options: GenerateOptions): AsyncIterable { return { [Symbol.asyncIterator](): AsyncIterator { return { next: () => Promise.reject(original), return: () => { cleanupCalls += 1 return new Promise>(() => {}) }, } }, } } }() const ctx = new Context() await ctx.plugin(LlmService) ctx.llm.registerAdapter(['test-model'], adapter) const stream = ctx.llm.stream({ provider: 'test-model', model: 'test-model', messages: [] }) const failure = (async (): Promise => { try { for await (const _chunk of stream) { /* drain */ } } catch (error: unknown) { return error } return new Error('expected adapter iteration to fail') })() let timer: ReturnType | undefined const timeout = new Promise((resolve) => { timer = setTimeout(() => { resolve(new Error('adapter failure did not settle promptly')) }, 100) }) const caught = await Promise.race([failure, timeout]) if (timer !== undefined) clearTimeout(timer) expect(caught).toBe(original) expect(isLlmAdapterFailure(stream, caught)).toBe(true) expect(cleanupCalls).toBe(0) }) it('awaits one adapter return on downstream close and leaves its rejection unclassified', async () => { const cleanup = new Error('cleanup failed') let cleanupCalls = 0 const adapter = new class extends LlmAdapter { stream(_options: GenerateOptions): AsyncIterable { return { [Symbol.asyncIterator](): AsyncIterator { return { next: () => Promise.resolve({ done: false, value: SCRIPT[0]! }), return: () => { cleanupCalls += 1 return Promise.reject(cleanup) }, } }, } } }() const ctx = new Context() await ctx.plugin(LlmService) ctx.llm.registerAdapter(['test-model'], adapter) const stream = ctx.llm.stream({ provider: 'test-model', model: 'test-model', messages: [] }) let caught: unknown try { for await (const _chunk of stream) break } catch (error: unknown) { caught = error } expect(caught).toBe(cleanup) expect(isLlmAdapterFailure(stream, caught)).toBe(false) expect(cleanupCalls).toBe(1) }) it('allows downstream close when the adapter iterator has no return method', async () => { const adapter = new class extends LlmAdapter { stream(_options: GenerateOptions): AsyncIterable { return { [Symbol.asyncIterator](): AsyncIterator { return { next: () => Promise.resolve({ done: false, value: SCRIPT[0]! }) } }, } } }() const ctx = new Context() await ctx.plugin(LlmService) ctx.llm.registerAdapter(['test-model'], adapter) let chunks = 0 for await (const _chunk of ctx.llm.stream({ provider: 'test-model', model: 'test-model', messages: [] })) { chunks += 1 break } expect(chunks).toBe(1) }) it('normalizes and tags non-Error adapter failures once', async () => { const adapter = new class extends LlmAdapter { stream(_options: GenerateOptions): AsyncIterable { return { [Symbol.asyncIterator](): AsyncIterator { // Third-party adapters can reject with arbitrary values. // eslint-disable-next-line @typescript-eslint/prefer-promise-reject-errors return { next: () => Promise.reject('plain provider failure') } }, } } }() const ctx = new Context() await ctx.plugin(LlmService) ctx.llm.registerAdapter(['test-model'], adapter) const stream = ctx.llm.stream({ provider: 'test-model', model: 'test-model', messages: [] }) let caught: unknown try { for await (const _chunk of stream) { /* drain */ } } catch (error: unknown) { caught = error } expect(caught).toBeInstanceOf(HarnessError) expect(caught).toMatchObject({ code: 'UNKNOWN', cause: 'plain provider failure' }) expect(isLlmAdapterFailure(stream, caught)).toBe(true) }) it('does not tag a failure thrown downstream while consuming adapter output', async () => { const downstream = new Error('consumer failed') const ctx = new Context() await ctx.plugin(LlmService) ctx.llm.registerAdapter(['test-model'], new ScriptedAdapter(SCRIPT)) const stream = ctx.llm.stream({ provider: 'test-model', model: 'test-model', messages: [] }) let caught: unknown try { for await (const _chunk of stream) throw downstream } catch (error: unknown) { caught = error } expect(caught).toBe(downstream) expect(isLlmAdapterFailure(stream, caught)).toBe(false) expect(isLlmAdapterFailure(new ScriptedAdapter(SCRIPT).stream({ provider: 'unbound', model: 'unbound', messages: [], }), caught)).toBe(false) expect(isLlmAdapterFailure(stream, 'consumer failed')).toBe(false) }) it('unregisters adapters when the owning fiber is disposed (HMR safety)', async () => { const ctx = new Context() await ctx.plugin(LlmService) const fiber = await ctx.plugin(Object.assign((inner: Context) => { inner.llm.registerAdapter(['scoped-model'], new ScriptedAdapter(SCRIPT)) }, { inject: ['llm'] })) expect(ctx.llm.listProviders()).toEqual([{ id: 'scoped-model', name: 'scoped-model' }]) await fiber.dispose() expect(ctx.llm.listProviders()).toEqual([]) }) it('discovers detached provider and advisory model metadata', async () => { const ctx = new Context() await ctx.plugin(LlmService) const provider = { id: 'catalog', name: 'Catalog Provider' } const model = { provider: 'catalog', id: 'fast', name: 'Fast', description: 'Low latency' } ctx.llm.registerAdapter(['catalog'], new CatalogAdapter(provider, [model])) const providers = ctx.llm.listProviders() const models = await ctx.llm.listModels('catalog') expect(providers).toEqual([provider]) expect(models).toEqual([model]) providers[0]!.name = 'mutated' models[0]!.name = 'mutated' provider.name = 'source mutated' model.name = 'source mutated' expect(ctx.llm.listProviders()).toEqual([{ id: 'catalog', name: 'Catalog Provider' }]) await expect(ctx.llm.listModels('catalog')).resolves.toEqual([{ provider: 'catalog', id: 'fast', name: 'source mutated', description: 'Low latency', }]) }) it('defaults adapters to their route name and an empty advisory model list', async () => { const ctx = new Context() await ctx.plugin(LlmService) ctx.llm.registerAdapter(['plain'], new ScriptedAdapter(SCRIPT)) expect(ctx.llm.listProviders()).toEqual([{ id: 'plain', name: 'plain' }]) await expect(ctx.llm.listModels('plain')).resolves.toEqual([]) await expect(ctx.llm.listModels('missing')).rejects.toMatchObject({ code: 'NO_ADAPTER' }) await expect(ctx.llm.resolveModelInfo('plain', 'unlisted')).resolves.toEqual({ provider: 'plain', id: 'unlisted', name: 'unlisted', }) await expect(ctx.llm.resolveModelInfo('missing', 'm')).rejects.toMatchObject({ code: 'NO_ADAPTER' }) }) it.each([ [{ provider: 1, id: 'model', name: 'Model' }, 'non-string provider'], [{ provider: 'other', id: 'model', name: 'Model' }, 'mismatched provider'], [{ provider: 'route', id: 1, name: 'Model' }, 'non-string id'], [{ provider: 'route', id: 'other', name: 'Model' }, 'mismatched id'], [{ provider: 'route', id: 'model', name: 1 }, 'non-string name'], [{ provider: 'route', id: 'model', name: '' }, 'empty name'], [{ provider: 'route', id: 'model', name: 'Model', description: 1 }, 'non-string description'], ] as const)('rejects invalid exact model metadata (%s: %s)', async (metadata, _label) => { const ctx = new Context() await ctx.plugin(LlmService) const adapter = new class extends ScriptedAdapter { override resolveModel(): Promise { return Promise.resolve(metadata as unknown as LlmResolvedModelInfo) } }(SCRIPT) ctx.llm.registerAdapter(['route'], adapter) await expect(ctx.llm.resolveModelInfo('route', 'model')) .rejects.toMatchObject({ code: 'INVALID_MODEL_INFO' }) }) it('resolves detached model context independently of advisory catalog membership', async () => { const ctx = new Context() await ctx.plugin(LlmService) const source = { contextWindow: 32_000 } ctx.llm.registerAdapter(['route'], new CatalogAdapter( { id: 'route', name: 'Route' }, [], { unlisted: source }, )) const resolved = await ctx.llm.resolveModelInfo('route', 'unlisted') expect(resolved.context).toEqual({ contextWindow: 32_000 }) source.contextWindow = 64_000 expect(resolved.context).toEqual({ contextWindow: 32_000 }) await expect(ctx.llm.resolveModelInfo('route', 'other')).resolves.toEqual({ provider: 'route', id: 'other', name: 'other', }) }) it('resolves detached adapter-owned reasoning metadata and materializes its default', async () => { const ctx = new Context() await ctx.plugin(LlmService) const source = { efforts: [ { id: ReasoningEffortId('standard'), name: 'Standard' }, { id: ReasoningEffortId('ultra'), name: 'Ultra', description: 'Largest budget' }, ], defaultEffort: ReasoningEffortId('standard'), } ctx.llm.registerAdapter(['route'], new CatalogAdapter( { id: 'route', name: 'Route' }, [], {}, { model: source }, )) const resolved = await ctx.llm.resolveModelInfo('route', 'model') expect(resolved.reasoning).toEqual(source) source.efforts[0]!.name = 'mutated' expect(resolved.reasoning?.efforts[0]?.name).toBe('Standard') await expect(ctx.llm.resolveCallConfig({ provider: 'route', model: 'model' })).resolves.toEqual({ provider: 'route', model: 'model', reasoningEffort: ReasoningEffortId('standard'), }) const explicit = { provider: 'route', model: 'model', reasoningEffort: ReasoningEffortId('ultra') } await expect(ctx.llm.resolveCallConfig(explicit)).resolves.toBe(explicit) }) it.each([ [{ efforts: [] }, 'empty effort list'], [{ efforts: [{ id: '', name: 'Empty' }] }, 'empty id'], [{ efforts: [{ id: 'valid', name: '' }] }, 'empty name'], [{ efforts: [{ id: 'valid', name: 'Valid', description: 1 }] }, 'non-string description'], [{ efforts: [{ id: 'same', name: 'One' }, { id: 'same', name: 'Two' }] }, 'duplicate id'], [{ efforts: [{ id: 'valid', name: 'Valid' }], defaultEffort: 'other' }, 'unknown default'], ] as const)('rejects invalid model reasoning metadata (%s: %s)', async (metadata, _label) => { const ctx = new Context() await ctx.plugin(LlmService) ctx.llm.registerAdapter(['route'], new CatalogAdapter( { id: 'route', name: 'Route' }, [], {}, { model: metadata as unknown as LlmModelReasoningInfo }, )) await expect(ctx.llm.resolveModelInfo('route', 'model')) .rejects.toMatchObject({ code: 'INVALID_MODEL_REASONING' }) }) it('rejects unsupported reasoning efforts without clamping', async () => { const ctx = new Context() await ctx.plugin(LlmService) ctx.llm.registerAdapter(['route'], new CatalogAdapter( { id: 'route', name: 'Route' }, [], {}, { model: { efforts: [{ id: ReasoningEffortId('ultra'), name: 'Ultra' }] } }, )) await expect(ctx.llm.resolveCallConfig({ provider: 'route', model: 'model', reasoningEffort: ReasoningEffortId('standard'), })).rejects.toMatchObject({ code: 'UNSUPPORTED_REASONING_EFFORT' }) await expect(ctx.llm.resolveCallConfig({ provider: 'route', model: 'plain', reasoningEffort: ReasoningEffortId('standard'), })).rejects.toMatchObject({ code: 'UNSUPPORTED_REASONING_EFFORT' }) }) it('resolves reasoning defaults at the final adapter boundary after routing middleware', async () => { const ctx = new Context() await ctx.plugin(LlmService) const adapter = new class extends RecordingAdapter { override resolveModel(provider: string, model: string): Promise { const reasoning: LlmModelReasoningInfo = { efforts: [{ id: ReasoningEffortId('standard'), name: 'Standard' }], defaultEffort: ReasoningEffortId('standard'), } return Promise.resolve({ provider, id: model, name: model, reasoning, }) } }(SCRIPT) ctx.llm.registerAdapter(['routed'], adapter) const disposeRouting = ctx.on('llm/stream', (options, next) => { options.provider = 'routed' return next() }) for await (const _chunk of ctx.llm.stream({ provider: 'initial', model: 'model', messages: [], })) { /* drain */ } expect(adapter.lastOptions?.reasoningEffort).toBe(ReasoningEffortId('standard')) disposeRouting() const frozenRequest: GenerateOptions = Object.freeze({ provider: 'routed', model: 'model', messages: [], }) for await (const _chunk of ctx.llm.stream(frozenRequest)) { /* drain */ } expect(adapter.lastOptions?.reasoningEffort).toBe(ReasoningEffortId('standard')) expect(Object.isFrozen(adapter.lastOptions)).toBe(true) }) it('pins one adapter registration across asynchronous exact-model resolution and dispatch', async () => { const ctx = new Context() await ctx.plugin(LlmService) const started = Promise.withResolvers() const reasoning = Promise.withResolvers() const first = new class extends RecordingAdapter { override async resolveModel( provider: string, model: string, _signal?: AbortSignal, ): Promise { started.resolve(undefined) return { provider, id: model, name: model, reasoning: await reasoning.promise, } } }(SCRIPT) const disposeFirst = ctx.llm.registerAdapter(['route'], first) const draining = (async () => { for await (const _chunk of ctx.llm.stream({ provider: 'route', model: 'model', messages: [], })) { /* drain */ } })() await started.promise disposeFirst() const second = new RecordingAdapter(SCRIPT) ctx.llm.registerAdapter(['route'], second) reasoning.resolve({ efforts: [{ id: ReasoningEffortId('high'), name: 'High' }], defaultEffort: ReasoningEffortId('high'), }) await draining expect(first.lastOptions?.reasoningEffort).toBe(ReasoningEffortId('high')) expect(second.lastOptions).toBeUndefined() }) it('prepares a one-shot registration-bound call and rejects config drift', async () => { const ctx = new Context() await ctx.plugin(LlmService) const adapter = new CatalogAdapter( { id: 'route', name: 'Route' }, [], {}, { model: { efforts: [{ id: ReasoningEffortId('high'), name: 'High' }], defaultEffort: ReasoningEffortId('high'), }, }, ) ctx.llm.registerAdapter(['route'], adapter) const prepared = await ctx.llm.prepareCall({ provider: 'route', model: 'model' }) expect(Object.isFrozen(prepared.config)).toBe(true) const stream = prepared.stream({ ...prepared.config, model: 'other', messages: [], }) await expect((async () => { for await (const _chunk of stream) { /* drain */ } })()).rejects.toMatchObject({ code: 'INVALID_PREPARED_CALL' }) expect(() => prepared.stream({ ...prepared.config, messages: [], })).toThrow(expect.objectContaining({ code: 'INVALID_PREPARED_CALL' })) }) it('passes cancellation through exact-model resolution', async () => { const ctx = new Context() await ctx.plugin(LlmService) const started = Promise.withResolvers() const adapter = new class extends ScriptedAdapter { override resolveModel( _provider: string, _model: string, signal?: AbortSignal, ): Promise { started.resolve(undefined) return new Promise((_resolve, reject) => { if (signal === undefined) { reject(new Error('missing reasoning signal')) return } if (signal.aborted) { reject(signal.reason instanceof Error ? signal.reason : new Error('reasoning aborted')) return } signal.addEventListener('abort', () => { reject(signal.reason instanceof Error ? signal.reason : new Error('reasoning aborted')) }, { once: true }) }) } }(SCRIPT) ctx.llm.registerAdapter(['route'], adapter) const controller = new AbortController() const resolving = ctx.llm.resolveCallConfig( { provider: 'route', model: 'model' }, controller.signal, ) await started.promise const reason = new Error('cancel reasoning') controller.abort(reason) await expect(resolving).rejects.toBe(reason) }) it.each([0, -1, 1.5, Number.NaN])( 'rejects invalid adapter model context %s', async (contextWindow) => { const ctx = new Context() await ctx.plugin(LlmService) ctx.llm.registerAdapter(['route'], new CatalogAdapter( { id: 'route', name: 'Route' }, [], { model: { contextWindow } }, )) await expect(ctx.llm.resolveModelInfo('route', 'model')) .rejects.toMatchObject({ code: 'INVALID_MODEL_CONTEXT' }) }, ) it.each([ [{ id: 1, name: 'Name' }, 'non-string id'], [{ id: 'other', name: 'Name' }, 'mismatched id'], [{ id: 'route', name: 1 }, 'non-string name'], [{ id: 'route', name: '' }, 'empty name'], ] as const)('rejects invalid provider metadata atomically (%s: %s)', async (metadata, _label) => { const ctx = new Context() await ctx.plugin(LlmService) const adapter = new CatalogAdapter(metadata as unknown as LlmProviderInfo, []) expect(() => ctx.llm.registerAdapter(['route'], adapter)).toThrow(expect.objectContaining({ code: 'INVALID_ADAPTER' })) expect(ctx.llm.listProviders()).toEqual([]) }) it.each([ [{ provider: 1, id: 'm', name: 'M' }, 'non-string provider'], [{ provider: 'other', id: 'm', name: 'M' }, 'mismatched provider'], [{ provider: 'route', id: 1, name: 'M' }, 'non-string id'], [{ provider: 'route', id: '', name: 'M' }, 'empty id'], [{ provider: 'route', id: 'm', name: 1 }, 'non-string name'], [{ provider: 'route', id: 'm', name: '' }, 'empty name'], [{ provider: 'route', id: 'm', name: 'M', description: 1 }, 'non-string description'], ] as const)('rejects invalid model metadata (%s: %s)', async (metadata, _label) => { const ctx = new Context() await ctx.plugin(LlmService) ctx.llm.registerAdapter(['route'], new CatalogAdapter( { id: 'route', name: 'Route' }, [metadata as unknown as LlmModelInfo], )) await expect(ctx.llm.listModels('route')).rejects.toMatchObject({ code: 'INVALID_CATALOG' }) }) it('rejects duplicate model ids in one provider catalog', async () => { const ctx = new Context() await ctx.plugin(LlmService) const model = { provider: 'route', id: 'same', name: 'Same' } ctx.llm.registerAdapter(['route'], new CatalogAdapter({ id: 'route', name: 'Route' }, [model, model])) await expect(ctx.llm.listModels('route')).rejects.toMatchObject({ code: 'INVALID_CATALOG' }) }) it('lets llm/stream waterfall listeners wrap the underlying stream', async () => { const ctx = new Context() await ctx.plugin(LlmService) ctx.llm.registerAdapter(['test-model'], new ScriptedAdapter(SCRIPT)) ctx.on('llm/stream', function (_options, next) { const inner = next() return (async function * () { yield { type: 'block-start', index: 99, blockType: 'text' } satisfies StreamChunk yield { type: 'block-end', index: 99, block: { type: 'text', text: '' } } satisfies StreamChunk yield * inner })() }) const chunks: StreamChunk[] = [] for await (const chunk of ctx.llm.stream({ provider: 'test-model', model: 'dynamic-model', messages: [] })) chunks.push(chunk) expect(chunks).toHaveLength(6) expect(chunks[0]).toMatchObject({ index: 99 }) }) it('resolves the provider after llm/stream listeners have had a chance to route it', async () => { const ctx = new Context() await ctx.plugin(LlmService) const adapter = new RecordingAdapter(SCRIPT) ctx.llm.registerAdapter(['routed'], adapter) ctx.on('llm/stream', (options, next) => { options.provider = 'routed' return next() }) for await (const _chunk of ctx.llm.stream({ provider: 'initial', model: 'm', messages: [] })) { /* drain */ } expect(adapter.lastOptions?.provider).toBe('routed') }) it('keeps replay state when historical and target providers belong to the same adapter instance', async () => { const ctx = new Context() await ctx.plugin(LlmService) const adapter = new RecordingAdapter(SCRIPT) ctx.llm.registerAdapter(['historical', 'target'], adapter) const replayState = { private: 'state' } for await (const _chunk of ctx.llm.stream({ provider: 'target', model: 'new-model', messages: [{ role: 'assistant', content: [{ type: 'text', text: 'old response' }], provenance: { provider: 'historical', model: 'old-model', replayState }, }], })) { /* drain */ } expect(adapter.lastOptions?.messages[0]?.provenance).toEqual({ provider: 'historical', model: 'old-model', replayState, }) }) it('strips replay state but preserves provenance when the target uses a different adapter instance', async () => { const ctx = new Context() await ctx.plugin(LlmService) ctx.llm.registerAdapter(['historical'], new RecordingAdapter(SCRIPT)) const target = new RecordingAdapter(SCRIPT) ctx.llm.registerAdapter(['target'], target) for await (const _chunk of ctx.llm.stream({ provider: 'target', model: 'new-model', messages: [{ role: 'assistant', content: [{ type: 'text', text: 'old response' }], provenance: { provider: 'historical', model: 'old-model', replayState: { private: 'state' } }, }], })) { /* drain */ } expect(target.lastOptions?.messages[0]?.provenance).toEqual({ provider: 'historical', model: 'old-model' }) }) it('preserves immutability while stripping replay state from frozen requests', async () => { const ctx = new Context() await ctx.plugin(LlmService) ctx.llm.registerAdapter(['historical'], new RecordingAdapter(SCRIPT)) const target = new RecordingAdapter(SCRIPT) ctx.llm.registerAdapter(['target'], target) const options = Object.freeze({ provider: 'target', model: 'new-model', messages: [{ role: 'assistant' as const, content: [{ type: 'text' as const, text: 'old response' }], provenance: { provider: 'historical', model: 'old-model', replayState: { private: 'state' } }, }], }) for await (const _chunk of ctx.llm.stream(options)) { /* drain */ } expect(target.lastOptions).not.toBe(options) expect(Object.isFrozen(target.lastOptions)).toBe(true) expect(target.lastOptions?.messages[0]?.provenance).toEqual({ provider: 'historical', model: 'old-model' }) }) it('creates LlmError with a code for programmatic handling', () => { const err = new LlmError('something went wrong', 'CUSTOM_CODE') expect(err).toBeInstanceOf(Error) expect(err.name).toBe('LlmError') expect(err.message).toBe('something went wrong') expect(err.code).toBe('CUSTOM_CODE') }) it('rejects non-serializable structured failure facts at construction', () => { expect(() => new LlmError('busy', 'RATE_LIMIT', { status: 42 })).toThrow(/status/) expect(() => new LlmError('busy', 'RATE_LIMIT', { providerRetryAfterMs: Number.NaN })) .toThrow(/providerRetryAfterMs/) expect(() => new LlmError('busy', 'RATE_LIMIT', { requestId: ProviderRequestId('') })).toThrow(/requestId/) expect(() => new LlmError(1 as never, 'RATE_LIMIT')).toThrow(/message/) expect(() => new LlmError('busy', 1 as never)).toThrow(/code/) expect(() => new LlmError('busy', 'RATE_LIMIT', { requestId: 1 as never })).toThrow(/requestId/) }) it('LlmError extends the shared HarnessError base', async () => { const { HarnessError, isHarnessError } = await import('@deepseek-ai/dsh-llm') const cause = new Error('root cause') const err = new LlmError('boom', 'AUTH', { cause }) expect(err).toBeInstanceOf(HarnessError) expect(isHarnessError(err)).toBe(true) expect(err.code).toBe('AUTH') expect(err.cause).toBe(cause) }) it('HarnessError carries a code, names itself by subclass, and chains cause', async () => { const { HarnessError, isHarnessError } = await import('@deepseek-ai/dsh-llm') const root = new Error('root cause') const err = new HarnessError('wrapper', 'UNKNOWN', { cause: root }) expect(err).toBeInstanceOf(Error) expect(err.name).toBe('HarnessError') expect(err.code).toBe('UNKNOWN') expect(err.cause).toBe(root) expect(isHarnessError(err)).toBe(true) expect(isHarnessError(root)).toBe(false) expect(isHarnessError('nope')).toBe(false) }) it('removes the adapter when the returned disposer is called', async () => { const ctx = new Context() await ctx.plugin(LlmService) const dispose = ctx.llm.registerAdapter(['m1'], new ScriptedAdapter(SCRIPT)) expect(ctx.llm.listProviders()).toEqual([{ id: 'm1', name: 'm1' }]) dispose() expect(ctx.llm.listProviders()).toEqual([]) }) it('rejects duplicate adapter registration with DUPLICATE_ADAPTER code', async () => { const ctx = new Context() await ctx.plugin(LlmService) ctx.llm.registerAdapter(['m1'], new ScriptedAdapter(SCRIPT)) try { ctx.llm.registerAdapter(['m1'], new ScriptedAdapter(SCRIPT)) expect.fail('expected error') } catch (error: unknown) { expect(error).toBeInstanceOf(LlmError) expect((error as LlmError).message).toContain('already registered') expect((error as LlmError).code).toBe('DUPLICATE_ADAPTER') } }) it('rejects empty and internally duplicated provider registrations atomically', async () => { const ctx = new Context() await ctx.plugin(LlmService) const adapter = new ScriptedAdapter(SCRIPT) expect(() => ctx.llm.registerAdapter([], adapter)).toThrow(expect.objectContaining({ code: 'INVALID_ADAPTER' })) expect(() => ctx.llm.registerAdapter([''], adapter)).toThrow(expect.objectContaining({ code: 'INVALID_ADAPTER' })) expect(() => ctx.llm.registerAdapter(['first', 'first'], adapter)).toThrow(expect.objectContaining({ code: 'DUPLICATE_ADAPTER' })) expect(ctx.llm.listProviders()).toEqual([]) }) it('re-registers a model after its prior registration is disposed', async () => { const ctx = new Context() await ctx.plugin(LlmService) const dispose = ctx.llm.registerAdapter(['m1'], new ScriptedAdapter(SCRIPT)) expect(ctx.llm.listProviders()).toEqual([{ id: 'm1', name: 'm1' }]) dispose() expect(ctx.llm.listProviders()).toEqual([]) // The duplicate check is not wedged: the same model registers cleanly again. const disposeAgain = ctx.llm.registerAdapter(['m1'], new ScriptedAdapter(SCRIPT)) expect(ctx.llm.listProviders()).toEqual([{ id: 'm1', name: 'm1' }]) disposeAgain() expect(ctx.llm.listProviders()).toEqual([]) }) })