import { createServer } from 'node:http' import type { IncomingMessage, Server, ServerResponse } from 'node:http' import { afterEach, describe, expect, it } from 'vitest' import { Context } from 'cordis' import LlmService, { userAgent } from '@deepseek-ai/dsh-llm' import * as LlmPiAi from '@deepseek-ai/dsh-llm-pi-ai' import { getBuiltinModels } from '@earendil-works/pi-ai/providers/all' import { discoverModels } from '../src/discovery.ts' const servers: Server[] = [] afterEach(async () => { await Promise.all(servers.splice(0).map(server => new Promise(resolve => server.close(resolve)))) }) interface ListingServer { url: string paths: string[] headers: IncomingMessage['headers'][] } /** * A stand-in provider that answers one scripted `GET /models`. `chunks` writes * without a declared length, which is how a real streamed reply arrives. */ async function listingServer(behavior: { status?: number body?: string chunks?: string[] holdOpenMs?: number }): Promise { const paths: string[] = [] const headers: IncomingMessage['headers'][] = [] const server = createServer((request: IncomingMessage, response: ServerResponse) => { paths.push(request.url ?? '') headers.push(request.headers) if (behavior.chunks !== undefined) { // No declared length: the ceiling has to hold on what is read. response.writeHead(behavior.status ?? 200, { 'content-type': 'application/json' }) for (const chunk of behavior.chunks) response.write(chunk) if (behavior.holdOpenMs === undefined) { response.end(); return } // Left open so a caller's cancellation lands while the body is still // being read rather than after it completed. setTimeout(() => { response.end() }, behavior.holdOpenMs) return } const body = behavior.body ?? '{}' response.writeHead(behavior.status ?? 200, { 'content-type': 'application/json', 'content-length': String(Buffer.byteLength(body)), }) response.end(body) }) servers.push(server) await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)) const address = server.address() if (address === null || typeof address === 'string') throw new Error('no port') return { url: `http://127.0.0.1:${address.port}`, paths, headers } } /** A bare dormant mount: discovery is offered whether or not a route exists. */ async function harness(): Promise { const ctx = new Context() await ctx.plugin(LlmService) await ctx.plugin(LlmPiAi, {}) return ctx } describe('catalog-route model discovery', () => { it('answers from the installed registry, with capacities and no network call', async () => { const server = await listingServer({ body: JSON.stringify({ data: [{ id: 'from-the-endpoint' }] }) }) const ctx = await harness() const models = await ctx.llm.discoverModels('llm-pi-ai', { provider: 'deepseek', baseURL: server.url }) // pi-ai's own registry is the authority for its own providers, and it // carries what a listing endpoint would not disclose. expect(models.map(model => model.id).sort()) .toEqual(getBuiltinModels('deepseek').map(model => model.id).sort()) expect(models.every(model => (model.contextWindow ?? 0) > 0 && (model.maxTokens ?? 0) > 0)).toBe(true) expect(server.paths).toEqual([]) }) it('needs no endpoint for a route the catalog describes', async () => { const ctx = await harness() await expect(ctx.llm.discoverModels('llm-pi-ai', { provider: 'deepseek' })).resolves.not.toHaveLength(0) }) it('says where a route the catalog does not describe must get its models', async () => { const ctx = await harness() await expect(ctx.llm.discoverModels('llm-pi-ai', { provider: 'acme-gateway' })) .rejects.toThrow(/ships no catalog for provider "acme-gateway".*set a baseURL/s) // A form that cleared the field says the same thing as one that never had it. await expect(ctx.llm.discoverModels('llm-pi-ai', { provider: 'acme-gateway', baseURL: '' })) .rejects.toThrow(/set a baseURL/) // The seam refuses a request naming neither, so the module's own guard for // that shape is only reachable by calling it directly. await expect(discoverModels({})).rejects.toThrow(/set a baseURL/) }) }) describe('draft-provider model discovery', () => { it('reads an OpenAI-compatible listing and keeps the capacities it discloses', async () => { const server = await listingServer({ body: JSON.stringify({ data: [ { id: 'acme-large', display_name: 'Acme Large', context_length: 65_536, max_output_tokens: 4096 }, { id: 'acme-small' }, ], }), }) const ctx = await harness() const models = await ctx.llm.discoverModels('llm-pi-ai', { baseURL: `${server.url}/v1`, apiKey: 'probe-key' }) expect(models).toEqual([ { id: 'acme-large', name: 'Acme Large', contextWindow: 65_536, maxTokens: 4096 }, { id: 'acme-small' }, ]) expect(server.paths).toEqual(['/v1/models']) expect(server.headers[0]?.authorization).toBe('Bearer probe-key') expect(server.headers[0]?.['user-agent']).toBe(userAgent()) }) it('keeps a deployment path instead of resolving it away', async () => { const server = await listingServer({ body: JSON.stringify({ data: [{ id: 'm' }] }) }) const ctx = await harness() await ctx.llm.discoverModels('llm-pi-ai', { baseURL: `${server.url}/openai/v1/` }) expect(server.paths).toEqual(['/openai/v1/models']) }) it('offers no credential when the draft names none', async () => { const server = await listingServer({ body: JSON.stringify({ data: [{ id: 'm' }] }) }) const ctx = await harness() await ctx.llm.discoverModels('llm-pi-ai', { baseURL: server.url }) expect(server.headers[0]?.authorization).toBeUndefined() }) it('drops unusable rows rather than failing the whole listing', async () => { const server = await listingServer({ body: JSON.stringify({ data: [ { id: 'good' }, { id: '' }, { name: 'no id at all' }, null, { id: 'good' }, { id: 'zero-capacity', context_length: 0, max_tokens: -1 }, ], }), }) const ctx = await harness() expect(await ctx.llm.discoverModels('llm-pi-ai', { baseURL: server.url })) .toEqual([{ id: 'good' }, { id: 'zero-capacity' }]) }) it('points at the credential for a rejected one, and only then', async () => { const ctx = await harness() for (const status of [401, 403]) { const refused = await listingServer({ status, body: '{"error":"nope"}' }) await expect(ctx.llm.discoverModels('llm-pi-ai', { baseURL: refused.url, apiKey: 'wrong' })) .rejects.toThrow(new RegExp(`answered ${status}; check the API key`)) } // A server fault is not a credential problem, so it must not send the user // off to re-check a key that is fine. const broken = await listingServer({ status: 500, body: '{"error":"boom"}' }) await expect(ctx.llm.discoverModels('llm-pi-ai', { baseURL: broken.url, apiKey: 'fine' })) .rejects.toThrow(/answered 500$/) }) it('reports a reply that is not a model listing', async () => { const server = await listingServer({ body: '{"models":[]}' }) const ctx = await harness() await expect(ctx.llm.discoverModels('llm-pi-ai', { baseURL: server.url })) .rejects.toThrow(/no "data" array; enter this provider's models by hand/) const broken = await listingServer({ body: 'not json at all' }) await expect(ctx.llm.discoverModels('llm-pi-ai', { baseURL: broken.url })) .rejects.toThrow(/did not answer with JSON/) }) it('refuses an oversized reply, whether its length is declared or streamed', async () => { const ctx = await harness() // Just over the four-megabyte ceiling, as one padded model row. const oversized = `{"data":[{"id":"m","pad":"${'x'.repeat(4 * 1024 * 1024)}"}]}` const declared = await listingServer({ body: oversized }) await expect(ctx.llm.discoverModels('llm-pi-ai', { baseURL: declared.url })) .rejects.toThrow(/answered with more than 4194304 bytes/) // A streamed reply declares no length, so the ceiling has to hold on the // body the harness actually read. const streamed = await listingServer({ chunks: ['{"data":[{"id":"m","pad":"', 'x'.repeat(4 * 1024 * 1024), '"}]}'] }) await expect(ctx.llm.discoverModels('llm-pi-ai', { baseURL: streamed.url })) .rejects.toThrow(/answered with more than 4194304 bytes/) }) it('reports an unreachable endpoint instead of an empty catalog', async () => { const ctx = await harness() // Port 9 is the discard service: nothing accepts a connection there. await expect(ctx.llm.discoverModels('llm-pi-ai', { baseURL: 'http://127.0.0.1:9/v1' })) .rejects.toMatchObject({ code: 'DISCOVERY_FAILED' }) }) it.each(['anthropic-messages', 'azure-openai-responses', 'openai-codex-responses', 'google-generative-ai'])( 'says it cannot interrogate %s rather than guessing a shape', async (api) => { // Azure authenticates with an `api-key` header and an `api-version` // query despite its OpenAI lineage, and Codex uses OAuth; guessing at // either would report an auth failure as a provider with no models. const ctx = await harness() await expect(ctx.llm.discoverModels('llm-pi-ai', { baseURL: 'https://gateway.example/v1', api })) .rejects.toMatchObject({ code: 'DISCOVERY_UNSUPPORTED' }) }, ) it('reports cancellation during the body read as an abort, not a raw reason', async () => { const ctx = await harness() const controller = new AbortController() // Chunked, so the headers arrive and the cancellation lands mid-body. const slow = await listingServer({ chunks: ['{"data":[', '{"id":"a"}'], holdOpenMs: 400 }) const probe = ctx.llm.discoverModels('llm-pi-ai', { baseURL: slow.url, signal: controller.signal }) setTimeout(() => { controller.abort('test cancellation') }, 40) await expect(probe).rejects.toMatchObject({ code: 'ABORTED' }) }) it('honors caller cancellation', async () => { const ctx = await harness() const aborted = AbortSignal.abort('test cancellation') await expect(ctx.llm.discoverModels('llm-pi-ai', { baseURL: 'http://127.0.0.1:9/v1', signal: aborted, })).rejects.toMatchObject({ code: 'ABORTED' }) }) it('is offered for the namespace, and refuses one it does not serve', async () => { const ctx = await harness() expect(ctx.llm.listModelDiscoveryNamespaces()).toEqual(['llm-pi-ai']) await expect(ctx.llm.discoverModels('llm-deepseek', { baseURL: 'https://api.deepseek.com' })) .rejects.toMatchObject({ code: 'NO_DISCOVERY' }) await expect(ctx.llm.discoverModels('llm-pi-ai', { baseURL: '' })) .rejects.toMatchObject({ code: 'INVALID_DISCOVERY' }) }) it('withdraws the offer when the plugin unloads', async () => { const ctx = new Context() await ctx.plugin(LlmService) const fiber = await ctx.plugin(LlmPiAi, {}) expect(ctx.llm.listModelDiscoveryNamespaces()).toEqual(['llm-pi-ai']) await fiber.dispose() expect(ctx.llm.listModelDiscoveryNamespaces()).toEqual([]) }) })