Coder/deepseek-harness
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(gui): harden multimodal image attachments

Browse Source
This commit is contained in:
Yichen Jiang
2026-07-23 19:38:37 +08:00
parent eea595fcb4
commit 580e05b794
61 changed files with 1700 additions and 214 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
packages/llm/llm-pi-ai/README.md
View File

@@ -34,6 +34,8 @@ Supported profile fields are `provider`, `apiKey`, `baseURL`, `headers`, `reason
The adapter forces pi-ai's SDK `maxRetries` to zero so one `stream()` call makes one provider request. The removed profile fields `maxRetries` and `maxRetryDelayMs` fail load instead of silently multiplying or hiding the separately composed agent-level retry budget. Idle expiry aborts the SDK's stable request signal and surfaces `TIMEOUT`; an earlier caller abort remains `ABORTED`.
Image requests resolve the optional `ctx.attachments` service when the request is dispatched, so Cordis plugin load order does not freeze attachment availability. A visual request still fails explicitly with `UNSUPPORTED_CONTENT` when the service or the selected model's image capability is absent.
## Provider/model routing and replay
The selected pi-ai catalog descriptor supplies the protocol implementation. This includes native API differences such as OpenAI models whose descriptor uses the Responses API rather than Chat Completions; the harness adapter does not hardcode endpoint selection by model name.

15
packages/llm/llm-pi-ai/src/adapter.ts
View File

@@ -25,8 +25,8 @@ import { toStreamChunks } from './stream.ts'
export interface PiAiAdapterOptions {
/** Validated provider profiles this adapter instance owns. */
profiles: readonly PiAiProviderProfile[]
/** Durable image resolver used only when a request contains image references. */
attachments?: AttachmentStore
/** Resolve durable image storage at request time so plugin load order does not become capability state. */
resolveAttachments?: () => AttachmentStore | undefined
}
/**
@@ -72,12 +72,12 @@ function requestHeaders(headers: Readonly<Record<string, string>> | undefined):
*/
export class PiAiAdapter extends LlmAdapter {
private readonly profiles: ReadonlyMap<string, ResolvedPiAiProviderProfile>
private readonly attachments: AttachmentStore | undefined
private readonly resolveAttachments: () => AttachmentStore | undefined
constructor(options: PiAiAdapterOptions) {
super()
this.profiles = new Map(resolveProfiles(options.profiles).map(profile => [profile.provider, profile]))
this.attachments = options.attachments
this.resolveAttachments = options.resolveAttachments ?? (() => undefined)
}
override listModels(provider: string): Promise<readonly LlmModelInfo[]> {
@@ -136,12 +136,13 @@ export class PiAiAdapter extends LlmAdapter {
if (containsImage && !model.input.includes('image')) {
throw new LlmError(`pi-ai model "${model.id}" does not support image input`, 'UNSUPPORTED_CONTENT')
}
if (containsImage && this.attachments === undefined) {
const attachments = containsImage ? this.resolveAttachments() : undefined
if (containsImage && attachments === undefined) {
throw new LlmError('pi-ai image input requires the durable attachment service', 'UNSUPPORTED_CONTENT')
}
const context = this.attachments === undefined
const context = attachments === undefined
? toPiContext(options)
: await toPiContext(options, this.attachments)
: await toPiContext(options, attachments)
const events = streamSimple(model, context, {
...profileOptions(profile),
...options.temperature === undefined ? {} : { temperature: options.temperature },

3
packages/llm/llm-pi-ai/src/index.ts
View File

@@ -36,10 +36,9 @@ export const inject = ['llm']
/** Register one generic pi-ai adapter for all configured provider routes. */
export function apply(ctx: Context, config: Config): void {
const profiles = resolveProfiles(config.providers)
const attachments = ctx.get('attachments')
const adapter = new PiAiAdapter({
profiles,
...(attachments === undefined ? {} : { attachments }),
resolveAttachments: () => ctx.get('attachments'),
})
ctx.llm.registerAdapter(profiles.map(entry => entry.provider), adapter)
}

96
packages/llm/llm-pi-ai/tests/adapter.spec.ts
View File

@@ -2,6 +2,13 @@ import { createServer } from 'node:http'
import type { IncomingMessage, Server, ServerResponse } from 'node:http'
import { afterEach, describe, expect, it, vi } from 'vitest'
import { Context } from 'cordis'
import { AttachmentId, AttachmentStore } from '@deepseek-ai/dsh-attachment'
import type {
ImageAttachmentLimits,
ImageAttachmentRef,
SaveImageAttachment,
StoredImageAttachment,
} from '@deepseek-ai/dsh-attachment'
import LlmService, { CONTEXT_WINDOW_EXCEEDED_CODE, LlmError, userAgent } from '@deepseek-ai/dsh-llm'
import * as LlmPiAi from '@deepseek-ai/dsh-llm-pi-ai'
import { PiAiAdapter } from '@deepseek-ai/dsh-llm-pi-ai'
@@ -88,6 +95,14 @@ const textEvents = [
'[DONE]',
]
const IMAGE_REF: ImageAttachmentRef = {
attachmentId: AttachmentId(`sha256:${'a'.repeat(64)}`),
mediaType: 'image/png',
bytes: 1,
width: 1,
height: 1,
}
async function harness(baseURL: string, overrides: Record<string, unknown> = {}): Promise<Context> {
const ctx = new Context()
await ctx.plugin(LlmService)
@@ -189,6 +204,55 @@ describe('PiAiAdapter provider routing', () => {
expect(server.paths).toEqual(['/v1/responses'])
})
it('resolves an attachment service mounted after the adapter when dispatching an image', async () => {
const server = await mockServer([{ status: 401, body: JSON.stringify({ error: { message: 'expected mock failure' } }) }])
const attachmentId = AttachmentId(`sha256:${'a'.repeat(64)}`)
const ref: ImageAttachmentRef = {
attachmentId,
mediaType: 'image/png',
bytes: 1,
width: 1,
height: 1,
}
const readImage = vi.fn((_ref: ImageAttachmentRef): Promise<StoredImageAttachment> =>
Promise.resolve({ ref, data: Uint8Array.of(1) }))
class LateAttachmentStore extends AttachmentStore {
readonly imageLimits: ImageAttachmentLimits = {
maxImageBytes: 1,
maxImagesPerMessage: 1,
maxMessageImageBytes: 1,
maxImagePixels: 1,
mediaTypes: ['image/png'],
}
saveImage(_input: SaveImageAttachment): Promise<ImageAttachmentRef> {
return Promise.reject(new Error('not used'))
}
readImage(value: ImageAttachmentRef): Promise<StoredImageAttachment> {
return readImage(value)
}
}
const ctx = new Context()
await ctx.plugin(LlmService)
await ctx.plugin(LlmPiAi, {
providers: [{ provider: 'openai', apiKey: 'test-key', baseURL: `${server.url}/v1` }],
})
await ctx.plugin(LateAttachmentStore)
const result = await assemble(ctx, {
provider: 'openai',
model: 'gpt-4.1',
messages: [{ role: 'user', content: [{ type: 'image', attachment: ref }] }],
})
expect(result.finish.kind).toBe('error')
expect(readImage).toHaveBeenCalledWith(ref)
expect(server.paths).toEqual(['/v1/responses'])
})
it('forces one wire request for an SDK-retryable provider failure', async () => {
const server = await mockServer([
{
@@ -387,6 +451,38 @@ describe('provider profile lifecycle', () => {
expect(new LlmError('x', 'X')).toBeInstanceOf(Error)
})
it('rejects unsupported or unresolved image input before provider I/O', async () => {
const adapter = new PiAiAdapter({
profiles: [{ provider: 'openai' }, { provider: 'deepseek' }],
})
const drain = async (options: Parameters<PiAiAdapter['stream']>[0]): Promise<void> => {
for await (const _chunk of adapter.stream(options)) { /* drain */ }
}
await expect(drain({
provider: 'deepseek',
model: 'deepseek-v4-flash',
messages: [{ role: 'user', content: [{ type: 'image', attachment: IMAGE_REF }] }],
})).rejects.toMatchObject({ code: 'UNSUPPORTED_CONTENT' })
await expect(drain({
provider: 'openai',
model: 'gpt-4.1',
messages: [{ role: 'user', content: [{ type: 'image', attachment: IMAGE_REF }] }],
})).rejects.toMatchObject({ code: 'UNSUPPORTED_CONTENT' })
await expect(drain({
provider: 'openai',
model: 'gpt-4.1',
messages: [{
role: 'user',
content: [{
type: 'tool-result',
toolCallId: 'call-image' as never,
content: [{ type: 'image', attachment: IMAGE_REF }],
}],
}],
})).rejects.toMatchObject({ code: 'UNSUPPORTED_CONTENT' })
})
it('validates direct-constructor profiles at the embedding boundary', () => {
expect(() => new PiAiAdapter({
profiles: [{ provider: 'openai', streamIdleTimeoutMs: 0 }],

207
packages/llm/llm-pi-ai/tests/context.spec.ts Normal file
View File

@@ -0,0 +1,207 @@
import { describe, expect, it, vi } from 'vitest'
import { AttachmentId } from '@deepseek-ai/dsh-attachment'
import type { AttachmentStore, ImageAttachmentRef } from '@deepseek-ai/dsh-attachment'
import { CallId } from '@deepseek-ai/dsh-llm'
import type { GenerateOptions } from '@deepseek-ai/dsh-llm'
import { toPiContext } from '../src/context.ts'
import { toPiAssistant } from '../src/replay.ts'
const ref: ImageAttachmentRef = {
attachmentId: AttachmentId(`sha256:${'a'.repeat(64)}`),
mediaType: 'image/png',
bytes: 1,
width: 1,
height: 1,
}
const attachments = {
readImage: vi.fn(() => Promise.resolve({ ref, data: Uint8Array.of(1) })),
} as unknown as AttachmentStore
function request(messages: GenerateOptions['messages']): GenerateOptions {
return {
provider: 'openai',
model: 'gpt-4.1',
system: 'system prompt',
tools: [{ name: 'lookup', description: 'look up', parameters: { type: 'object' } }],
messages,
}
}
describe('pi-ai request context conversion', () => {
it('omits absent and empty request-level optional fields', () => {
const base = { provider: 'openai', model: 'gpt-4.1', messages: [] }
expect(toPiContext(base)).toEqual({ messages: [] })
expect(toPiContext({ ...base, tools: [] })).toEqual({ messages: [] })
})
it('converts complete text-only history and rejects nested images without storage', () => {
const callId = CallId('call-1')
expect(toPiContext(request([
{ role: 'system', content: [{ type: 'text', text: 'history system' }] },
{
role: 'assistant',
content: [{ type: 'tool-call', id: callId, name: 'lookup', arguments: '{}' }],
},
{
role: 'user',
content: [
{ type: 'text', text: 'after tool' },
{
type: 'tool-result',
toolCallId: callId,
content: [{ type: 'text', text: '' }],
},
],
},
]))).toMatchObject({
systemPrompt: 'system prompt',
tools: [{ name: 'lookup' }],
messages: [
{ role: 'user', content: 'history system' },
{ role: 'assistant' },
{ role: 'user', content: 'after tool' },
{
role: 'toolResult',
toolCallId: 'call-1',
toolName: 'lookup',
content: [{ type: 'text', text: '(no output)' }],
isError: false,
},
],
})
expect(() => toPiContext(request([{
role: 'user',
content: [{
type: 'tool-result',
toolCallId: callId,
content: [{ type: 'image', attachment: ref }],
}],
}]))).toThrow(/durable attachment service/)
})
it('resolves user and tool-result images while preserving explicit fallbacks', async () => {
const callId = CallId('missing-call')
const knownCallId = CallId('known-call')
const context = await toPiContext(request([
{ role: 'user', content: [{ type: 'text', text: '' }] },
{
role: 'assistant',
content: [
{ type: 'text', text: 'calling' },
{ type: 'tool-call', id: knownCallId, name: 'lookup', arguments: '{}' },
],
},
{
role: 'user',
content: [
{ type: 'image', attachment: ref },
{ type: 'text', text: 'caption' },
{ type: 'reasoning', text: 'ignored' },
],
},
{
role: 'user',
content: [{
type: 'tool-result',
toolCallId: knownCallId,
content: [{ type: 'text', text: '' }],
}],
},
{
role: 'user',
content: [{
type: 'tool-result',
toolCallId: callId,
isError: true,
content: [
{ type: 'tool-result', toolCallId: callId, content: [] },
{ type: 'image', attachment: ref },
],
}],
},
]), attachments)
expect(context.messages).toEqual([
{ role: 'user', content: '', timestamp: 0 },
expect.objectContaining({ role: 'assistant' }),
{
role: 'user',
content: [
{ type: 'image', data: 'AQ==', mimeType: 'image/png' },
{ type: 'text', text: 'caption' },
],
timestamp: 0,
},
{
role: 'toolResult',
toolCallId: 'known-call',
toolName: 'lookup',
content: [{ type: 'text', text: '(no output)' }],
isError: false,
timestamp: 0,
},
{
role: 'toolResult',
toolCallId: 'missing-call',
toolName: 'unknown',
content: [{ type: 'image', data: 'AQ==', mimeType: 'image/png' }],
isError: true,
timestamp: 0,
},
])
})
it('keeps empty text-only users while separating result-only messages', () => {
const callId = CallId('unknown-call')
expect(toPiContext(request([
{ role: 'user', content: [] },
{
role: 'assistant',
content: [
{ type: 'text', text: 'answer' },
{ type: 'tool-call', id: CallId('other-call'), name: 'lookup', arguments: '{}' },
],
},
{
role: 'user',
content: [{
type: 'tool-result',
toolCallId: callId,
content: [{ type: 'text', text: 'result' }],
}],
},
]))).toMatchObject({
messages: [
{ role: 'user', content: '' },
{ role: 'assistant' },
{ role: 'toolResult', toolName: 'unknown' },
],
})
})
it('handles in-history system and assistant messages explicitly on the image path', async () => {
await expect(toPiContext(request([{
role: 'system',
content: [{ type: 'image', attachment: ref }],
}]), attachments)).rejects.toMatchObject({ code: 'UNSUPPORTED_CONTENT' })
await expect(toPiContext(request([
{ role: 'system', content: [{ type: 'text', text: 'history system' }] },
{ role: 'assistant', content: [{ type: 'text', text: 'answer' }] },
{ role: 'user', content: [{ type: 'text', text: 'plain' }] },
]), attachments)).resolves.toMatchObject({
messages: [
{ role: 'user', content: 'history system' },
{ role: 'assistant' },
{ role: 'user', content: 'plain' },
],
})
expect(() => toPiAssistant({
role: 'assistant',
content: [{ type: 'image', attachment: ref }],
})).toThrow(/assistant image output/)
})
})

74
packages/llm/llm-pi-ai/tests/provider-apis.e2e.ts
View File

@@ -1,5 +1,13 @@
import { readFile } from 'node:fs/promises'
import { afterEach, describe, expect, it } from 'vitest'
import { Context } from 'cordis'
import { AttachmentId, AttachmentStore } from '@deepseek-ai/dsh-attachment'
import type {
ImageAttachmentLimits,
ImageAttachmentRef,
SaveImageAttachment,
StoredImageAttachment,
} from '@deepseek-ai/dsh-attachment'
import LlmService, { CallId } from '@deepseek-ai/dsh-llm'
import type { Message, ToolSchema } from '@deepseek-ai/dsh-llm'
import * as LlmPiAi from '@deepseek-ai/dsh-llm-pi-ai'
@@ -17,6 +25,8 @@ interface ProviderCase {
const openAIBaseURL = process.env.DSH_PI_AI_OPENAI_BASE_URL
const azureOpenAIKey = process.env.AZURE_OPENAI_API_KEY
const anthropicApiKey = process.env.ANTHROPIC_API_KEY ?? process.env.DEEPSEEK_API_KEY
const anthropicBaseURL = process.env.DSH_PI_AI_ANTHROPIC_BASE_URL ?? process.env.DEEPSEEK_BASE_URL
const providerCases: ProviderCase[] = [
{
@@ -32,13 +42,14 @@ const providerCases: ProviderCase[] = [
provider: 'anthropic',
api: 'anthropic-messages',
model: process.env.DSH_PI_AI_ANTHROPIC_MODEL ?? 'claude-opus-4-8',
...process.env.ANTHROPIC_API_KEY ? { apiKey: process.env.ANTHROPIC_API_KEY } : {},
...anthropicApiKey === undefined ? {} : { apiKey: anthropicApiKey },
...anthropicBaseURL === undefined ? {} : { baseURL: anthropicBaseURL },
},
]
const contexts: Context[] = []
async function harness(): Promise<Context> {
async function harness(image?: StoredImageAttachment): Promise<Context> {
const ctx = new Context()
contexts.push(ctx)
await ctx.plugin(LlmService)
@@ -50,6 +61,30 @@ async function harness(): Promise<Context> {
...profile.headers === undefined ? {} : { headers: profile.headers },
})),
})
if (image !== undefined) {
const fixture = image
class E2eAttachmentStore extends AttachmentStore {
readonly imageLimits: ImageAttachmentLimits = {
maxImageBytes: fixture.data.byteLength,
maxImagesPerMessage: 1,
maxMessageImageBytes: fixture.data.byteLength,
maxImagePixels: fixture.ref.width * fixture.ref.height,
mediaTypes: [fixture.ref.mediaType],
}
saveImage(_input: SaveImageAttachment): Promise<ImageAttachmentRef> {
return Promise.reject(new Error('e2e attachment fixture is read-only'))
}
readImage(ref: ImageAttachmentRef): Promise<StoredImageAttachment> {
if (ref.attachmentId !== fixture.ref.attachmentId) {
return Promise.reject(new Error('unknown e2e attachment fixture'))
}
return Promise.resolve(fixture)
}
}
await ctx.plugin(E2eAttachmentStore)
}
return ctx
}
@@ -158,6 +193,41 @@ for (const profile of providerCases) {
expect(textOf(second).toLowerCase()).toContain('ocean')
expect(expectNativeReplay(second, profile).stopReason).toBe('stop')
})
if (profile.provider === 'anthropic') {
it('sends a real image through the authenticated Anthropic visual path', async () => {
const data = new Uint8Array(await readFile(
new URL('../../../../assets/community-wecom-survey.png', import.meta.url),
))
const ref: ImageAttachmentRef = {
attachmentId: AttachmentId(`sha256:${'a'.repeat(64)}`),
mediaType: 'image/png',
bytes: data.byteLength,
width: 256,
height: 256,
name: 'qr-code.png',
}
const ctx = await harness({ ref, data })
const result = await assemble(ctx, {
provider: profile.provider,
model: profile.model,
messages: [{
role: 'user',
content: [
{
type: 'text',
text: 'What type of machine-readable symbol is shown in the attached image? Reply with exactly: QR code',
},
{ type: 'image', attachment: ref, alt: 'machine-readable symbol' },
],
}],
maxTokens: 256,
})
expectFinish(result, 'stop')
expect(textOf(result).toLowerCase()).toContain('qr code')
})
}
},
)
}

13
packages/llm/token-meter/tests/token-meter.spec.ts
View File

@@ -1,5 +1,6 @@
import { describe, expect, expectTypeOf, it } from 'vitest'
import { Context } from 'cordis'
import { AttachmentId } from '@deepseek-ai/dsh-attachment'
import { CallId } from '@deepseek-ai/dsh-llm'
import type { ContentBlock, Message, TokenUsage } from '@deepseek-ai/dsh-llm'
import SessionStore, { Session, SessionId, canonicalHeader } from '@deepseek-ai/dsh-session'
@@ -116,6 +117,16 @@ describe('TokenMeterService pricing', () => {
const blocks: ContentBlock[] = [
{ type: 'text', text: 'abcd' },
{ type: 'reasoning', text: 'ab' },
{
type: 'image',
attachment: {
attachmentId: AttachmentId(`sha256:${'a'.repeat(64)}`),
mediaType: 'image/png',
bytes: 1,
width: 1024,
height: 513,
},
},
{ type: 'tool-call', id: CallId('c'), name: 'read', arguments: '{"x":1}' },
{
type: 'tool-result',
@@ -126,7 +137,7 @@ describe('TokenMeterService pricing', () => {
{ type: 'future-block', payload: 'abcd' } as unknown as ContentBlock,
]
const estimated = service.estimateMessage({ role: 'assistant', content: blocks })
expect(estimated).toBeGreaterThan(30)
expect(estimated).toBe(813)
expect(service.estimateMessage(textMessage('abcd'))).toBe(9)
})