Add web multimodal image attachments

This commit is contained in:
Yichen Jiang
2026-07-23 15:20:47 +08:00
parent 3e3ea47296
commit cb4c11b869
116 changed files with 3177 additions and 151 deletions

View File

@@ -116,6 +116,8 @@ export class DeepSeekAdapter extends LlmAdapter {
id: model.id,
name: model.name ?? model.id,
...model.description === undefined ? {} : { description: model.description },
inputModalities: ['text'],
outputModalities: ['text'],
})))
}

View File

@@ -2,10 +2,12 @@
* Serialize harness messages into DeepSeek chat completions. User text is joined; assistant text
* becomes `content`, tool calls become `tool_calls`, and tool results become separate tool messages.
* Assistant reasoning is replayed as `reasoning_content` only on tool-call turns, as required by
* thinking-mode passback. Unknown declaration-merged block types are skipped rather than rejected.
* thinking-mode passback. Core image blocks are rejected explicitly because this wire route is text-only;
* unknown declaration-merged block types retain the adapter's documented extension fallback.
* @module dsh-llm-deepseek/serialize
*/
import { LlmError } from '@deepseek-ai/dsh-llm'
import type { ContentBlock, GenerateOptions, Message } from '@deepseek-ai/dsh-llm'
import type { WireMessage, WireRequest, WireTool } from './types.ts'
@@ -23,6 +25,16 @@ function flattenText(blocks: ContentBlock[]): string {
.join('')
}
/** Reject core image content before any text-flattening path can silently erase it. */
function assertTextOnly(blocks: readonly ContentBlock[]): void {
for (const block of blocks) {
if (block.type === 'image') {
throw new LlmError('The DeepSeek chat-completions adapter does not support image content.', 'UNSUPPORTED_CONTENT')
}
if (block.type === 'tool-result') assertTextOnly(block.content)
}
}
/** Serialize one assistant message (text + reasoning + tool calls). */
function serializeAssistant(message: Message): WireMessage {
const text = flattenText(message.content)
@@ -68,6 +80,7 @@ function serializeAssistant(message: Message): WireMessage {
export function serializeMessages(messages: Message[]): WireMessage[] {
const wire: WireMessage[] = []
for (const message of messages) {
assertTextOnly(message.content)
if (message.role === 'system') {
wire.push({ role: 'system', content: flattenText(message.content) })
continue

View File

@@ -528,8 +528,8 @@ describe('plugin registration and config', () => {
await ctx.plugin(LlmDeepSeek, { apiKey: 'k', baseURL: 'http://127.0.0.1:1' })
expect(ctx.llm.listProviders()).toEqual([{ id: 'deepseek', name: 'DeepSeek' }])
await expect(ctx.llm.listModels('deepseek')).resolves.toEqual([
{ provider: 'deepseek', id: 'deepseek-v4-flash', name: 'deepseek-v4-flash' },
{ provider: 'deepseek', id: 'deepseek-v4-pro', name: 'deepseek-v4-pro' },
{ provider: 'deepseek', id: 'deepseek-v4-flash', name: 'deepseek-v4-flash', inputModalities: ['text'], outputModalities: ['text'] },
{ provider: 'deepseek', id: 'deepseek-v4-pro', name: 'deepseek-v4-pro', inputModalities: ['text'], outputModalities: ['text'] },
])
await expect(ctx.llm.resolveModelContext('deepseek', 'deepseek-v4-flash'))
.resolves.toEqual({ contextWindow: 128_000 })
@@ -540,8 +540,8 @@ describe('plugin registration and config', () => {
await ctx.plugin(LlmService)
LlmDeepSeek.apply(ctx, { apiKey: 'k', baseURL: 'http://127.0.0.1:1' })
await expect(ctx.llm.listModels('deepseek')).resolves.toEqual([
{ provider: 'deepseek', id: 'deepseek-v4-flash', name: 'deepseek-v4-flash' },
{ provider: 'deepseek', id: 'deepseek-v4-pro', name: 'deepseek-v4-pro' },
{ provider: 'deepseek', id: 'deepseek-v4-flash', name: 'deepseek-v4-flash', inputModalities: ['text'], outputModalities: ['text'] },
{ provider: 'deepseek', id: 'deepseek-v4-pro', name: 'deepseek-v4-pro', inputModalities: ['text'], outputModalities: ['text'] },
])
})
@@ -562,8 +562,8 @@ describe('plugin registration and config', () => {
],
})
await expect(ctx.llm.listModels('deepseek')).resolves.toEqual([
{ provider: 'deepseek', id: 'private-fast', name: 'private-fast' },
{ provider: 'deepseek', id: 'private-reasoner', name: 'Private Reasoner', description: 'Higher reasoning budget' },
{ provider: 'deepseek', id: 'private-fast', name: 'private-fast', inputModalities: ['text'], outputModalities: ['text'] },
{ provider: 'deepseek', id: 'private-reasoner', name: 'Private Reasoner', description: 'Higher reasoning budget', inputModalities: ['text'], outputModalities: ['text'] },
])
await expect(ctx.llm.resolveModelContext('deepseek', 'private-fast'))
.resolves.toEqual({ contextWindow: 32_000 })

View File

@@ -1,5 +1,6 @@
import { describe, expect, it } from 'vitest'
import { CallId } from '@deepseek-ai/dsh-llm'
import { AttachmentId } from '@deepseek-ai/dsh-attachment'
import type { ContentBlock, GenerateOptions, Message } from '@deepseek-ai/dsh-llm'
import { serializeMessages, serializeRequest } from '../src/serialize.ts'
@@ -123,6 +124,19 @@ describe('serializeMessages', () => {
expect(wire).toEqual([{ role: 'user', content: 'see chart' }])
})
it('rejects image blocks instead of silently flattening them away', () => {
expect(() => serializeMessages([{
role: 'user',
content: [{
type: 'image',
attachment: {
attachmentId: AttachmentId(`sha256:${'a'.repeat(64)}`),
mediaType: 'image/png', bytes: 68, width: 1, height: 1,
},
}],
}])).toThrow(expect.objectContaining({ code: 'UNSUPPORTED_CONTENT' }))
})
it('emits an empty user message rather than dropping block-less messages', () => {
const wire = serializeMessages([{ role: 'user', content: [] }])
expect(wire).toEqual([{ role: 'user', content: '' }])