Add web multimodal image attachments

This commit is contained in:
Yichen Jiang
2026-07-23 15:20:47 +08:00
parent 3e3ea47296
commit cb4c11b869
116 changed files with 3177 additions and 151 deletions

View File

@@ -528,8 +528,8 @@ describe('plugin registration and config', () => {
await ctx.plugin(LlmDeepSeek, { apiKey: 'k', baseURL: 'http://127.0.0.1:1' })
expect(ctx.llm.listProviders()).toEqual([{ id: 'deepseek', name: 'DeepSeek' }])
await expect(ctx.llm.listModels('deepseek')).resolves.toEqual([
{ provider: 'deepseek', id: 'deepseek-v4-flash', name: 'deepseek-v4-flash' },
{ provider: 'deepseek', id: 'deepseek-v4-pro', name: 'deepseek-v4-pro' },
{ provider: 'deepseek', id: 'deepseek-v4-flash', name: 'deepseek-v4-flash', inputModalities: ['text'], outputModalities: ['text'] },
{ provider: 'deepseek', id: 'deepseek-v4-pro', name: 'deepseek-v4-pro', inputModalities: ['text'], outputModalities: ['text'] },
])
await expect(ctx.llm.resolveModelContext('deepseek', 'deepseek-v4-flash'))
.resolves.toEqual({ contextWindow: 128_000 })
@@ -540,8 +540,8 @@ describe('plugin registration and config', () => {
await ctx.plugin(LlmService)
LlmDeepSeek.apply(ctx, { apiKey: 'k', baseURL: 'http://127.0.0.1:1' })
await expect(ctx.llm.listModels('deepseek')).resolves.toEqual([
{ provider: 'deepseek', id: 'deepseek-v4-flash', name: 'deepseek-v4-flash' },
{ provider: 'deepseek', id: 'deepseek-v4-pro', name: 'deepseek-v4-pro' },
{ provider: 'deepseek', id: 'deepseek-v4-flash', name: 'deepseek-v4-flash', inputModalities: ['text'], outputModalities: ['text'] },
{ provider: 'deepseek', id: 'deepseek-v4-pro', name: 'deepseek-v4-pro', inputModalities: ['text'], outputModalities: ['text'] },
])
})
@@ -562,8 +562,8 @@ describe('plugin registration and config', () => {
],
})
await expect(ctx.llm.listModels('deepseek')).resolves.toEqual([
{ provider: 'deepseek', id: 'private-fast', name: 'private-fast' },
{ provider: 'deepseek', id: 'private-reasoner', name: 'Private Reasoner', description: 'Higher reasoning budget' },
{ provider: 'deepseek', id: 'private-fast', name: 'private-fast', inputModalities: ['text'], outputModalities: ['text'] },
{ provider: 'deepseek', id: 'private-reasoner', name: 'Private Reasoner', description: 'Higher reasoning budget', inputModalities: ['text'], outputModalities: ['text'] },
])
await expect(ctx.llm.resolveModelContext('deepseek', 'private-fast'))
.resolves.toEqual({ contextWindow: 32_000 })

View File

@@ -1,5 +1,6 @@
import { describe, expect, it } from 'vitest'
import { CallId } from '@deepseek-ai/dsh-llm'
import { AttachmentId } from '@deepseek-ai/dsh-attachment'
import type { ContentBlock, GenerateOptions, Message } from '@deepseek-ai/dsh-llm'
import { serializeMessages, serializeRequest } from '../src/serialize.ts'
@@ -123,6 +124,19 @@ describe('serializeMessages', () => {
expect(wire).toEqual([{ role: 'user', content: 'see chart' }])
})
it('rejects image blocks instead of silently flattening them away', () => {
expect(() => serializeMessages([{
role: 'user',
content: [{
type: 'image',
attachment: {
attachmentId: AttachmentId(`sha256:${'a'.repeat(64)}`),
mediaType: 'image/png', bytes: 68, width: 1, height: 1,
},
}],
}])).toThrow(expect.objectContaining({ code: 'UNSUPPORTED_CONTENT' }))
})
it('emits an empty user message rather than dropping block-less messages', () => {
const wire = serializeMessages([{ role: 'user', content: [] }])
expect(wire).toEqual([{ role: 'user', content: '' }])