Add web multimodal image attachments
This commit is contained in:
@@ -116,6 +116,8 @@ export class DeepSeekAdapter extends LlmAdapter {
|
||||
id: model.id,
|
||||
name: model.name ?? model.id,
|
||||
...model.description === undefined ? {} : { description: model.description },
|
||||
inputModalities: ['text'],
|
||||
outputModalities: ['text'],
|
||||
})))
|
||||
}
|
||||
|
||||
|
||||
@@ -2,10 +2,12 @@
|
||||
* Serialize harness messages into DeepSeek chat completions. User text is joined; assistant text
|
||||
* becomes `content`, tool calls become `tool_calls`, and tool results become separate tool messages.
|
||||
* Assistant reasoning is replayed as `reasoning_content` only on tool-call turns, as required by
|
||||
* thinking-mode passback. Unknown declaration-merged block types are skipped rather than rejected.
|
||||
* thinking-mode passback. Core image blocks are rejected explicitly because this wire route is text-only;
|
||||
* unknown declaration-merged block types retain the adapter's documented extension fallback.
|
||||
* @module dsh-llm-deepseek/serialize
|
||||
*/
|
||||
|
||||
import { LlmError } from '@deepseek-ai/dsh-llm'
|
||||
import type { ContentBlock, GenerateOptions, Message } from '@deepseek-ai/dsh-llm'
|
||||
import type { WireMessage, WireRequest, WireTool } from './types.ts'
|
||||
|
||||
@@ -23,6 +25,16 @@ function flattenText(blocks: ContentBlock[]): string {
|
||||
.join('')
|
||||
}
|
||||
|
||||
/** Reject core image content before any text-flattening path can silently erase it. */
|
||||
function assertTextOnly(blocks: readonly ContentBlock[]): void {
|
||||
for (const block of blocks) {
|
||||
if (block.type === 'image') {
|
||||
throw new LlmError('The DeepSeek chat-completions adapter does not support image content.', 'UNSUPPORTED_CONTENT')
|
||||
}
|
||||
if (block.type === 'tool-result') assertTextOnly(block.content)
|
||||
}
|
||||
}
|
||||
|
||||
/** Serialize one assistant message (text + reasoning + tool calls). */
|
||||
function serializeAssistant(message: Message): WireMessage {
|
||||
const text = flattenText(message.content)
|
||||
@@ -68,6 +80,7 @@ function serializeAssistant(message: Message): WireMessage {
|
||||
export function serializeMessages(messages: Message[]): WireMessage[] {
|
||||
const wire: WireMessage[] = []
|
||||
for (const message of messages) {
|
||||
assertTextOnly(message.content)
|
||||
if (message.role === 'system') {
|
||||
wire.push({ role: 'system', content: flattenText(message.content) })
|
||||
continue
|
||||
|
||||
@@ -528,8 +528,8 @@ describe('plugin registration and config', () => {
|
||||
await ctx.plugin(LlmDeepSeek, { apiKey: 'k', baseURL: 'http://127.0.0.1:1' })
|
||||
expect(ctx.llm.listProviders()).toEqual([{ id: 'deepseek', name: 'DeepSeek' }])
|
||||
await expect(ctx.llm.listModels('deepseek')).resolves.toEqual([
|
||||
{ provider: 'deepseek', id: 'deepseek-v4-flash', name: 'deepseek-v4-flash' },
|
||||
{ provider: 'deepseek', id: 'deepseek-v4-pro', name: 'deepseek-v4-pro' },
|
||||
{ provider: 'deepseek', id: 'deepseek-v4-flash', name: 'deepseek-v4-flash', inputModalities: ['text'], outputModalities: ['text'] },
|
||||
{ provider: 'deepseek', id: 'deepseek-v4-pro', name: 'deepseek-v4-pro', inputModalities: ['text'], outputModalities: ['text'] },
|
||||
])
|
||||
await expect(ctx.llm.resolveModelContext('deepseek', 'deepseek-v4-flash'))
|
||||
.resolves.toEqual({ contextWindow: 128_000 })
|
||||
@@ -540,8 +540,8 @@ describe('plugin registration and config', () => {
|
||||
await ctx.plugin(LlmService)
|
||||
LlmDeepSeek.apply(ctx, { apiKey: 'k', baseURL: 'http://127.0.0.1:1' })
|
||||
await expect(ctx.llm.listModels('deepseek')).resolves.toEqual([
|
||||
{ provider: 'deepseek', id: 'deepseek-v4-flash', name: 'deepseek-v4-flash' },
|
||||
{ provider: 'deepseek', id: 'deepseek-v4-pro', name: 'deepseek-v4-pro' },
|
||||
{ provider: 'deepseek', id: 'deepseek-v4-flash', name: 'deepseek-v4-flash', inputModalities: ['text'], outputModalities: ['text'] },
|
||||
{ provider: 'deepseek', id: 'deepseek-v4-pro', name: 'deepseek-v4-pro', inputModalities: ['text'], outputModalities: ['text'] },
|
||||
])
|
||||
})
|
||||
|
||||
@@ -562,8 +562,8 @@ describe('plugin registration and config', () => {
|
||||
],
|
||||
})
|
||||
await expect(ctx.llm.listModels('deepseek')).resolves.toEqual([
|
||||
{ provider: 'deepseek', id: 'private-fast', name: 'private-fast' },
|
||||
{ provider: 'deepseek', id: 'private-reasoner', name: 'Private Reasoner', description: 'Higher reasoning budget' },
|
||||
{ provider: 'deepseek', id: 'private-fast', name: 'private-fast', inputModalities: ['text'], outputModalities: ['text'] },
|
||||
{ provider: 'deepseek', id: 'private-reasoner', name: 'Private Reasoner', description: 'Higher reasoning budget', inputModalities: ['text'], outputModalities: ['text'] },
|
||||
])
|
||||
await expect(ctx.llm.resolveModelContext('deepseek', 'private-fast'))
|
||||
.resolves.toEqual({ contextWindow: 32_000 })
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { CallId } from '@deepseek-ai/dsh-llm'
|
||||
import { AttachmentId } from '@deepseek-ai/dsh-attachment'
|
||||
import type { ContentBlock, GenerateOptions, Message } from '@deepseek-ai/dsh-llm'
|
||||
import { serializeMessages, serializeRequest } from '../src/serialize.ts'
|
||||
|
||||
@@ -123,6 +124,19 @@ describe('serializeMessages', () => {
|
||||
expect(wire).toEqual([{ role: 'user', content: 'see chart' }])
|
||||
})
|
||||
|
||||
it('rejects image blocks instead of silently flattening them away', () => {
|
||||
expect(() => serializeMessages([{
|
||||
role: 'user',
|
||||
content: [{
|
||||
type: 'image',
|
||||
attachment: {
|
||||
attachmentId: AttachmentId(`sha256:${'a'.repeat(64)}`),
|
||||
mediaType: 'image/png', bytes: 68, width: 1, height: 1,
|
||||
},
|
||||
}],
|
||||
}])).toThrow(expect.objectContaining({ code: 'UNSUPPORTED_CONTENT' }))
|
||||
})
|
||||
|
||||
it('emits an empty user message rather than dropping block-less messages', () => {
|
||||
const wire = serializeMessages([{ role: 'user', content: [] }])
|
||||
expect(wire).toEqual([{ role: 'user', content: '' }])
|
||||
|
||||
Reference in New Issue
Block a user