Coder/deepseek-harness
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add web multimodal image attachments

Browse Source
This commit is contained in:
Yichen Jiang
2026-07-23 15:20:47 +08:00
parent 3e3ea47296
commit cb4c11b869
116 changed files with 3177 additions and 151 deletions
Download Patch File Download Diff File Expand all files Collapse all files

153
packages/llm/llm-pi-ai/src/context.ts
View File

@@ -4,9 +4,10 @@
* @module dsh-llm-pi-ai/context
*/
import { CallId } from '@deepseek-ai/dsh-llm'
import type { GenerateOptions, Message } from '@deepseek-ai/dsh-llm'
import type { Context as PiContext, Message as PiMessage, Tool as PiTool } from '@earendil-works/pi-ai'
import { CallId, LlmError } from '@deepseek-ai/dsh-llm'
import type { ContentBlock, GenerateOptions, Message } from '@deepseek-ai/dsh-llm'
import type { AttachmentStore } from '@deepseek-ai/dsh-attachment'
import type { Context as PiContext, ImageContent, Message as PiMessage, TextContent, Tool as PiTool } from '@earendil-works/pi-ai'
import { toPiAssistant } from './replay.ts'
/** Join the text blocks of a harness message. */
@@ -17,20 +18,122 @@ function flattenText(message: Message): string {
.join('')
}
async function userContent(
blocks: readonly ContentBlock[],
attachments: AttachmentStore,
): Promise<string | (TextContent | ImageContent)[]> {
const content: (TextContent | ImageContent)[] = []
for (const block of blocks) {
switch (block.type) {
case 'text':
if (block.text.length > 0) content.push({ type: 'text', text: block.text })
break
case 'image': {
const stored = await attachments.readImage(block.attachment)
content.push({
type: 'image',
data: Buffer.from(stored.data).toString('base64'),
mimeType: stored.ref.mediaType,
})
break
}
case 'tool-result':
break
default:
// Other merge-extensible blocks are not user-input vocabulary for pi-ai.
break
}
}
if (content.every(block => block.type === 'text')) return content.map(block => block.text).join('')
return content
}
function toolsOf(options: GenerateOptions): PiTool[] | undefined {
return options.tools?.map(tool => ({
name: tool.name,
description: tool.description,
// ToolSchema.parameters is a JSON Schema object; pi-ai's TSchema
// (TypeBox) is structurally JSON Schema, so it assigns directly.
parameters: tool.parameters,
}))
}
/** Assemble the request-level pi-ai context envelope shared by both conversion paths. */
function piContext(options: GenerateOptions, messages: PiMessage[]): PiContext {
const tools = toolsOf(options)
return {
...options.system !== undefined ? { systemPrompt: options.system } : {},
messages,
...tools !== undefined && tools.length > 0 ? { tools } : {},
}
}
function textOnlyContext(options: GenerateOptions): PiContext {
const toolNames = new Map<CallId, string>()
const messages: PiMessage[] = []
for (const message of options.messages) {
if (message.content.some(block => block.type === 'image'
|| (block.type === 'tool-result' && block.content.some(piece => piece.type === 'image')))) {
throw new LlmError('pi-ai image conversion requires the durable attachment service', 'UNSUPPORTED_CONTENT')
}
if (message.role === 'system') {
messages.push({ role: 'user', content: flattenText(message), timestamp: 0 })
continue
}
if (message.role === 'assistant') {
const assistant = toPiAssistant(message)
for (const block of assistant.content) if (block.type === 'toolCall') toolNames.set(CallId(block.id), block.name)
messages.push(assistant)
continue
}
const text = flattenText(message)
const results = message.content.filter(block => block.type === 'tool-result')
if (text.length > 0 || results.length === 0) messages.push({ role: 'user', content: text, timestamp: 0 })
for (const result of results) {
messages.push({
role: 'toolResult',
toolCallId: result.toolCallId,
toolName: toolNames.get(result.toolCallId) ?? 'unknown',
content: [{
type: 'text',
text: result.content.filter(block => block.type === 'text').map(block => block.text).join('') || '(no output)',
}],
isError: result.isError ?? false,
timestamp: 0,
})
}
}
return piContext(options, messages)
}
/**
* Convert harness history to a pi-ai Context. Tool results need the tool
* NAME (pi-ai's `toolName`), which the harness doesn't carry on the result
* block — it is recovered from the preceding assistant tool-call with the
* same id.
* Convert text-only harness history to a synchronous pi-ai Context. Tool
* result names are recovered from preceding assistant tool calls.
* @param options - the harness request; `options.system` maps to pi-ai's single `systemPrompt` slot.
* @returns the pi-ai context; `tools` is omitted entirely when the request declares none.
* @returns the pi-ai context; `tools` is omitted when the request declares none.
*/
export function toPiContext(options: GenerateOptions): PiContext {
export function toPiContext(options: GenerateOptions): PiContext
/**
* Convert harness history to a pi-ai Context while resolving durable images.
* Tool result names are recovered from preceding assistant tool calls.
* @param options - the harness request; `options.system` maps to pi-ai's single `systemPrompt` slot.
* @param attachments - durable byte resolver for image references.
* @returns the asynchronously resolved pi-ai context.
*/
export function toPiContext(options: GenerateOptions, attachments: AttachmentStore): Promise<PiContext>
export function toPiContext(options: GenerateOptions, attachments?: AttachmentStore): PiContext | Promise<PiContext> {
return attachments === undefined ? textOnlyContext(options) : toPiContextWithImages(options, attachments)
}
async function toPiContextWithImages(options: GenerateOptions, attachments: AttachmentStore): Promise<PiContext> {
const toolNames = new Map<CallId, string>()
const messages: PiMessage[] = []
for (const message of options.messages) {
if (message.role === 'system') {
if (message.content.some(block => block.type === 'image')) {
throw new LlmError('pi-ai cannot represent an image in an in-history system message', 'UNSUPPORTED_CONTENT')
}
// pi-ai has a single systemPrompt slot; in-history system messages are
// folded into user messages to preserve order (rare in practice — the
// harness sends the system prompt via options.system).
@@ -46,40 +149,26 @@ export function toPiContext(options: GenerateOptions): PiContext {
continue
}
// user role: text + tool results (each result becomes its own message).
const text = flattenText(message)
const regular = message.content.filter(block => block.type !== 'tool-result')
const content = await userContent(regular, attachments)
const results = message.content.filter(block => block.type === 'tool-result')
if (text.length > 0 || results.length === 0) {
messages.push({ role: 'user', content: text, timestamp: 0 })
if (content.length > 0 || results.length === 0) {
messages.push({ role: 'user', content, timestamp: 0 })
}
for (const result of results) {
const resultContent = await userContent(result.content, attachments)
messages.push({
role: 'toolResult',
toolCallId: result.toolCallId,
toolName: toolNames.get(result.toolCallId) ?? 'unknown',
content: [{
type: 'text',
text: result.content
.filter(block => block.type === 'text')
.map(block => block.text)
.join('') || '(no output)',
}],
content: typeof resultContent === 'string'
? [{ type: 'text', text: resultContent || '(no output)' }]
: resultContent,
isError: result.isError ?? false,
timestamp: 0,
})
}
}
const tools: PiTool[] | undefined = options.tools?.map(tool => ({
name: tool.name,
description: tool.description,
// ToolSchema.parameters is a JSON Schema object; pi-ai's TSchema
// (TypeBox) is structurally JSON Schema, so it assigns directly.
parameters: tool.parameters,
}))
return {
...options.system !== undefined ? { systemPrompt: options.system } : {},
messages,
...tools !== undefined && tools.length > 0 ? { tools } : {},
}
return piContext(options, messages)
}