fix(gui): harden multimodal image attachments

This commit is contained in:
Yichen Jiang
2026-07-23 19:38:37 +08:00
parent eea595fcb4
commit 580e05b794
61 changed files with 1700 additions and 214 deletions

View File

@@ -1,4 +1,5 @@
import { describe, expect, it } from 'vitest'
import { AttachmentId } from '@deepseek-ai/dsh-attachment'
import type { ContentBlock } from '@deepseek-ai/dsh-llm'
import type { TurnEndReason } from '@deepseek-ai/dsh-session'
import { SessionId } from '@deepseek-ai/dsh-session'
@@ -35,6 +36,17 @@ describe('turnEndToStopReason', () => {
describe('harnessBlockToAcpContent', () => {
it('maps a text block to ACP text content', () => {
expect(harnessBlockToAcpContent({ type: 'text', text: 'hi' })).toEqual({ type: 'text', text: 'hi' })
const attachmentId = AttachmentId(`sha256:${'a'.repeat(64)}`)
expect(harnessBlockToAcpContent({
type: 'image',
attachment: {
attachmentId,
mediaType: 'image/png',
bytes: 1,
width: 1,
height: 1,
},
})).toEqual({ type: 'text', text: `[image attachment ${attachmentId}]` })
})
it('returns undefined for non-text blocks (reasoning / plugin-added)', () => {

View File

@@ -124,6 +124,6 @@ Append-only; newly visible content follows the reusable request prefix and does
## Known Limitations and Deferred Work
- **One configured session owns the transcript and editor** — questions from other agents can still use the shared overlay provider, but session rendering and prompt input remain bound to `sessionId`.
- **Tool cards are text terminal presentations** — terminal, diff, and generic cards use tool-owned titles/content, but session content currently has no image block for inline image rendering.
- **Images use explicit text markers** — the terminal cannot render inline raster images, so user, assistant, tool-result, and streaming image blocks render as `[image attachment <id>]` instead of disappearing.
- **Non-TTY operation is intentionally unsupported** — app bundles that need automation must compose a one-shot or server front door (`dsh-cli-demo`, `dsh-acp`) rather than expecting an internal fallback.
- **Manual `/skill:` invocation always reloads the full skill body** — the TUI does not detect a skill already present in the conversation, so repeated invocations append its instructions again.

View File

@@ -424,6 +424,9 @@ function contentText(content: readonly ContentBlock[]): string {
case 'tool-result':
parts.push(contentText(block.content))
break
case 'image':
parts.push(`[image attachment ${block.attachment.attachmentId}]`)
break
default: {
const rawType = (block as { type?: unknown }).type
parts.push(`[${typeof rawType === 'string' ? rawType : 'content'}]`)
@@ -648,7 +651,14 @@ class AssistantMessageComponent extends Container {
constructor(content: readonly ContentBlock[], showReasoning: boolean, palette: Palette, mdTheme: MarkdownTheme) {
super()
const reasoning = displayText(textBlocks(content, 'reasoning').trim())
const text = displayText(textBlocks(content, 'text').trim())
const text = displayText(content
.flatMap(block => block.type === 'text'
? [block.text]
: block.type === 'image'
? [`[image attachment ${block.attachment.attachmentId}]`]
: [])
.join('\n\n')
.trim())
if (reasoning && showReasoning) {
this.addChild(new Spacer(1))
this.addChild(new Text(palette.italic(palette.muted('Reasoning')), 1, 0))
@@ -668,6 +678,7 @@ class AssistantMessageComponent extends Container {
interface StreamingBlock {
type: string
text: string
block?: ContentBlock
}
class StreamingAssistantComponent extends Container {
@@ -691,6 +702,8 @@ class StreamingAssistantComponent extends Container {
this.blocks.set(chunk.index, block)
} else if (chunk.type === 'block-end' && (chunk.block.type === 'text' || chunk.block.type === 'reasoning')) {
this.blocks.set(chunk.index, { type: chunk.block.type, text: chunk.block.text })
} else if (chunk.type === 'block-end' && chunk.block.type === 'image') {
this.blocks.set(chunk.index, { type: 'image', text: '', block: chunk.block })
}
this.rebuild()
}
@@ -707,6 +720,7 @@ class StreamingAssistantComponent extends Container {
.flatMap<ContentBlock>(([, block]) => {
if (block.type === 'text') return [{ type: 'text', text: block.text }]
if (block.type === 'reasoning') return [{ type: 'reasoning', text: block.text }]
if (block.type === 'image' && block.block?.type === 'image') return [block.block]
return []
})
const component = new AssistantMessageComponent(content, this.showReasoning, this.palette, this.mdTheme)

View File

@@ -434,6 +434,29 @@ describe('pi-tui chat lifecycle and transcript', () => {
step: 1,
chunk: { type: 'tool-call-delta', index: 2, id: 'stream-tool' as never, argumentsDelta: '{}' },
})
result.session.append('assistant/chunk', {
turn: 3,
step: 1,
chunk: { type: 'block-start', index: 3, blockType: 'image' },
})
result.session.append('assistant/chunk', {
turn: 3,
step: 1,
chunk: {
type: 'block-end',
index: 3,
block: {
type: 'image',
attachment: {
attachmentId: 'sha256:stream-image' as never,
mediaType: 'image/png',
bytes: 1,
width: 1,
height: 1,
},
},
},
})
result.session.append('assistant/chunk', {
turn: 3,
step: 1,
@@ -441,6 +464,7 @@ describe('pi-tui chat lifecycle and transcript', () => {
})
await tick()
expect(result.terminal.output).toContain('live thought')
expect(result.terminal.output).toContain('sha256:stream-image]')
result.terminal.send('\x12')
await tick()
appendAssistant(
@@ -729,6 +753,16 @@ describe('pi-tui chat lifecycle and transcript', () => {
{ type: 'text', text: '# Heading\n\n[link](https://example.com) `code`\n\n```ts\nconst x = 1\n```\n\n> quote\n\n---\n\n- item\n\n**bold** *italic* ~~strike~~' },
{ type: 'tool-call', id: 'nested' as never, name: 'nested_tool', arguments: '{}' },
{ type: 'tool-result', toolCallId: 'nested' as never, content: [{ type: 'reasoning', text: 'nested result' }] },
{
type: 'image',
attachment: {
attachmentId: 'sha256:user-image' as never,
mediaType: 'image/png',
bytes: 1,
width: 1,
height: 1,
},
},
{ type: 'future-block' } as never,
{} as never,
],
@@ -737,6 +771,16 @@ describe('pi-tui chat lifecycle and transcript', () => {
appendAssistant(session, [
{ type: 'reasoning', text: 'styled reasoning' },
{ type: 'text', text: 'styled answer' },
{
type: 'image',
attachment: {
attachmentId: 'sha256:assistant-image' as never,
mediaType: 'image/png',
bytes: 1,
width: 1,
height: 1,
},
},
], { inputTokens: 2_000_000, outputTokens: 1_500_000 })
session.append('todo/write', { todos: [
{ content: 'done', status: 'completed' },
@@ -756,6 +800,8 @@ describe('pi-tui chat lifecycle and transcript', () => {
expect(result.terminal.output).toContain('Heading')
expect(result.terminal.output).toContain('nested_tool({})')
expect(result.terminal.output).toContain('nested result')
expect(result.terminal.output).toContain('sha256:user-image]')
expect(result.terminal.output).toContain('[image attachment sha256:assistant-image]')
expect(result.terminal.output).toContain('[future-block]')
expect(result.terminal.output).toContain('[content]')
expect(result.terminal.output).toContain('↑2.0m ↓1.5m')