fix(gui): harden multimodal image attachments
This commit is contained in:
@@ -1,4 +1,5 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { AttachmentId } from '@deepseek-ai/dsh-attachment'
|
||||
import type { ContentBlock } from '@deepseek-ai/dsh-llm'
|
||||
import type { TurnEndReason } from '@deepseek-ai/dsh-session'
|
||||
import { SessionId } from '@deepseek-ai/dsh-session'
|
||||
@@ -35,6 +36,17 @@ describe('turnEndToStopReason', () => {
|
||||
describe('harnessBlockToAcpContent', () => {
|
||||
it('maps a text block to ACP text content', () => {
|
||||
expect(harnessBlockToAcpContent({ type: 'text', text: 'hi' })).toEqual({ type: 'text', text: 'hi' })
|
||||
const attachmentId = AttachmentId(`sha256:${'a'.repeat(64)}`)
|
||||
expect(harnessBlockToAcpContent({
|
||||
type: 'image',
|
||||
attachment: {
|
||||
attachmentId,
|
||||
mediaType: 'image/png',
|
||||
bytes: 1,
|
||||
width: 1,
|
||||
height: 1,
|
||||
},
|
||||
})).toEqual({ type: 'text', text: `[image attachment ${attachmentId}]` })
|
||||
})
|
||||
|
||||
it('returns undefined for non-text blocks (reasoning / plugin-added)', () => {
|
||||
|
||||
@@ -124,6 +124,6 @@ Append-only; newly visible content follows the reusable request prefix and does
|
||||
## Known Limitations and Deferred Work
|
||||
|
||||
- **One configured session owns the transcript and editor** — questions from other agents can still use the shared overlay provider, but session rendering and prompt input remain bound to `sessionId`.
|
||||
- **Tool cards are text terminal presentations** — terminal, diff, and generic cards use tool-owned titles/content, but session content currently has no image block for inline image rendering.
|
||||
- **Images use explicit text markers** — the terminal cannot render inline raster images, so user, assistant, tool-result, and streaming image blocks render as `[image attachment <id>]` instead of disappearing.
|
||||
- **Non-TTY operation is intentionally unsupported** — app bundles that need automation must compose a one-shot or server front door (`dsh-cli-demo`, `dsh-acp`) rather than expecting an internal fallback.
|
||||
- **Manual `/skill:` invocation always reloads the full skill body** — the TUI does not detect a skill already present in the conversation, so repeated invocations append its instructions again.
|
||||
|
||||
@@ -424,6 +424,9 @@ function contentText(content: readonly ContentBlock[]): string {
|
||||
case 'tool-result':
|
||||
parts.push(contentText(block.content))
|
||||
break
|
||||
case 'image':
|
||||
parts.push(`[image attachment ${block.attachment.attachmentId}]`)
|
||||
break
|
||||
default: {
|
||||
const rawType = (block as { type?: unknown }).type
|
||||
parts.push(`[${typeof rawType === 'string' ? rawType : 'content'}]`)
|
||||
@@ -648,7 +651,14 @@ class AssistantMessageComponent extends Container {
|
||||
constructor(content: readonly ContentBlock[], showReasoning: boolean, palette: Palette, mdTheme: MarkdownTheme) {
|
||||
super()
|
||||
const reasoning = displayText(textBlocks(content, 'reasoning').trim())
|
||||
const text = displayText(textBlocks(content, 'text').trim())
|
||||
const text = displayText(content
|
||||
.flatMap(block => block.type === 'text'
|
||||
? [block.text]
|
||||
: block.type === 'image'
|
||||
? [`[image attachment ${block.attachment.attachmentId}]`]
|
||||
: [])
|
||||
.join('\n\n')
|
||||
.trim())
|
||||
if (reasoning && showReasoning) {
|
||||
this.addChild(new Spacer(1))
|
||||
this.addChild(new Text(palette.italic(palette.muted('Reasoning')), 1, 0))
|
||||
@@ -668,6 +678,7 @@ class AssistantMessageComponent extends Container {
|
||||
interface StreamingBlock {
|
||||
type: string
|
||||
text: string
|
||||
block?: ContentBlock
|
||||
}
|
||||
|
||||
class StreamingAssistantComponent extends Container {
|
||||
@@ -691,6 +702,8 @@ class StreamingAssistantComponent extends Container {
|
||||
this.blocks.set(chunk.index, block)
|
||||
} else if (chunk.type === 'block-end' && (chunk.block.type === 'text' || chunk.block.type === 'reasoning')) {
|
||||
this.blocks.set(chunk.index, { type: chunk.block.type, text: chunk.block.text })
|
||||
} else if (chunk.type === 'block-end' && chunk.block.type === 'image') {
|
||||
this.blocks.set(chunk.index, { type: 'image', text: '', block: chunk.block })
|
||||
}
|
||||
this.rebuild()
|
||||
}
|
||||
@@ -707,6 +720,7 @@ class StreamingAssistantComponent extends Container {
|
||||
.flatMap<ContentBlock>(([, block]) => {
|
||||
if (block.type === 'text') return [{ type: 'text', text: block.text }]
|
||||
if (block.type === 'reasoning') return [{ type: 'reasoning', text: block.text }]
|
||||
if (block.type === 'image' && block.block?.type === 'image') return [block.block]
|
||||
return []
|
||||
})
|
||||
const component = new AssistantMessageComponent(content, this.showReasoning, this.palette, this.mdTheme)
|
||||
|
||||
@@ -434,6 +434,29 @@ describe('pi-tui chat lifecycle and transcript', () => {
|
||||
step: 1,
|
||||
chunk: { type: 'tool-call-delta', index: 2, id: 'stream-tool' as never, argumentsDelta: '{}' },
|
||||
})
|
||||
result.session.append('assistant/chunk', {
|
||||
turn: 3,
|
||||
step: 1,
|
||||
chunk: { type: 'block-start', index: 3, blockType: 'image' },
|
||||
})
|
||||
result.session.append('assistant/chunk', {
|
||||
turn: 3,
|
||||
step: 1,
|
||||
chunk: {
|
||||
type: 'block-end',
|
||||
index: 3,
|
||||
block: {
|
||||
type: 'image',
|
||||
attachment: {
|
||||
attachmentId: 'sha256:stream-image' as never,
|
||||
mediaType: 'image/png',
|
||||
bytes: 1,
|
||||
width: 1,
|
||||
height: 1,
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
result.session.append('assistant/chunk', {
|
||||
turn: 3,
|
||||
step: 1,
|
||||
@@ -441,6 +464,7 @@ describe('pi-tui chat lifecycle and transcript', () => {
|
||||
})
|
||||
await tick()
|
||||
expect(result.terminal.output).toContain('live thought')
|
||||
expect(result.terminal.output).toContain('sha256:stream-image]')
|
||||
result.terminal.send('\x12')
|
||||
await tick()
|
||||
appendAssistant(
|
||||
@@ -729,6 +753,16 @@ describe('pi-tui chat lifecycle and transcript', () => {
|
||||
{ type: 'text', text: '# Heading\n\n[link](https://example.com) `code`\n\n```ts\nconst x = 1\n```\n\n> quote\n\n---\n\n- item\n\n**bold** *italic* ~~strike~~' },
|
||||
{ type: 'tool-call', id: 'nested' as never, name: 'nested_tool', arguments: '{}' },
|
||||
{ type: 'tool-result', toolCallId: 'nested' as never, content: [{ type: 'reasoning', text: 'nested result' }] },
|
||||
{
|
||||
type: 'image',
|
||||
attachment: {
|
||||
attachmentId: 'sha256:user-image' as never,
|
||||
mediaType: 'image/png',
|
||||
bytes: 1,
|
||||
width: 1,
|
||||
height: 1,
|
||||
},
|
||||
},
|
||||
{ type: 'future-block' } as never,
|
||||
{} as never,
|
||||
],
|
||||
@@ -737,6 +771,16 @@ describe('pi-tui chat lifecycle and transcript', () => {
|
||||
appendAssistant(session, [
|
||||
{ type: 'reasoning', text: 'styled reasoning' },
|
||||
{ type: 'text', text: 'styled answer' },
|
||||
{
|
||||
type: 'image',
|
||||
attachment: {
|
||||
attachmentId: 'sha256:assistant-image' as never,
|
||||
mediaType: 'image/png',
|
||||
bytes: 1,
|
||||
width: 1,
|
||||
height: 1,
|
||||
},
|
||||
},
|
||||
], { inputTokens: 2_000_000, outputTokens: 1_500_000 })
|
||||
session.append('todo/write', { todos: [
|
||||
{ content: 'done', status: 'completed' },
|
||||
@@ -756,6 +800,8 @@ describe('pi-tui chat lifecycle and transcript', () => {
|
||||
expect(result.terminal.output).toContain('Heading')
|
||||
expect(result.terminal.output).toContain('nested_tool({})')
|
||||
expect(result.terminal.output).toContain('nested result')
|
||||
expect(result.terminal.output).toContain('sha256:user-image]')
|
||||
expect(result.terminal.output).toContain('[image attachment sha256:assistant-image]')
|
||||
expect(result.terminal.output).toContain('[future-block]')
|
||||
expect(result.terminal.output).toContain('[content]')
|
||||
expect(result.terminal.output).toContain('↑2.0m ↓1.5m')
|
||||
|
||||
Reference in New Issue
Block a user