fix: address ds-review-bot v6/v7 findings on the image-input assembly

- resolveLlmRoute: reuse the yml pi-ai row for providers it already routes
  (DUPLICATE_ADAPTER boot failure) and detect an unset model by origin, not
  by comparison against one deployment default; covered by a new spec.
- LlmService.resolveModelInfoFor preserves (and validates) modality
  metadata, arming the host image preflight for exact-route resolution.
- session.selectModel refuses a text-only target once the session log
  carries an image on any replayed route; an accepted switch would strand
  every later turn with no in-product recovery.
- The composer no longer gates image intake on the handshake activeModel
  snapshot (wrong authority for a per-session decision); the host preflight
  plus the error strip own capability, deployment limits stay client-side.
- InputHub shell teardown releases the scope's draft images (File objects
  and object URLs leaked for the page lifetime).
- session.prompt image parts carry optional alt into the durable block;
  ImageBlock documents assistant-side rendering as forward compatibility.
- Assembled built-client lane apps/web/tests/image-display.snapshot.ts pins
  the history galleries over the authorized attachment route, the lightbox,
  and the composer paste rail; the attachment rail is an accessible group.
- Docs: validateImage on the seam page, fixture byte metadata matches its
  PNG, and the Agent Note claims now match the shipped coverage.
This commit is contained in:
creatixchu
2026-07-29 18:56:40 +08:00
parent 22e48c1953
commit adce3b833d
21 changed files with 526 additions and 36 deletions

View File

@@ -106,7 +106,7 @@ async function durablePromptContent(ctx: Context, content: readonly PromptConten
mediaType: item.part.mediaType,
...item.part.name === undefined ? {} : { name: item.part.name },
})
return { type: 'image', attachment }
return { type: 'image', attachment, ...item.part.alt === undefined ? {} : { alt: item.part.alt } }
}))
}
@@ -127,6 +127,32 @@ function imageInContent(content: unknown, attachmentId: string): ImageAttachment
return undefined
}
/** True when any block (nested tool-result content included) is an image block. */
function contentHasImage(content: unknown): boolean {
if (!Array.isArray(content)) return false
for (const value of content) {
if (typeof value !== 'object' || value === null || Array.isArray(value)) continue
const block = value as { type?: unknown; content?: unknown }
if (block.type === 'image') return true
if (block.type === 'tool-result' && contentHasImage(block.content)) return true
}
return false
}
/**
* True when the session log already carries image content on any route a
* model request replays (message content, wrapped messages, streamed blocks).
* The log is immutable, so a true here is permanent for the session's life.
*/
function sessionHasImage(events: readonly SessionEvent[]): boolean {
return events.some((event) => {
const data = event.data as { content?: unknown; message?: { content?: unknown }; chunk?: { type?: unknown; block?: unknown } }
if (contentHasImage(data.content)) return true
if (data.message !== undefined && contentHasImage(data.message.content)) return true
return event.type === 'assistant/chunk' && data.chunk?.type === 'block-end' && contentHasImage([data.chunk.block])
})
}
function referencedImage(events: readonly SessionEvent[], attachmentId: string): ImageAttachmentRef | undefined {
for (const event of events) {
const data = event.data as { content?: unknown; chunk?: { type?: unknown; block?: unknown } }
@@ -1111,6 +1137,20 @@ export function createApiProxy(ctx: Context, defaults: ApiProxyDefaults): ApiPro
? {}
: { reasoningEffort: ReasoningEffortId(reasoningEffort) },
})
// An image-bearing log replays into every later request, and both
// wire routes reject image content on text-only models — accepting
// this selection would strand the session (every turn fails, no
// in-product recovery). Refuse at the selection boundary instead.
if (sessionHasImage(found.agent.session.events)) {
const info = await ctx.llm.resolveModelInfo(resolved.provider, resolved.model)
if (info.inputModalities !== undefined && !info.inputModalities.includes('image')) {
return err(request, {
code: 'model-unavailable',
message: `Model "${resolved.model}" does not accept image input, but this session's history already contains images; select an image-capable model.`,
details: { provider, model },
})
}
}
const selected: AgentLlmTarget = {
provider: resolved.provider,
model: resolved.model,

View File

@@ -198,7 +198,7 @@ export const imageMediaTypeSchema = z.union([
/** Prompt wire content is intentionally narrower than merge-extensible durable core content. */
export const promptContentPartSchema = z.discriminatedUnion('type', [
z.object({ type: z.literal('text'), text: z.string() }),
z.object({ type: z.literal('image'), mediaType: imageMediaTypeSchema, data: z.string(), name: z.string().optional() }),
z.object({ type: z.literal('image'), mediaType: imageMediaTypeSchema, data: z.string(), name: z.string().optional(), alt: z.string().optional() }),
])
/** session.prompt request payload. */

View File

@@ -162,7 +162,7 @@ export interface SessionSummary {
/** Browser-submitted prompt content; image bytes are promoted to durable references by the host. */
export type PromptContentPart =
| { type: 'text'; text: string }
| { type: 'image'; mediaType: ImageMediaType; data: string; name?: string }
| { type: 'image'; mediaType: ImageMediaType; data: string; name?: string; alt?: string }
/** Session-domain unary methods (the map keys session.* of RpcMethodMap). */
export interface SessionsApi {

View File

@@ -230,4 +230,91 @@ describe('Web session model selection', () => {
.toEqual({ provider: 'deepseek', model: 'private-preview', reasoningEffort: 'max' })
await ctx.fiber.dispose()
})
it('refuses a text-only selection once the session log carries an image', async () => {
const { ctx, sessionId, agent } = await harness()
ctx.llm.registerAdapter(['text-only'], new class extends CatalogAdapter {
override resolveModel(provider: string, model: string): Promise<LlmResolvedModelInfo> {
return Promise.resolve({ provider, id: model, name: model, inputModalities: ['text'] })
}
}('Text Only', []))
ctx.llm.registerAdapter(['vision'], new class extends CatalogAdapter {
override resolveModel(provider: string, model: string): Promise<LlmResolvedModelInfo> {
return Promise.resolve({ provider, id: model, name: model, inputModalities: ['text', 'image'] })
}
}('Vision', []))
const api = createApiProxy(ctx, { provider: 'deepseek', model: 'deepseek-chat', cwd: '/tmp', workspaceRoot: '/tmp' })
// Before any image lands, a text-only selection is legitimate.
expect(expectValue(await api.sessions.selectModel(request({
sessionId, provider: 'text-only', model: 'plain',
}))).selected).toEqual({ provider: 'text-only', model: 'plain' })
agent.session.append('user/message', {
id: 'msg-image', role: 'user', source: { kind: 'user' },
content: [{ type: 'image', attachment: { attachmentId: 'att-1', mediaType: 'image/png', bytes: 8, width: 1, height: 1 } }],
} as never, { surfaceOp: 'append' })
// The log is immutable: a text-only route would fail every later turn.
const stranded = await api.sessions.selectModel(request({
sessionId, provider: 'text-only', model: 'plain',
}))
expect(stranded.result).toMatchObject({
ok: false,
error: { code: 'model-unavailable', message: expect.stringMatching(/history already contains images/) as unknown },
})
// Image-capable and modality-unknown routes stay selectable.
expect(expectValue(await api.sessions.selectModel(request({
sessionId, provider: 'vision', model: 'sees',
}))).selected).toEqual({ provider: 'vision', model: 'sees' })
expect(expectValue(await api.sessions.selectModel(request({
sessionId, provider: 'deepseek', model: 'deepseek-chat',
}))).selected).toEqual({ provider: 'deepseek', model: 'deepseek-chat', reasoningEffort: 'high' })
await ctx.fiber.dispose()
})
it('detects images on every replayed route: wrapped messages, streamed blocks, nested tool results', async () => {
const image = { type: 'image', attachment: { attachmentId: 'att-x', mediaType: 'image/png', bytes: 8, width: 1, height: 1 } }
const cases: { label: string; append: (agent: Agent) => void }[] = [
{
label: 'steering message wrapper',
append: (agent) => {
agent.session.append('steering/message', {
turn: 1, message: { id: 'st-1', role: 'user', source: { kind: 'user' }, content: [image] },
} as never, { surfaceOp: 'append' })
},
},
{
label: 'streamed assistant block',
append: (agent) => {
agent.session.append('assistant/chunk', {
turn: 1, step: 0, chunk: { type: 'block-end', index: 0, block: image },
} as never)
},
},
{
label: 'nested tool-result content',
append: (agent) => {
agent.session.append('user/message', {
id: 'tr-1', role: 'user', source: { kind: 'tool', callId: 'c1' },
content: [{ type: 'tool-result', toolCallId: 'c1', content: [image], isError: false }],
} as never, { surfaceOp: 'append' })
},
},
]
for (const { label, append } of cases) {
const { ctx, sessionId, agent } = await harness()
ctx.llm.registerAdapter(['text-only'], new class extends CatalogAdapter {
override resolveModel(provider: string, model: string): Promise<LlmResolvedModelInfo> {
return Promise.resolve({ provider, id: model, name: model, inputModalities: ['text'] })
}
}('Text Only', []))
const api = createApiProxy(ctx, { provider: 'deepseek', model: 'deepseek-chat', cwd: '/tmp', workspaceRoot: '/tmp' })
append(agent)
const stranded = await api.sessions.selectModel(request({ sessionId, provider: 'text-only', model: 'plain' }))
expect(stranded.result.ok, label).toBe(false)
await ctx.fiber.dispose()
}
})
})