fix: address ds-review-bot v6/v7 findings on the image-input assembly
- resolveLlmRoute: reuse the yml pi-ai row for providers it already routes (DUPLICATE_ADAPTER boot failure) and detect an unset model by origin, not by comparison against one deployment default; covered by a new spec. - LlmService.resolveModelInfoFor preserves (and validates) modality metadata, arming the host image preflight for exact-route resolution. - session.selectModel refuses a text-only target once the session log carries an image on any replayed route; an accepted switch would strand every later turn with no in-product recovery. - The composer no longer gates image intake on the handshake activeModel snapshot (wrong authority for a per-session decision); the host preflight plus the error strip own capability, deployment limits stay client-side. - InputHub shell teardown releases the scope's draft images (File objects and object URLs leaked for the page lifetime). - session.prompt image parts carry optional alt into the durable block; ImageBlock documents assistant-side rendering as forward compatibility. - Assembled built-client lane apps/web/tests/image-display.snapshot.ts pins the history galleries over the authorized attachment route, the lightbox, and the composer paste rail; the attachment rail is an accessible group. - Docs: validateImage on the seam page, fixture byte metadata matches its PNG, and the Agent Note claims now match the shipped coverage.
This commit is contained in:
@@ -333,11 +333,23 @@ export class LlmService extends Service {
|
||||
'INVALID_MODEL_CONTEXT',
|
||||
)
|
||||
}
|
||||
for (const modalities of [resolved.inputModalities, resolved.outputModalities]) {
|
||||
if (modalities !== undefined && (!Array.isArray(modalities) || modalities.some(m => typeof m !== 'string'))) {
|
||||
throw new LlmError(
|
||||
`adapter returned invalid modality metadata for provider "${provider}" model "${model}"`,
|
||||
'INVALID_MODEL_INFO',
|
||||
)
|
||||
}
|
||||
}
|
||||
const info: LlmResolvedModelInfo = {
|
||||
provider,
|
||||
id: model,
|
||||
name: resolved.name,
|
||||
...resolved.description === undefined ? {} : { description: resolved.description },
|
||||
// Capability metadata rides through: an explicit modality omission is
|
||||
// negative capability downstream preflights act on (image admission).
|
||||
...resolved.inputModalities === undefined ? {} : { inputModalities: resolved.inputModalities },
|
||||
...resolved.outputModalities === undefined ? {} : { outputModalities: resolved.outputModalities },
|
||||
...context === undefined ? {} : { context: { contextWindow: context.contextWindow } },
|
||||
}
|
||||
const reasoning = resolved.reasoning
|
||||
|
||||
@@ -47,12 +47,17 @@ export interface ReasoningBlock {
|
||||
text: string
|
||||
}
|
||||
|
||||
/** A durable raster image reference, valid in user or assistant content. */
|
||||
/**
|
||||
* A durable raster image reference, valid in user or assistant content. The
|
||||
* block is deliberately role-neutral; assistant-side rendering is forward
|
||||
* compatibility — the current production adapters declare text-only output,
|
||||
* so only user content carries images today.
|
||||
*/
|
||||
export interface ImageBlock {
|
||||
type: 'image'
|
||||
/** Immutable bytes and intrinsic display metadata owned by the attachment service. */
|
||||
attachment: ImageAttachmentRef
|
||||
/** Optional provider- and UI-facing alternative text. */
|
||||
/** Optional provider- and UI-facing alternative text, carried from the prompt wire's image part. */
|
||||
alt?: string
|
||||
}
|
||||
|
||||
|
||||
@@ -857,6 +857,8 @@ describe('LlmService', () => {
|
||||
[{ provider: 'route', id: 'model', name: 1 }, 'non-string name'],
|
||||
[{ provider: 'route', id: 'model', name: '' }, 'empty name'],
|
||||
[{ provider: 'route', id: 'model', name: 'Model', description: 1 }, 'non-string description'],
|
||||
[{ provider: 'route', id: 'model', name: 'Model', inputModalities: 'text' }, 'non-array input modalities'],
|
||||
[{ provider: 'route', id: 'model', name: 'Model', outputModalities: [1] }, 'non-string output modality'],
|
||||
] as const)('rejects invalid exact model metadata (%s: %s)', async (metadata, _label) => {
|
||||
const ctx = new Context()
|
||||
await ctx.plugin(LlmService)
|
||||
@@ -871,6 +873,27 @@ describe('LlmService', () => {
|
||||
.rejects.toMatchObject({ code: 'INVALID_MODEL_INFO' })
|
||||
})
|
||||
|
||||
it('preserves modality metadata through exact model resolution', async () => {
|
||||
const ctx = new Context()
|
||||
await ctx.plugin(LlmService)
|
||||
const adapter = new class extends ScriptedAdapter {
|
||||
override resolveModel(): Promise<LlmResolvedModelInfo> {
|
||||
return Promise.resolve({
|
||||
provider: 'route', id: 'model', name: 'Model',
|
||||
inputModalities: ['text', 'image'], outputModalities: ['text'],
|
||||
})
|
||||
}
|
||||
}(SCRIPT)
|
||||
ctx.llm.registerAdapter(['route'], adapter)
|
||||
|
||||
// Downstream preflights (image admission) act on this exact field; a
|
||||
// rebuild that drops it silently reads as "modalities unknown".
|
||||
await expect(ctx.llm.resolveModelInfo('route', 'model')).resolves.toEqual({
|
||||
provider: 'route', id: 'model', name: 'Model',
|
||||
inputModalities: ['text', 'image'], outputModalities: ['text'],
|
||||
})
|
||||
})
|
||||
|
||||
it('resolves detached model context independently of advisory catalog membership', async () => {
|
||||
const ctx = new Context()
|
||||
await ctx.plugin(LlmService)
|
||||
|
||||
Reference in New Issue
Block a user