fix: harden Web image admission

This commit is contained in:
Tianyi Cui
2026-07-30 01:58:36 +08:00
parent d6c82001b3
commit 515d48875e
52 changed files with 999 additions and 444 deletions

View File

@@ -2,5 +2,5 @@
# side as of the last confirmed-consistent state. Both languages carry equal authority;
# after editing either side, bring the other along and re-record with:
# pnpm run verify-translation-pairing --write packages/host/apiproxy/README.md
README.md: 1f0daedc54888a1951bc83c474f83287aaf42307
README.zh.md: abf5417cdbe93f1199c621ac101249986969da93
README.md: 693b2c26a9ec1e7ea31030a3028f05706adbfc3b
README.zh.md: b1766d901d9ed5743cbc766166bfb310c565ef5f

View File

@@ -14,7 +14,7 @@ The layering/protocol decisions are recorded in the [GUI layering and RPC protoc
Session titles ride the generic projection pair like every other domain — the history-tail `projections` block plus `session/projection` frames under the `title` key (the bespoke `session/title` frame is retired). Titles do not join `session.list`; cold sessions remain metadata-only there until opening or resuming attaches their logs. `session.rename` accepts an explicit user title (resuming a cold session first), delegating to `ctx.sessionTitle.rename` — the accepted `session/title` event pins the title against automatic regeneration — and returns the normalized title plus its event seq so a client settles its `title` projection cell ahead of the push frame; a title that normalizes to empty returns `title-invalid`.
Session model routing is a session-domain contract. `session.models` returns the selected provider/model/reasoning target with provider-grouped advisory models, exact-route reasoning metadata, and provider-local lookup failures. `session.selectModel` validates the optional adapter-owned reasoning effort and replaces the complete target selected for the next prompt-assembly boundary. Catalog membership is not validation: an adapter may resolve an unlisted model, while an unavailable route or unsupported effort returns `model-unavailable`.
Session model routing is a session-domain contract. `session.models` returns the selected provider/model/reasoning target with provider-grouped advisory models, exact-route reasoning metadata, and provider-local lookup failures. `session.selectModel` validates the optional adapter-owned reasoning effort and replaces the complete target selected for the next prompt-assembly boundary. Selection is serialized with image-bearing prompt admission and rejects a text-only target while an image is pending publication or remains in the current derived history; an image removed by compaction no longer blocks selection. Catalog membership is not validation: an adapter may resolve an unlisted model, while an unavailable route or unsupported effort returns `model-unavailable`.
Workspace and Session lists are separate reconnect baselines. `workspace.create` creates a unique name or adopts an existing directory, `workspace.delete` removes only the Workspace registration, `session.create` accepts an optional preallocated Session id, and `host/workspace-changed`, `host/workspace-removed`, plus `host/session-added` carry committed increments in either arrival order. Registration deletion preserves the directory and session logs; its Sessions remain in `session.list` and become Ungrouped. `SessionSummary.blank` and the `host/session-added` frame carry the derived zero-events bit: clients hide blank sessions and reuse them per workspace, flip blank on the first `host/session-status(running:true)`, and treat `session.list` as the reconnect authority; cold summaries are never blank because lazy persistence keeps never-appended sessions out of `list()`.

View File

@@ -14,7 +14,7 @@
会话标题与其他所有领域一样搭乘这对通用投影机制——历史尾页的 `projections` 块外加 `title` 键下的 `session/projection` 帧(专设的 `session/title` 帧已下线)。标题不会加入 `session.list`;冷会话在其中仍只有元数据,直到打开或恢复操作附加其日志。`session.rename` 接受用户显式标题(冷会话先恢复),委托给 `ctx.sessionTitle.rename`——被接受的 `session/title` 事件将标题钉住、不再被自动生成覆盖——并返回规范化后的标题及其事件 seq让 client 在推送帧到达前就结算自己的 `title` 投影格;规范化后为空的标题返回 `title-invalid`
会话模型路由属于会话领域契约。`session.models` 返回选中的提供方模型推理reasoning目标以及按提供方分组的建议性模型、精确路由推理元数据和逐提供方查询失败记录。`session.selectModel` 校验由适配器持有的可选推理强度,并替换将在下一提示词组装边界使用的完整目标。目录成员关系不构成校验:适配器可以解析未列出的模型,而不可用路由或不受支持的推理强度会返回 `model-unavailable`
会话模型路由属于会话领域契约。`session.models` 返回选中的提供方模型推理reasoning目标以及按提供方分组的建议性模型、精确路由推理元数据和逐提供方查询失败记录。`session.selectModel` 校验由适配器持有的可选推理强度,并替换将在下一提示词组装边界使用的完整目标。模型选择与包含图片的提示词准入串行执行当图片正等待发布或仍存在于当前派生历史中时会拒绝选择纯文本目标被压缩compaction移除的图片不再阻止选择。目录成员关系不构成校验:适配器可以解析未列出的模型,而不可用路由或不受支持的推理强度会返回 `model-unavailable`
Workspace 列表与 Session 列表是相互独立的重连基线。`workspace.create` 会创建唯一名称或接纳现有目录,`workspace.delete` 只移除 Workspace 注册记录,`session.create` 接受可选的预分配 Session id`host/workspace-changed``host/workspace-removed``host/session-added` 则以任意到达顺序携带已提交的增量。删除注册记录会保留目录和会话日志;相关 Session 仍留在 `session.list` 中,并进入 Ungrouped。`SessionSummary.blank``host/session-added` 帧携带派生的零事件位:客户端隐藏空白会话并按 workspace 复用它们,在首个 `host/session-status(running:true)` 时翻转 blank并以 `session.list` 作为重连权威;冷会话摘要永远不是空白:惰性持久化让从未追加过事件的会话根本不出现在 `list()` 中。

View File

@@ -92,30 +92,29 @@ async function durablePromptContent(ctx: Context, content: readonly PromptConten
throw new AttachmentError('Prompt exceeds the configured aggregate image-byte limit.', 'IMAGES_TOO_LARGE')
}
for (const image of images) {
ctx.attachments.validateImage({
await ctx.attachments.validateImage({
data: image.data,
mediaType: image.part.mediaType,
...image.part.name === undefined ? {} : { name: image.part.name },
})
}
return Promise.all(prepared.map(async (item): Promise<ContentBlock> => {
if (!('data' in item)) return { type: 'text', text: item.text }
const blocks: ContentBlock[] = []
for (const item of prepared) {
if (!('data' in item)) {
blocks.push({ type: 'text', text: item.text })
continue
}
const attachment = await ctx.attachments.saveImage({
data: item.data,
mediaType: item.part.mediaType,
...item.part.name === undefined ? {} : { name: item.part.name },
})
return { type: 'image', attachment }
}))
blocks.push({ type: 'image', attachment })
}
return blocks
}
/**
* The ONE recursive block walk shared by attachment authorization and the
* model-selection gate (nested tool-result content included). Both consumers
* must agree on what counts as replayed image content — a route added to one
* walker but not the other would silently skip authorization or stranding
* protection — so there is exactly one walker, parameterized by match.
*/
/** Search durable event content for an image reference, including nested tool results. */
function imageBlockIn(content: unknown, match: (ref: ImageAttachmentRef) => boolean): ImageAttachmentRef | undefined {
if (!Array.isArray(content)) return undefined
for (const value of content) {
@@ -148,18 +147,15 @@ function imageInEvent(event: SessionEvent, match: (ref: ImageAttachmentRef) => b
return undefined
}
/** True when any block (nested tool-result content included) is an image block. */
function contentHasImage(content: unknown): boolean {
return imageBlockIn(content, () => true) !== undefined
/** True when typed model content contains an image, including nested tool results. */
function contentHasImage(content: readonly ContentBlock[]): boolean {
return content.some(block => block.type === 'image'
|| (block.type === 'tool-result' && contentHasImage(block.content)))
}
/**
* True when the session log already carries image content on any route a
* model request replays (message content, wrapped messages, streamed blocks).
* The log is immutable, so a true here is permanent for the session's life.
*/
function sessionHasImage(events: readonly SessionEvent[]): boolean {
return events.some(event => imageInEvent(event, () => true) !== undefined)
/** True when the current model-visible surface contains an image. */
function messagesHaveImage(messages: readonly { content: readonly ContentBlock[] }[]): boolean {
return messages.some(message => contentHasImage(message.content))
}
function referencedImage(events: readonly SessionEvent[], attachmentId: string): ImageAttachmentRef | undefined {
@@ -564,6 +560,14 @@ export function createApiProxy(ctx: Context, defaults: ApiProxyDefaults): ApiPro
const pendingQuestions = new Map<RpcId, PendingQuestion>()
const pendingApprovals = new Map<RpcId, PendingApproval>()
const muxQueues = new Set<FrameQueue<RpcRequest<MuxFrame>>>()
const imageAdmissionChains = new WeakMap<Agent, Promise<void>>()
/** Serialize model selection with image prompt admission for one agent. */
function serializeImageAdmission<T>(agent: Agent, operation: () => Promise<T>): Promise<T> {
const result = (imageAdmissionChains.get(agent) ?? Promise.resolve()).then(operation)
imageAdmissionChains.set(agent, result.then(() => undefined, () => undefined))
return result
}
/**
* Install or return the session-local target that prompt assembly snapshots.
@@ -619,18 +623,26 @@ export function createApiProxy(ctx: Context, defaults: ApiProxyDefaults): ApiPro
* Per-session inbox occurrence mirror serving the mux-open queue snapshot
* (the same refresh-recovery baseline as pending questions). Each terminal
* inbox event retires one matching occurrence, so repeated sends of the same
* identified message remain visible until every occurrence is claimed.
* identified message remain visible until every occurrence is published or
* discarded. Dequeue is not publication: the log append follows it.
*/
const queuedMirror = new Map<SessionId, { message: UserMessage; steering: boolean }[]>()
const queuedMirror = new Map<SessionId, { message: UserMessage; steering: boolean; claimed: boolean }[]>()
ctx.effect(() => {
const retire = (agent: Agent, id: MessageId, placement?: InboxPlacement): void => {
const entries = queuedMirror.get(agent.id)
const retire = (sessionId: SessionId, id: MessageId, placement?: InboxPlacement): void => {
const entries = queuedMirror.get(sessionId)
if (entries === undefined) return
const index = entries.findIndex(entry =>
entry.message.id === id
&& (placement === undefined || entry.steering === (placement === 'steering')))
if (index !== -1) entries.splice(index, 1)
if (entries.length === 0) queuedMirror.delete(agent.id)
if (entries.length === 0) queuedMirror.delete(sessionId)
}
const retireClaimed = (sessionId: SessionId): void => {
const entries = queuedMirror.get(sessionId)
if (entries === undefined) return
const pending = entries.filter(entry => !entry.claimed)
if (pending.length === 0) queuedMirror.delete(sessionId)
else queuedMirror.set(sessionId, pending)
}
const disposers = [
ctx.on('agent/inbox/enqueue', (agent: Agent, message: UserMessage, placement) => {
@@ -640,7 +652,7 @@ export function createApiProxy(ctx: Context, defaults: ApiProxyDefaults): ApiPro
queuedMirror.set(agent.id, entries)
}
const steering = placement === 'steering'
entries.push({ message, steering })
entries.push({ message, steering, claimed: false })
broadcast({
type: 'session/queued',
sessionId: agent.id,
@@ -649,10 +661,26 @@ export function createApiProxy(ctx: Context, defaults: ApiProxyDefaults): ApiPro
})
}),
ctx.on('agent/inbox/dequeue', (agent: Agent, message: UserMessage, placement) => {
retire(agent, message.id, placement)
// A later claim proves any earlier claimed item either published (and
// was retired by session/event) or its admission ended without one.
retireClaimed(agent.id)
const entry = queuedMirror.get(agent.id)?.find(candidate =>
candidate.message.id === message.id
&& candidate.steering === (placement === 'steering'))
if (entry !== undefined) entry.claimed = true
}),
ctx.on('session/event', (session: Session, event: SessionEvent) => {
if (event.type === 'user/message') {
retire(session.id, event.data.id, 'queued')
} else if (event.type === 'steering/message') {
retire(session.id, (event.data as { message: UserMessage }).message.id, 'steering')
}
}),
ctx.on('agent/inbox/discard', (agent: Agent, messages: UserMessage[]) => {
for (const message of messages) retire(agent, message.id)
for (const message of messages) retire(agent.id, message.id)
}),
ctx.on('agent/status', (agent: Agent, status: AgentStatus) => {
if (status === 'idle') retireClaimed(agent.id)
}),
ctx.on('session/disposed', (session: Session) => {
queuedMirror.delete(session.id)
@@ -1133,48 +1161,47 @@ export function createApiProxy(ctx: Context, defaults: ApiProxyDefaults): ApiPro
const { sessionId, provider, model, reasoningEffort } = request.payload
const found = await agentFor(sessionId)
if ('error' in found) return err(request, found.error)
try {
const resolved = await ctx.llm.resolveCallConfig({
provider,
model,
...reasoningEffort === undefined
? {}
: { reasoningEffort: ReasoningEffortId(reasoningEffort) },
})
// An image-bearing log replays into every later request, and both
// wire routes reject image content on text-only models — accepting
// this selection would strand the session (every turn fails, no
// in-product recovery). Refuse at the selection boundary instead.
// The pending inbox counts too: a queued image prompt enters the log
// only when claimed, which would happen AFTER this switch landed.
const queuedImage = (queuedMirror.get(sessionId) ?? [])
.some(entry => contentHasImage(entry.message.content))
if (queuedImage || sessionHasImage(found.agent.session.events)) {
const info = await ctx.llm.resolveModelInfo(resolved.provider, resolved.model)
if (info.inputModalities !== undefined && !info.inputModalities.includes('image')) {
return err(request, {
code: 'model-unavailable',
message: `Model "${resolved.model}" does not accept image input, but this session's history already contains images; select an image-capable model.`,
details: { provider, model },
})
return serializeImageAdmission(found.agent, async () => {
try {
const resolved = await ctx.llm.resolveCallConfig({
provider,
model,
...reasoningEffort === undefined
? {}
: { reasoningEffort: ReasoningEffortId(reasoningEffort) },
})
// A current image-bearing surface replays into the next request,
// while a dequeued prompt remains pending until its message event
// publishes. Refuse a text-only route at this shared boundary.
const queuedImage = (queuedMirror.get(sessionId) ?? [])
.some(entry => contentHasImage(entry.message.content))
if (queuedImage || messagesHaveImage(found.agent.session.deriveMessages())) {
const info = await ctx.llm.resolveModelInfo(resolved.provider, resolved.model)
if (info.inputModalities !== undefined && !info.inputModalities.includes('image')) {
return err(request, {
code: 'model-unavailable',
message: `Model "${resolved.model}" does not accept image input, but this session's history already contains images; select an image-capable model.`,
details: { provider, model },
})
}
}
const selected: AgentLlmTarget = {
provider: resolved.provider,
model: resolved.model,
...resolved.reasoningEffort === undefined
? {}
: { reasoningEffort: resolved.reasoningEffort },
}
targetFor(found.agent).current = selected
return ok(request, { selected: { ...selected } })
} catch (error: unknown) {
return err(request, {
code: 'model-unavailable',
message: error instanceof Error ? error.message : String(error),
details: { provider, model },
})
}
const selected: AgentLlmTarget = {
provider: resolved.provider,
model: resolved.model,
...resolved.reasoningEffort === undefined
? {}
: { reasoningEffort: resolved.reasoningEffort },
}
targetFor(found.agent).current = selected
return ok(request, { selected: { ...selected } })
} catch (error: unknown) {
return err(request, {
code: 'model-unavailable',
message: error instanceof Error ? error.message : String(error),
details: { provider, model },
})
}
})
},
async rename(request) {
@@ -1214,36 +1241,40 @@ export function createApiProxy(ctx: Context, defaults: ApiProxyDefaults): ApiPro
const agent = found.agent
// The rpcId rides MessageSource into user/message (merge declaration in api/sessions.ts; provisional correlation).
const source: MessageSource = { kind: 'user', rpcId: request.rpcId }
try {
if (content.some(part => part.type === 'image')) {
const target = targetFor(agent).current
const provider = target.provider
const model = target.model
const modelInfo = await ctx.llm.resolveModelInfo(provider, model)
if (modelInfo.inputModalities !== undefined && !modelInfo.inputModalities.includes('image')) {
const hasImage = content.some(part => part.type === 'image')
const admit = async (): Promise<RpcResponse<{ accepted: true }>> => {
try {
if (hasImage) {
const target = targetFor(agent).current
const provider = target.provider
const model = target.model
const modelInfo = await ctx.llm.resolveModelInfo(provider, model)
if (modelInfo.inputModalities !== undefined && !modelInfo.inputModalities.includes('image')) {
return err(request, {
code: 'attachment-error',
message: `Model "${model}" does not support image input.`,
details: { reason: 'MODEL_DOES_NOT_SUPPORT_IMAGES' },
})
}
}
const durable = await durablePromptContent(ctx, content)
const message: UserMessage = createUserMessage({ content: durable, source })
if (mode === 'steer') agent.steer(message)
else agent.followup(message)
} catch (error: unknown) {
if (error instanceof AttachmentError) {
return err(request, {
code: 'attachment-error',
message: `Model "${model}" does not support image input.`,
details: { reason: 'MODEL_DOES_NOT_SUPPORT_IMAGES' },
message: error.message,
details: { reason: error.code },
})
}
// A synchronous throw from steer/followup means disposed or invalid input; surface as agent-busy with the reason attached.
return err(request, { code: 'agent-busy', message: 'prompt rejected', details: { reason: String(error) } })
}
const durable = await durablePromptContent(ctx, content)
const message: UserMessage = createUserMessage({ content: durable, source })
if (mode === 'steer') agent.steer(message)
else agent.followup(message)
} catch (error: unknown) {
if (error instanceof AttachmentError) {
return err(request, {
code: 'attachment-error',
message: error.message,
details: { reason: error.code },
})
}
// A synchronous throw from steer/followup means disposed or invalid input; surface as agent-busy with the reason attached.
return err(request, { code: 'agent-busy', message: 'prompt rejected', details: { reason: String(error) } })
return ok(request, { accepted: true as const })
}
return ok(request, { accepted: true as const })
return hasImage ? serializeImageAdmission(agent, admit) : admit()
},
async attachment(request) {

View File

@@ -302,7 +302,7 @@ describe('session/queued frames', () => {
expect(replayFrames.filter(f => f.type === 'session/queued')).toEqual(liveFrames)
})
it('retires mirror entries on their terminal dequeue', async () => {
it('retains each dequeued entry until its durable message publishes', async () => {
const ctx = await harness()
const api = createApiProxy(ctx, DEFAULTS)
const agent = stubAgent(ctx)
@@ -311,15 +311,50 @@ describe('session/queued frames', () => {
ctx.emit('agent/inbox/enqueue', agent, queued, 'queued')
ctx.emit('agent/inbox/enqueue', agent, steering, 'steering')
ctx.emit('agent/inbox/dequeue', agent, queued, 'queued')
ctx.emit('agent/inbox/dequeue', agent, steering, 'steering')
const pendingAbort = new AbortController()
const pending = await collect<MuxFrame>(
api.events.mux({ rpcId: RpcId('t-mux-dequeued'), payload: {} }, pendingAbort.signal), 3, pendingAbort)
expect(pending.filter(f => f.type === 'session/queued')).toEqual([
{ type: 'session/queued', sessionId: agent.id, message: queued, steering: false },
{ type: 'session/queued', sessionId: agent.id, message: steering, steering: true },
])
agent.session.append('user/message', queued, { surfaceOp: 'append' })
ctx.emit('agent/inbox/dequeue', agent, steering, 'steering')
agent.session.append('steering/message', { turn: 1, message: steering }, { surfaceOp: 'append' })
const abort = new AbortController()
const frames = await collect<MuxFrame>(
api.events.mux({ rpcId: RpcId('t-mux-after'), payload: {} }, abort.signal), 1, abort)
expect(frames.filter(f => f.type === 'session/queued')).toHaveLength(0)
})
it('retires the matching placement when one message identity is queued and steering', async () => {
it('retires claimed entries whose admission ends without publication', async () => {
const ctx = await harness()
const api = createApiProxy(ctx, DEFAULTS)
const agent = stubAgent(ctx)
const rejected = inboxMessage('m-rejected', 'rejected')
const successor = inboxMessage('m-successor', 'successor')
ctx.emit('agent/inbox/enqueue', agent, rejected, 'queued')
ctx.emit('agent/inbox/dequeue', agent, rejected, 'queued')
ctx.emit('agent/inbox/enqueue', agent, successor, 'queued')
ctx.emit('agent/inbox/dequeue', agent, successor, 'queued')
const pendingAbort = new AbortController()
const pending = await collect<MuxFrame>(
api.events.mux({ rpcId: RpcId('t-mux-rejected'), payload: {} }, pendingAbort.signal), 2, pendingAbort)
expect(pending.filter(f => f.type === 'session/queued')).toEqual([
{ type: 'session/queued', sessionId: agent.id, message: successor, steering: false },
])
ctx.emit('agent/status', agent, 'idle')
const idleAbort = new AbortController()
const idle = await collect<MuxFrame>(
api.events.mux({ rpcId: RpcId('t-mux-rejected-idle'), payload: {} }, idleAbort.signal), 1, idleAbort)
expect(idle.filter(f => f.type === 'session/queued')).toHaveLength(0)
})
it('retires the matching published placement when one message identity is queued and steering', async () => {
const ctx = await harness()
const api = createApiProxy(ctx, DEFAULTS)
const agent = stubAgent(ctx)
@@ -328,6 +363,7 @@ describe('session/queued frames', () => {
ctx.emit('agent/inbox/enqueue', agent, repeated, 'steering')
ctx.emit('agent/inbox/dequeue', agent, inboxMessage('unknown', 'not queued'), 'queued')
ctx.emit('agent/inbox/dequeue', agent, repeated, 'steering')
agent.session.append('steering/message', { turn: 1, message: repeated }, { surfaceOp: 'append' })
const abort = new AbortController()
const frames = await collect<MuxFrame>(

View File

@@ -117,10 +117,26 @@ function expectValue<T>(response: { result: { ok: true; value: T } | { ok: false
return response.result.value
}
function registerTextOnly(ctx: Context): void {
ctx.llm.registerAdapter(['text-only'], new class extends CatalogAdapter {
override resolveModel(provider: string, model: string): Promise<LlmResolvedModelInfo> {
return Promise.resolve({ provider, id: model, name: model, inputModalities: ['text'] })
}
}('Text Only', []))
}
describe('Web session model selection', () => {
it('accepts ordered multi-image prompts and rejects configured batch-limit excess before persistence', async () => {
const { ctx, agent, sessionId } = await harness()
const validateImage = vi.fn((_input: { data: Uint8Array }): void => {})
let secondValidationStarted!: () => void
let releaseSecondValidation!: () => void
const secondStarted = new Promise<void>((resolve) => { secondValidationStarted = resolve })
const secondReleased = new Promise<void>((resolve) => { releaseSecondValidation = resolve })
const validateImage = vi.fn(async (input: { data: Uint8Array }): Promise<void> => {
if (input.data[0] !== 2) return
secondValidationStarted()
await secondReleased
})
const saveImage = vi.fn((input: { data: Uint8Array; mediaType: 'image/png'; name?: string }) => {
return Promise.resolve({
attachmentId: `att-${String(input.data[0])}`,
@@ -141,11 +157,15 @@ describe('Web session model selection', () => {
const api = createApiProxy(ctx, { provider: 'deepseek', model: 'deepseek-chat', cwd: '/tmp', workspaceRoot: '/tmp' })
const first = { type: 'image' as const, mediaType: 'image/png' as const, data: 'AQ==', name: 'first.png' }
const second = { type: 'image' as const, mediaType: 'image/png' as const, data: 'Ag==', name: 'second.png' }
const accepted = await api.sessions.prompt(request({
const accepting = api.sessions.prompt(request({
sessionId,
mode: 'queue' as const,
content: [first, { type: 'text' as const, text: 'compare' }, second],
}))
await secondStarted
expect(saveImage).not.toHaveBeenCalled()
releaseSecondValidation()
const accepted = await accepting
expect(accepted.result).toMatchObject({ ok: true, value: { accepted: true } })
expect(validateImage.mock.calls.map(([input]) => [...input.data])).toEqual([[1], [2]])
expect(saveImage.mock.calls.map(([input]) => [...input.data])).toEqual([[1], [2]])
@@ -294,13 +314,9 @@ describe('Web session model selection', () => {
await ctx.fiber.dispose()
})
it('refuses a text-only selection once the session log carries an image', async () => {
it('refuses a text-only selection while current derived history carries an image', async () => {
const { ctx, sessionId, agent } = await harness()
ctx.llm.registerAdapter(['text-only'], new class extends CatalogAdapter {
override resolveModel(provider: string, model: string): Promise<LlmResolvedModelInfo> {
return Promise.resolve({ provider, id: model, name: model, inputModalities: ['text'] })
}
}('Text Only', []))
registerTextOnly(ctx)
ctx.llm.registerAdapter(['vision'], new class extends CatalogAdapter {
override resolveModel(provider: string, model: string): Promise<LlmResolvedModelInfo> {
return Promise.resolve({ provider, id: model, name: model, inputModalities: ['text', 'image'] })
@@ -318,7 +334,7 @@ describe('Web session model selection', () => {
content: [{ type: 'image', attachment: { attachmentId: 'att-1', mediaType: 'image/png', bytes: 8, width: 1, height: 1 } }],
} as never, { surfaceOp: 'append' })
// The log is immutable: a text-only route would fail every later turn.
// The image remains on the current request surface, so a text-only route would fail the next turn.
const stranded = await api.sessions.selectModel(request({
sessionId, provider: 'text-only', model: 'plain',
}))
@@ -337,31 +353,87 @@ describe('Web session model selection', () => {
await ctx.fiber.dispose()
})
it('refuses a text-only selection while an image prompt is still queued (not yet logged)', async () => {
it('keeps a dequeued image pending until publication, then follows the compacted surface', async () => {
const { ctx, sessionId, agent } = await harness()
ctx.llm.registerAdapter(['text-only'], new class extends CatalogAdapter {
override resolveModel(provider: string, model: string): Promise<LlmResolvedModelInfo> {
return Promise.resolve({ provider, id: model, name: model, inputModalities: ['text'] })
}
}('Text Only', []))
registerTextOnly(ctx)
const api = createApiProxy(ctx, { provider: 'deepseek', model: 'deepseek-chat', cwd: '/tmp', workspaceRoot: '/tmp' })
// The queued message enters the session log only when claimed — after a
// model switch would already have landed. The pending-inbox mirror must
// therefore gate the switch too.
ctx.emit('agent/inbox/enqueue', agent, {
const queued = {
id: 'q-1', role: 'user', source: { kind: 'user' },
content: [{ type: 'image', attachment: { attachmentId: 'att-q', mediaType: 'image/png', bytes: 8, width: 1, height: 1 } }],
} as never, 'queued')
const stranded = await api.sessions.selectModel(request({ sessionId, provider: 'text-only', model: 'plain' }))
expect(stranded.result.ok).toBe(false)
// Claiming the message drains the mirror; the log now owns the decision.
ctx.emit('agent/inbox/dequeue', agent, { id: 'q-1' } as never, 'queued')
} as never
ctx.emit('agent/inbox/enqueue', agent, queued, 'queued')
expect((await api.sessions.selectModel(request({ sessionId, provider: 'text-only', model: 'plain' }))).result.ok).toBe(false)
// Dequeue precedes the authoritative append, so it cannot open a switch window.
ctx.emit('agent/inbox/dequeue', agent, queued, 'queued')
expect((await api.sessions.selectModel(request({ sessionId, provider: 'text-only', model: 'plain' }))).result.ok).toBe(false)
const imageEvent = agent.session.append('user/message', queued, { surfaceOp: 'append' })
expect((await api.sessions.selectModel(request({ sessionId, provider: 'text-only', model: 'plain' }))).result.ok).toBe(false)
// Publication retires the mirror; once compaction shadows the image, the
// current model-visible surface no longer requires an image-capable route.
agent.session.append('user/message', {
id: 'summary', role: 'user', source: { kind: 'plugin', plugin: 'compact' },
content: [{ type: 'text', text: 'image summarized' }],
} as never, {
surfaceOp: { op: 'replace', start: imageEvent.seq, end: imageEvent.seq },
sourceEventSeqs: [imageEvent.seq],
})
expect(expectValue(await api.sessions.selectModel(request({
sessionId, provider: 'text-only', model: 'plain',
}))).selected).toEqual({ provider: 'text-only', model: 'plain' })
await ctx.fiber.dispose()
})
it('serializes an image save with a concurrent model selection', async () => {
const { ctx, sessionId, agent } = await harness()
registerTextOnly(ctx)
let saveStarted!: () => void
let releaseSave!: () => void
const started = new Promise<void>((resolve) => { saveStarted = resolve })
const released = new Promise<void>((resolve) => { releaseSave = resolve })
const ref = { attachmentId: 'att-race', mediaType: 'image/png' as const, bytes: 1, width: 1, height: 1 }
ctx.provide('attachments', {
imageLimits: {
maxImageBytes: 1,
maxImagesPerMessage: 1,
maxMessageImageBytes: 1,
maxImagePixels: 1,
mediaTypes: ['image/png'],
},
validateImage: () => Promise.resolve(),
saveImage: async () => {
saveStarted()
await released
return ref
},
} as never)
Object.assign(agent, {
followup(message: UserMessage) {
ctx.emit('agent/inbox/enqueue', agent, message, 'queued')
},
})
const api = createApiProxy(ctx, { provider: 'deepseek', model: 'deepseek-chat', cwd: '/tmp', workspaceRoot: '/tmp' })
const prompt = api.sessions.prompt(request({
sessionId,
mode: 'queue' as const,
content: [{ type: 'image' as const, mediaType: 'image/png' as const, data: 'AA==' }],
}))
await started
const selection = api.sessions.selectModel(request({ sessionId, provider: 'text-only', model: 'plain' }))
expect(await Promise.race([
selection.then(() => 'settled' as const),
new Promise<'pending'>((resolve) => { setTimeout(() => { resolve('pending') }, 0) }),
])).toBe('pending')
releaseSave()
expect((await prompt).result.ok).toBe(true)
expect((await selection).result.ok).toBe(false)
await ctx.fiber.dispose()
})
it('authorizes an attachment read referenced only from wrapped message content', async () => {
const { ctx, sessionId, agent } = await harness()
const ref = { attachmentId: 'att-w', mediaType: 'image/png' as const, bytes: 4, width: 1, height: 1 }
@@ -369,9 +441,8 @@ describe('Web session model selection', () => {
readImage: () => Promise.resolve({ ref, data: new Uint8Array([1, 2, 3, 4]) }),
} as never)
const api = createApiProxy(ctx, { provider: 'deepseek', model: 'deepseek-chat', cwd: '/tmp', workspaceRoot: '/tmp' })
// The only reference lives inside an assistant/message wrapper the same
// walk that gates model selection must authorize the read, or a real host
// denies galleries the fixture (with its own authorization mirror) serves.
// The only reference lives inside an assistant/message wrapper; the
// authorization walk must follow that durable event shape.
agent.session.append('assistant/message', {
turn: 1, step: 0,
message: { id: 'a-1', role: 'assistant', source: { kind: 'model', provider: 'p', model: 'm' }, content: [{ type: 'image', attachment: ref }] },
@@ -383,7 +454,7 @@ describe('Web session model selection', () => {
await ctx.fiber.dispose()
})
it('detects images on every replayed route: wrapped messages, streamed blocks, nested tool results', async () => {
it('detects images in wrapped messages and nested tool results on the current surface', async () => {
const image = { type: 'image', attachment: { attachmentId: 'att-x', mediaType: 'image/png', bytes: 8, width: 1, height: 1 } }
const cases: { label: string; append: (agent: Agent) => void }[] = [
{
@@ -394,14 +465,6 @@ describe('Web session model selection', () => {
} as never, { surfaceOp: 'append' })
},
},
{
label: 'streamed assistant block',
append: (agent) => {
agent.session.append('assistant/chunk', {
turn: 1, step: 0, chunk: { type: 'block-end', index: 0, block: image },
} as never)
},
},
{
label: 'nested tool-result content',
append: (agent) => {
@@ -414,11 +477,7 @@ describe('Web session model selection', () => {
]
for (const { label, append } of cases) {
const { ctx, sessionId, agent } = await harness()
ctx.llm.registerAdapter(['text-only'], new class extends CatalogAdapter {
override resolveModel(provider: string, model: string): Promise<LlmResolvedModelInfo> {
return Promise.resolve({ provider, id: model, name: model, inputModalities: ['text'] })
}
}('Text Only', []))
registerTextOnly(ctx)
const api = createApiProxy(ctx, { provider: 'deepseek', model: 'deepseek-chat', cwd: '/tmp', workspaceRoot: '/tmp' })
append(agent)
const stranded = await api.sessions.selectModel(request({ sessionId, provider: 'text-only', model: 'plain' }))