feat(fs): add a minimal read_image tool over the attachment and fs seams
The model reads a PNG/JPEG/WebP/GIF file, the bytes commit through the durable attachment lifecycle, and the tool result carries the real ImageBlock so the image enters context from the next request onward. FileSystem gains a bounded readBytes primitive (local + E2B providers); registration is conditional on the attachment store, and a strict execution gate refuses routes that do not declare image input, so a text route's durable history stays free of image blocks. llm-replay models may declare inputModalities, letting keyless ACP snapshots pin both the sha256-referenced success and the verbatim refusal. Supersedes the withdrawn route-scoped design of PR #598; the decision record is .agents/notes/implemented/feature/2026-08-10-minimal-read-image-tool.md.
This commit is contained in:
creatixchu
@@ -11,6 +11,7 @@ import type {} from '@deepseek-ai/dsh-user-approval'
|
||||
import { applyReadTool, READ_LIMIT, STREAM_MIN_SIZE } from './read.ts'
|
||||
import { applyWriteTool } from './write.ts'
|
||||
import { applyEditTool } from './edit.ts'
|
||||
import { applyReadImageTool } from './read-image.ts'
|
||||
import { READ_MAX_BYTES, READ_MAX_LINE_LENGTH } from './read-render.ts'
|
||||
import { FsSandboxSurface } from './sandbox.ts'
|
||||
|
||||
@@ -63,6 +64,12 @@ export function apply(ctx: Context, config: Config): void {
|
||||
maxBytes: resolved.readMaxBytes,
|
||||
streamMinSize: resolved.readStreamMinSize,
|
||||
})
|
||||
// read_image is composition-conditional: without a mounted attachment store
|
||||
// the deployment cannot durably commit image bytes, so the tool never
|
||||
// registers; the execute body keeps a defensive re-check for direct callers.
|
||||
ctx.inject(['attachments'], (imageCtx) => {
|
||||
applyReadImageTool(imageCtx)
|
||||
})
|
||||
// One escalation surface shared by both mutating tools: advertisement gating,
|
||||
// per-call policy resolution, and denial-marker mapping, all keyed off whether
|
||||
// the mounted ctx.fs confines (ctx.fs.sandboxMode).
|
||||
|
||||
231
packages/fs/tool-fs/src/read-image.ts
Normal file
231
packages/fs/tool-fs/src/read-image.ts
Normal file
@@ -0,0 +1,231 @@
|
||||
/**
|
||||
* The model-facing `read_image` tool: reads a PNG/JPEG/WebP/GIF file, durably
|
||||
* commits its bytes through the attachment service (the same lifecycle as a
|
||||
* user-uploaded image), and returns an image block so the image enters model
|
||||
* context from the next request onward.
|
||||
*
|
||||
* The route gate is deliberately stricter than the host upload preflight: a
|
||||
* tool result enters durable session history, so emitting an image on a route
|
||||
* that cannot carry it would break that route's continuation. Unknown
|
||||
* capability therefore refuses instead of relying on the adapter guard.
|
||||
* @module @deepseek-ai/dsh-tool-fs/src/read-image
|
||||
*/
|
||||
|
||||
import { basename, extname } from 'node:path'
|
||||
import type { Context } from 'cordis'
|
||||
import { AttachmentError, AttachmentId } from '@deepseek-ai/dsh-attachment'
|
||||
import type { ImageAttachmentRef, ImageMediaType } from '@deepseek-ai/dsh-attachment'
|
||||
import { createUserMessage } from '@deepseek-ai/dsh-llm'
|
||||
import type { ContentBlock } from '@deepseek-ai/dsh-llm'
|
||||
import { defineTool } from '@deepseek-ai/dsh-tools'
|
||||
import type { GenericCallView, ToolExecution } from '@deepseek-ai/dsh-tools'
|
||||
import { FsError } from '@deepseek-ai/dsh-fs'
|
||||
import type {} from '@deepseek-ai/dsh-fs'
|
||||
import { sessionResolveOptions } from './session-cwd.ts'
|
||||
|
||||
/** Extensions `read_image` accepts; magic-byte validation at the attachment service stays authoritative. */
|
||||
const IMAGE_EXTENSIONS: Readonly<Record<string, ImageMediaType>> = {
|
||||
'.png': 'image/png',
|
||||
'.jpg': 'image/jpeg',
|
||||
'.jpeg': 'image/jpeg',
|
||||
'.webp': 'image/webp',
|
||||
'.gif': 'image/gif',
|
||||
}
|
||||
|
||||
/** The canonical outcome declared by the `read_image` output schema. */
|
||||
export interface ImageReadValue {
|
||||
path: string
|
||||
image: {
|
||||
attachmentId: string
|
||||
mediaType: ImageMediaType
|
||||
bytes: number
|
||||
width: number
|
||||
height: number
|
||||
name?: string
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Map a model-supplied path to its declared image media type by extension.
|
||||
* @param filePath - the raw `file_path` argument (not yet resolved).
|
||||
* @returns the declared media type, or undefined when the path does not claim an image.
|
||||
*/
|
||||
export function imageMediaTypeForPath(filePath: string): ImageMediaType | undefined {
|
||||
return IMAGE_EXTENSIONS[extname(filePath).toLowerCase()]
|
||||
}
|
||||
|
||||
/**
|
||||
* Enforce the strict image-capability gate for the calling route. Resolves the
|
||||
* session's latest routed provider/model (request header config, then agent
|
||||
* options) and requires the exact resolved route to declare `image` input explicitly.
|
||||
* @param ctx - the plugin context used to resolve the optional `llm` service.
|
||||
* @param exec - the tool-execution context supplying the calling agent.
|
||||
* @param displayPath - the path rendered in refusal messages.
|
||||
*/
|
||||
export async function assertImageCapableRoute(ctx: Context, exec: ToolExecution, displayPath: string): Promise<void> {
|
||||
const routed = exec.agent?.session.requestHeader()?.config
|
||||
const provider = routed?.provider ?? exec.agent?.options.provider
|
||||
const model = routed?.model ?? exec.agent?.options.model
|
||||
const llm = ctx.get('llm')
|
||||
if (provider === undefined || model === undefined || llm === undefined) {
|
||||
throw new Error(`cannot read "${displayPath}" as an image: the current model route could not be resolved`)
|
||||
}
|
||||
const active = await llm.resolveModelInfo(provider, model, exec.signal)
|
||||
if (active.inputModalities === undefined || !active.inputModalities.includes('image')) {
|
||||
throw new Error(`cannot read "${displayPath}" as an image: model "${model}" does not declare image input; switch to an image-capable model to read images`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Re-brand a canonical image outcome into the durable attachment reference an
|
||||
* `ImageBlock` carries.
|
||||
* @param image - the canonical image metadata from the output schema.
|
||||
* @returns the branded attachment reference.
|
||||
*/
|
||||
export function imageRefFromValue(image: ImageReadValue['image']): ImageAttachmentRef {
|
||||
return {
|
||||
attachmentId: AttachmentId(image.attachmentId),
|
||||
mediaType: image.mediaType,
|
||||
bytes: image.bytes,
|
||||
width: image.width,
|
||||
height: image.height,
|
||||
...image.name === undefined ? {} : { name: image.name },
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Format an image read as the model-facing envelope beside its image block.
|
||||
* @param displayPath - the backend-resolved path rendered in the envelope's `<path>` element.
|
||||
* @param image - the canonical image metadata to summarize.
|
||||
* @returns the model-facing envelope; the image itself rides the adjacent image block.
|
||||
*/
|
||||
export function formatImageReadOutput(displayPath: string, image: ImageReadValue['image']): string {
|
||||
return `<path>${displayPath}</path>
|
||||
<type>image</type>
|
||||
<content>
|
||||
${image.mediaType} image, ${image.width}x${image.height} px, ${image.bytes} bytes
|
||||
</content>`
|
||||
}
|
||||
|
||||
/**
|
||||
* Project one canonical image read into its model-facing envelope and image.
|
||||
* @param value - the canonical image-read outcome.
|
||||
* @returns the two content blocks used by native and nested dispatches.
|
||||
*/
|
||||
function imageReadContent(value: ImageReadValue): ContentBlock[] {
|
||||
return [
|
||||
{ type: 'text', text: formatImageReadOutput(value.path, value.image) },
|
||||
{ type: 'image', attachment: imageRefFromValue(value.image) },
|
||||
]
|
||||
}
|
||||
|
||||
/**
|
||||
* Register the `read_image` tool. Execution gates on the optional
|
||||
* `attachments`/`llm` services and the calling route's declared image input;
|
||||
* registration itself is unconditional so denial happens at the operation
|
||||
* boundary rather than through schema omission.
|
||||
* @param ctx - the plugin context; registrations are effects scoped to it, and
|
||||
* execution uses its `fs` service plus the optional `attachments`/`llm` services.
|
||||
*/
|
||||
export function applyReadImageTool(ctx: Context): void {
|
||||
ctx.tools.register(defineTool({
|
||||
name: 'read_image',
|
||||
description: 'Read a PNG/JPEG/WebP/GIF file and return the image itself. Requires the current model to accept image input.',
|
||||
parameters: {
|
||||
file_path: { type: 'string', required: true, description: 'Path to the image file, resolved by the filesystem backend.' },
|
||||
},
|
||||
output: {
|
||||
schema: {
|
||||
type: 'object',
|
||||
additionalProperties: false,
|
||||
properties: {
|
||||
path: { type: 'string', required: true },
|
||||
image: {
|
||||
type: 'object',
|
||||
additionalProperties: false,
|
||||
required: true,
|
||||
properties: {
|
||||
attachmentId: { type: 'string', required: true },
|
||||
mediaType: { type: 'string', enum: ['image/png', 'image/jpeg', 'image/webp', 'image/gif'], required: true },
|
||||
bytes: { type: 'integer', required: true },
|
||||
width: { type: 'integer', required: true },
|
||||
height: { type: 'integer', required: true },
|
||||
name: { type: 'string' },
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
render: (_args, value) => imageReadContent(value),
|
||||
},
|
||||
// Content-addressed attachment writes are idempotent, so concurrent reads
|
||||
// of the same file cannot conflict.
|
||||
isConcurrencySafe: () => true,
|
||||
async execute(args, exec) {
|
||||
if (args.file_path.trim().length === 0) throw new Error('file_path must be a non-empty string')
|
||||
|
||||
// Every gate runs before any filesystem I/O so a refusal never leaks
|
||||
// partial reads or attachment writes.
|
||||
const mediaType = imageMediaTypeForPath(args.file_path)
|
||||
if (mediaType === undefined) {
|
||||
throw new Error(`cannot read "${args.file_path}": read_image only accepts PNG/JPEG/WebP/GIF paths`)
|
||||
}
|
||||
const attachments = ctx.get('attachments')
|
||||
if (attachments === undefined) {
|
||||
throw new Error(`cannot read "${args.file_path}" as an image: no attachment service is mounted`)
|
||||
}
|
||||
if (!attachments.imageLimits.mediaTypes.includes(mediaType)) {
|
||||
throw new Error(`cannot read "${args.file_path}": ${mediaType} images are not accepted by this deployment`)
|
||||
}
|
||||
await assertImageCapableRoute(ctx, exec, args.file_path)
|
||||
|
||||
const target = await ctx.fs.resolve(args.file_path, sessionResolveOptions(exec, args.file_path))
|
||||
const info = await ctx.fs.stat(target, exec.signal)
|
||||
if (!info) throw new FsError(`cannot read "${target.displayPath}": not found`, 'FS_NOT_FOUND')
|
||||
if (info.type !== 'file') throw new FsError(`cannot read "${target.displayPath}": not a regular file`, 'FS_NOT_REGULAR_FILE')
|
||||
|
||||
const data = await ctx.fs.readBytes(target, exec.signal, attachments.imageLimits.maxImageBytes)
|
||||
// Persist before returning: the image block must reference a durably
|
||||
// committed object by the time the tool/result event is appended.
|
||||
let ref: ImageAttachmentRef
|
||||
try {
|
||||
ref = await attachments.saveImage({ data, mediaType, name: basename(target.displayPath) })
|
||||
} catch (error: unknown) {
|
||||
if (!(error instanceof AttachmentError) || error.code !== 'IMAGE_TYPE_MISMATCH') throw error
|
||||
const extension = extname(target.displayPath).toLowerCase()
|
||||
throw new Error(
|
||||
`cannot read "${target.displayPath}": the ${extension} extension declares ${mediaType}, but the bytes use a different image format; rename the file to match its actual PNG/JPEG/WebP/GIF format`,
|
||||
{ cause: error },
|
||||
)
|
||||
}
|
||||
ctx.emit('fs/observed', target, { kind: 'present', version: info.version }, exec)
|
||||
const value: ImageReadValue = {
|
||||
path: target.displayPath,
|
||||
image: {
|
||||
attachmentId: ref.attachmentId,
|
||||
mediaType: ref.mediaType,
|
||||
bytes: ref.bytes,
|
||||
width: ref.width,
|
||||
height: ref.height,
|
||||
...ref.name === undefined ? {} : { name: ref.name },
|
||||
},
|
||||
}
|
||||
if (exec.parent !== undefined) {
|
||||
exec.deferContext(createUserMessage({
|
||||
content: imageReadContent(value),
|
||||
source: { kind: 'plugin', plugin: 'tool-fs' },
|
||||
}))
|
||||
}
|
||||
return value
|
||||
},
|
||||
// Pure display: a generic card in the read family with a follow-along
|
||||
// location on the image file.
|
||||
presentCall(args): GenericCallView {
|
||||
return {
|
||||
card: 'generic',
|
||||
title: `Read image ${args.file_path}`,
|
||||
kind: 'read',
|
||||
locations: [{ path: args.file_path }],
|
||||
}
|
||||
},
|
||||
}))
|
||||
}
|
||||
Reference in New Issue
Block a user