/** * The model-facing `read_image` tool: reads a PNG/JPEG/WebP/GIF file, durably * commits its bytes through the attachment service (the same lifecycle as a * user-uploaded image), and returns an image block so the image enters model * context from the next request onward. * * The route gate is deliberately stricter than the host upload preflight: a * tool result enters durable session history, so emitting an image on a route * that cannot carry it would break that route's continuation. Unknown * capability therefore refuses instead of relying on the adapter guard. * @module @deepseek-ai/dsh-tool-fs/src/read-image */ import { basename, extname } from 'node:path' import type { Context } from '@deepseek-ai/cordis' import { AttachmentError, AttachmentId } from '@deepseek-ai/dsh-attachment' import type { ImageAttachmentRef, ImageMediaType } from '@deepseek-ai/dsh-attachment' import { createUserMessage } from '@deepseek-ai/dsh-llm' import type { ContentBlock } from '@deepseek-ai/dsh-llm' import { defineTool } from '@deepseek-ai/dsh-tools' import type { GenericCallView, ToolExecution } from '@deepseek-ai/dsh-tools' import type {} from '@deepseek-ai/dsh-fs' import { resolveRegularReadTarget } from './read-target.ts' /** Extensions `read_image` accepts; magic-byte validation at the attachment service stays authoritative. */ const IMAGE_EXTENSIONS: Readonly> = { '.png': 'image/png', '.jpg': 'image/jpeg', '.jpeg': 'image/jpeg', '.webp': 'image/webp', '.gif': 'image/gif', } /** The canonical outcome declared by the `read_image` output schema. */ export interface ImageReadValue { path: string image: { attachmentId: string mediaType: ImageMediaType bytes: number width: number height: number name?: string } } /** * Map a model-supplied path to its declared image media type by extension. * @param filePath - the raw `file_path` argument (not yet resolved). * @returns the declared media type, or undefined when the path does not claim an image. */ export function imageMediaTypeForPath(filePath: string): ImageMediaType | undefined { return IMAGE_EXTENSIONS[extname(filePath).toLowerCase()] } /** * Enforce the strict image-capability gate for the calling route. Resolves the * session's latest routed provider/model (request header config, then agent * options) and requires the exact resolved route to declare `image` input explicitly. * @param ctx - the plugin context used to resolve the optional `llm` service. * @param exec - the tool-execution context supplying the calling agent. * @param requestedPath - the raw, not-yet-resolved path rendered in refusal messages. */ export async function assertImageCapableRoute(ctx: Context, exec: ToolExecution, requestedPath: string): Promise { const routed = exec.agent?.session.requestHeader()?.config const provider = routed?.provider ?? exec.agent?.options.provider const model = routed?.model ?? exec.agent?.options.model const llm = ctx.get('llm') if (provider === undefined || model === undefined || llm === undefined) { throw new Error(`cannot read "${requestedPath}" as an image: the current model route could not be resolved`) } const active = await llm.resolveModelInfo(provider, model, exec.signal) if (active.inputModalities === undefined || !active.inputModalities.includes('image')) { throw new Error(`cannot read "${requestedPath}" as an image: model "${model}" does not declare image input; switch to an image-capable model to read images`) } } /** * Re-brand a canonical image outcome into the durable attachment reference an * `ImageBlock` carries. * @param image - the canonical image metadata from the output schema. * @returns the branded attachment reference. */ export function imageRefFromValue(image: ImageReadValue['image']): ImageAttachmentRef { return { attachmentId: AttachmentId(image.attachmentId), mediaType: image.mediaType, bytes: image.bytes, width: image.width, height: image.height, ...image.name === undefined ? {} : { name: image.name }, } } /** * Format an image read as the model-facing envelope beside its image block. * @param displayPath - the backend-resolved path rendered in the envelope's `` element. * @param image - the canonical image metadata to summarize. * @returns the model-facing envelope; the image itself rides the adjacent image block. */ export function formatImageReadOutput(displayPath: string, image: ImageReadValue['image']): string { return `${displayPath} image ${image.mediaType} image, ${image.width}x${image.height} px, ${image.bytes} bytes ` } /** * Project one canonical image read into its model-facing envelope and image. * @param value - the canonical image-read outcome. * @returns the two content blocks used by native and nested dispatches. */ function imageReadContent(value: ImageReadValue): ContentBlock[] { return [ { type: 'text', text: formatImageReadOutput(value.path, value.image) }, { type: 'image', attachment: imageRefFromValue(value.image) }, ] } /** * Register the `read_image` tool into the given context. The composing plugin * owns the attachments gate: `src/index.ts` calls this inside * `ctx.inject(['attachments'], …)` so the tool exists only while a durable * store is mounted. Execution still re-checks `ctx.get('attachments')` for * direct callers and gates on the calling route's declared image input. * @param ctx - the registration scope; execution uses its `fs` service plus * the optional `attachments`/`llm` services. */ export function applyReadImageTool(ctx: Context): void { ctx.tools.register(defineTool({ name: 'read_image', description: 'Read a PNG/JPEG/WebP/GIF file and return the image itself. Requires the current model to accept image input.', parameters: { file_path: { type: 'string', required: true, description: 'Path to the image file, resolved by the filesystem backend.' }, }, output: { schema: { type: 'object', additionalProperties: false, properties: { path: { type: 'string', required: true }, image: { type: 'object', additionalProperties: false, required: true, properties: { attachmentId: { type: 'string', required: true }, mediaType: { type: 'string', enum: ['image/png', 'image/jpeg', 'image/webp', 'image/gif'], required: true }, bytes: { type: 'integer', required: true }, width: { type: 'integer', required: true }, height: { type: 'integer', required: true }, name: { type: 'string' }, }, }, }, }, render: (_args, value) => imageReadContent(value), }, // Content-addressed attachment writes are idempotent, so concurrent reads // of the same file cannot conflict. isConcurrencySafe: () => true, async execute(args, exec) { if (args.file_path.trim().length === 0) throw new Error('file_path must be a non-empty string') // Every gate runs before any filesystem I/O so a refusal never leaks // partial reads or attachment writes. const mediaType = imageMediaTypeForPath(args.file_path) if (mediaType === undefined) { throw new Error(`cannot read "${args.file_path}": read_image only accepts PNG/JPEG/WebP/GIF paths`) } const attachments = ctx.get('attachments') if (attachments === undefined) { throw new Error(`cannot read "${args.file_path}" as an image: no attachment service is mounted`) } if (!attachments.imageLimits.mediaTypes.includes(mediaType)) { throw new Error(`cannot read "${args.file_path}": ${mediaType} images are not accepted by this deployment`) } await assertImageCapableRoute(ctx, exec, args.file_path) const { target, info } = await resolveRegularReadTarget(ctx, exec, args.file_path) // The tool result is one message carrying one image, so the per-message // aggregate bound applies beside the per-image bound. const byteCap = Math.min(attachments.imageLimits.maxImageBytes, attachments.imageLimits.maxMessageImageBytes) const data = await ctx.fs.readBytes(target, exec.signal, byteCap) // Persist before returning: the image block must reference a durably // committed object by the time the tool/result event is appended. let ref: ImageAttachmentRef try { ref = await attachments.saveImage({ data, mediaType, name: basename(target.displayPath) }) } catch (error: unknown) { if (!(error instanceof AttachmentError) || error.code !== 'IMAGE_TYPE_MISMATCH') throw error const extension = extname(target.displayPath).toLowerCase() throw new Error( `cannot read "${target.displayPath}": the ${extension} extension declares ${mediaType}, but the bytes use a different image format; rename the file to match its actual format if it is PNG/JPEG/WebP/GIF, or convert it to one of those formats`, { cause: error }, ) } ctx.emit('fs/observed', target, { kind: 'present', version: info.version }, exec) const value: ImageReadValue = { path: target.displayPath, image: { attachmentId: ref.attachmentId, mediaType: ref.mediaType, bytes: ref.bytes, width: ref.width, height: ref.height, ...ref.name === undefined ? {} : { name: ref.name }, }, } if (exec.parent !== undefined) { exec.deferContext(createUserMessage({ content: imageReadContent(value), source: { kind: 'plugin', plugin: 'tool-fs' }, })) } return value }, // Pure display: a generic card in the read family with a follow-along // location on the image file. presentCall(args): GenericCallView { return { card: 'generic', title: `Read image ${args.file_path}`, kind: 'read', locations: [{ path: args.file_path }], } }, })) }