feat(fs): add a minimal read_image tool over the attachment and fs seams

The model reads a PNG/JPEG/WebP/GIF file, the bytes commit through the
durable attachment lifecycle, and the tool result carries the real
ImageBlock so the image enters context from the next request onward.
FileSystem gains a bounded readBytes primitive (local + E2B providers);
registration is conditional on the attachment store, and a strict
execution gate refuses routes that do not declare image input, so a
text route's durable history stays free of image blocks. llm-replay
models may declare inputModalities, letting keyless ACP snapshots pin
both the sha256-referenced success and the verbatim refusal.

Supersedes the withdrawn route-scoped design of PR #598; the decision
record is .agents/notes/implemented/feature/2026-08-10-minimal-read-image-tool.md.
This commit is contained in:
creatixchu
2026-08-10 15:09:07 +08:00
parent 3764ce62a5
commit 1861a3fc7c
65 changed files with 1973 additions and 67 deletions

View File

@@ -2,5 +2,5 @@
# side as of the last confirmed-consistent state. Both languages carry equal authority;
# after editing either side, bring the other along and re-record with:
# pnpm run verify-translation-pairing --write packages/support/llm-replay/README.md
README.md: ae7cb5414a28bcf33f9878f6f02220861df9a0d5
README.zh.md: 2acc0ff8e8c011126452ba3458aaeb0800d9d8f3
README.md: ade19b66f96ef26119944cbebba23c43e4b2117c
README.zh.md: 4752e03b04be175d7f3e37fc57fe9c74e1f8a303

View File

@@ -29,7 +29,7 @@ Replay keys every call by its calling session id (`GenerateOptions.sessionId`, s
| `file` | string | `$DSH_SNAPSHOT_FILE` | Path to the primary (parent) `session.jsonl` fixture. Required (config or env). |
| `overrideFile` | string | `$DSH_SNAPSHOT_OVERRIDE` | Optional `ReplayOverrideDoc` sidecar for the primary session: a bare `ReplayEntry[]` replaces its derived script, while `{ patches }` augments it by call index. |
| `childFiles` | string[] | `$DSH_SNAPSHOT_CHILD_FILES` (path-delimited) | Recorded subagent child-session logs for a nested scenario; empty for a single-session scenario. |
| `providers` | `ReplayProviderConfig[]` | — | Optional replay-only provider and model catalog. Each provider may set `retryPolicy`, and each model may publish `contextWindow`; configured routes dispatch through the replay adapter and never perform provider I/O. |
| `providers` | `ReplayProviderConfig[]` | — | Optional replay-only provider and model catalog. Each provider may set `retryPolicy`, and each model may publish `contextWindow` and declared `inputModalities` (so a scenario can exercise capability gates such as the image-capable `read_image` route check); configured routes dispatch through the replay adapter and never perform provider I/O. |
| `paceMs` | number | — (burst) | Optional per-chunk delay in ms so downstream transports (e.g. the web SSE mux observed by a real browser) see genuinely incremental delivery. A realism knob only — tests must not depend on it for correctness. Non-negative integer; abort during a pace wait cancels the stream promptly. |
```yaml

View File

@@ -29,7 +29,7 @@ fixture 就是持久化的会话日志(`<scenario>/session.jsonl`)。其 `as
| `file` | string | `$DSH_SNAPSHOT_FILE` | 主(父)`session.jsonl` fixture 的路径。必需(配置或 env)。 |
| `overrideFile` | string | `$DSH_SNAPSHOT_OVERRIDE` | 主会话的可选 `ReplayOverrideDoc` 伴随文件:裸 `ReplayEntry[]` 替换其派生脚本,`{ patches }` 则按调用索引增补该脚本。 |
| `childFiles` | string[] | `$DSH_SNAPSHOT_CHILD_FILES`(以路径分隔符分隔) | 嵌套场景中已记录的 subagent 子会话日志;单会话场景为空。 |
| `providers` | `ReplayProviderConfig[]` | 无 | 可选的仅回放提供方和模型目录。每个提供方可以设置 `retryPolicy`,每个模型可以发布 `contextWindow`;已配置路由通过回放适配器分派,绝不执行提供方 I/O。 |
| `providers` | `ReplayProviderConfig[]` | 无 | 可选的仅回放提供方和模型目录。每个提供方可以设置 `retryPolicy`,每个模型可以发布 `contextWindow` 和声明的 `inputModalities`(让场景能够触发能力门禁,例如 `read_image` 的图像路由检查);已配置路由通过回放适配器分派,绝不执行提供方 I/O。 |
| `paceMs` | number | 无(突发) | 可选的每分片毫秒延迟,使下游传输(例如真实浏览器观察到的 Web SSE(Server-Sent Events)多路复用器)看到真正的增量传递。它只是仿真开关,测试不得依赖它保证正确性。值必须是非负整数;pace 等待期间中止会迅速取消流。 |
```yaml

View File

@@ -19,6 +19,7 @@ import type {
LlmModelInfo,
LlmProviderInfo,
LlmResolvedModelInfo,
ModelModality,
ResolvedRetryPolicy,
RetryPolicyConfig,
StreamChunk,
@@ -51,6 +52,8 @@ export interface ReplayModelConfig {
description?: string
/** Optional positive integer context capacity published by the replay adapter. */
contextWindow?: number
/** Optional declared input modalities, so a scenario can exercise capability gates (e.g. image-capable `read_image`). */
inputModalities?: readonly ModelModality[]
}
/** One provider route exposed by the replay adapter. */
@@ -569,6 +572,7 @@ class ReplayAdapter extends LlmAdapter {
id: model.id,
name: model.name ?? model.id,
...model.description === undefined ? {} : { description: model.description },
...model.inputModalities === undefined ? {} : { inputModalities: [...model.inputModalities] },
})))
}
@@ -582,6 +586,9 @@ class ReplayAdapter extends LlmAdapter {
id: model,
name: configuredModel?.name ?? model,
...configuredModel?.description === undefined ? {} : { description: configuredModel.description },
...configuredModel?.inputModalities === undefined
? {}
: { inputModalities: [...configuredModel.inputModalities] },
...configuredModel?.contextWindow === undefined
? {}
: { context: { contextWindow: configuredModel.contextWindow } },

View File

@@ -592,7 +592,7 @@ describe('installLlmReplay (through the real LlmService)', () => {
backoff: { initialDelayMs: 1, maxDelayMs: 1, jitterRatio: 0 },
},
models: [
{ id: 'flash', contextWindow: 128_000 },
{ id: 'flash', contextWindow: 128_000, inputModalities: ['text', 'image'] },
{ id: 'pro', name: 'Pro', description: 'Larger model' },
],
},
@@ -605,13 +605,15 @@ describe('installLlmReplay (through the real LlmService)', () => {
{ id: 'empty', name: 'empty' },
])
await expect(ctx.llm.listModels('deepseek')).resolves.toEqual([
{ provider: 'deepseek', id: 'flash', name: 'flash' },
{ provider: 'deepseek', id: 'flash', name: 'flash', inputModalities: ['text', 'image'] },
{ provider: 'deepseek', id: 'pro', name: 'Pro', description: 'Larger model' },
])
await expect(ctx.llm.listModels('empty')).resolves.toEqual([])
await expect(ctx.llm.resolveModelInfo('deepseek', 'flash')).resolves.toMatchObject({
context: { contextWindow: 128_000 },
inputModalities: ['text', 'image'],
})
await expect(ctx.llm.resolveModelInfo('deepseek', 'pro')).resolves.not.toHaveProperty('inputModalities')
await expect(ctx.llm.resolveModelInfo('deepseek', 'pro')).resolves.not.toHaveProperty('context')
await expect(ctx.llm.resolveModelInfo('deepseek', 'unlisted')).resolves.not.toHaveProperty('context')
await expect(ctx.llm.resolveModelInfo('empty', 'unlisted')).resolves.not.toHaveProperty('context')