feat(fs): add a minimal read_image tool over the attachment and fs seams
The model reads a PNG/JPEG/WebP/GIF file, the bytes commit through the durable attachment lifecycle, and the tool result carries the real ImageBlock so the image enters context from the next request onward. FileSystem gains a bounded readBytes primitive (local + E2B providers); registration is conditional on the attachment store, and a strict execution gate refuses routes that do not declare image input, so a text route's durable history stays free of image blocks. llm-replay models may declare inputModalities, letting keyless ACP snapshots pin both the sha256-referenced success and the verbatim refusal. Supersedes the withdrawn route-scoped design of PR #598; the decision record is .agents/notes/implemented/feature/2026-08-10-minimal-read-image-tool.md.
This commit is contained in:
@@ -2,5 +2,5 @@
|
||||
# side as of the last confirmed-consistent state. Both languages carry equal authority;
|
||||
# after editing either side, bring the other along and re-record with:
|
||||
# pnpm run verify-translation-pairing --write packages/support/llm-replay/README.md
|
||||
README.md: ae7cb5414a28bcf33f9878f6f02220861df9a0d5
|
||||
README.zh.md: 2acc0ff8e8c011126452ba3458aaeb0800d9d8f3
|
||||
README.md: ade19b66f96ef26119944cbebba23c43e4b2117c
|
||||
README.zh.md: 4752e03b04be175d7f3e37fc57fe9c74e1f8a303
|
||||
|
||||
@@ -29,7 +29,7 @@ Replay keys every call by its calling session id (`GenerateOptions.sessionId`, s
|
||||
| `file` | string | `$DSH_SNAPSHOT_FILE` | Path to the primary (parent) `session.jsonl` fixture. Required (config or env). |
|
||||
| `overrideFile` | string | `$DSH_SNAPSHOT_OVERRIDE` | Optional `ReplayOverrideDoc` sidecar for the primary session: a bare `ReplayEntry[]` replaces its derived script, while `{ patches }` augments it by call index. |
|
||||
| `childFiles` | string[] | `$DSH_SNAPSHOT_CHILD_FILES` (path-delimited) | Recorded subagent child-session logs for a nested scenario; empty for a single-session scenario. |
|
||||
| `providers` | `ReplayProviderConfig[]` | — | Optional replay-only provider and model catalog. Each provider may set `retryPolicy`, and each model may publish `contextWindow`; configured routes dispatch through the replay adapter and never perform provider I/O. |
|
||||
| `providers` | `ReplayProviderConfig[]` | — | Optional replay-only provider and model catalog. Each provider may set `retryPolicy`, and each model may publish `contextWindow` and declared `inputModalities` (so a scenario can exercise capability gates such as the image-capable `read_image` route check); configured routes dispatch through the replay adapter and never perform provider I/O. |
|
||||
| `paceMs` | number | — (burst) | Optional per-chunk delay in ms so downstream transports (e.g. the web SSE mux observed by a real browser) see genuinely incremental delivery. A realism knob only — tests must not depend on it for correctness. Non-negative integer; abort during a pace wait cancels the stream promptly. |
|
||||
|
||||
```yaml
|
||||
|
||||
@@ -29,7 +29,7 @@ fixture 就是持久化的会话日志(`<scenario>/session.jsonl`)。其 `as
|
||||
| `file` | string | `$DSH_SNAPSHOT_FILE` | 主(父)`session.jsonl` fixture 的路径。必需(配置或 env)。 |
|
||||
| `overrideFile` | string | `$DSH_SNAPSHOT_OVERRIDE` | 主会话的可选 `ReplayOverrideDoc` 伴随文件:裸 `ReplayEntry[]` 替换其派生脚本,`{ patches }` 则按调用索引增补该脚本。 |
|
||||
| `childFiles` | string[] | `$DSH_SNAPSHOT_CHILD_FILES`(以路径分隔符分隔) | 嵌套场景中已记录的 subagent 子会话日志;单会话场景为空。 |
|
||||
| `providers` | `ReplayProviderConfig[]` | 无 | 可选的仅回放提供方和模型目录。每个提供方可以设置 `retryPolicy`,每个模型可以发布 `contextWindow`;已配置路由通过回放适配器分派,绝不执行提供方 I/O。 |
|
||||
| `providers` | `ReplayProviderConfig[]` | 无 | 可选的仅回放提供方和模型目录。每个提供方可以设置 `retryPolicy`,每个模型可以发布 `contextWindow` 和声明的 `inputModalities`(让场景能够触发能力门禁,例如 `read_image` 的图像路由检查);已配置路由通过回放适配器分派,绝不执行提供方 I/O。 |
|
||||
| `paceMs` | number | 无(突发) | 可选的每分片毫秒延迟,使下游传输(例如真实浏览器观察到的 Web SSE(Server-Sent Events)多路复用器)看到真正的增量传递。它只是仿真开关,测试不得依赖它保证正确性。值必须是非负整数;pace 等待期间中止会迅速取消流。 |
|
||||
|
||||
```yaml
|
||||
|
||||
@@ -19,6 +19,7 @@ import type {
|
||||
LlmModelInfo,
|
||||
LlmProviderInfo,
|
||||
LlmResolvedModelInfo,
|
||||
ModelModality,
|
||||
ResolvedRetryPolicy,
|
||||
RetryPolicyConfig,
|
||||
StreamChunk,
|
||||
@@ -51,6 +52,8 @@ export interface ReplayModelConfig {
|
||||
description?: string
|
||||
/** Optional positive integer context capacity published by the replay adapter. */
|
||||
contextWindow?: number
|
||||
/** Optional declared input modalities, so a scenario can exercise capability gates (e.g. image-capable `read_image`). */
|
||||
inputModalities?: readonly ModelModality[]
|
||||
}
|
||||
|
||||
/** One provider route exposed by the replay adapter. */
|
||||
@@ -569,6 +572,7 @@ class ReplayAdapter extends LlmAdapter {
|
||||
id: model.id,
|
||||
name: model.name ?? model.id,
|
||||
...model.description === undefined ? {} : { description: model.description },
|
||||
...model.inputModalities === undefined ? {} : { inputModalities: [...model.inputModalities] },
|
||||
})))
|
||||
}
|
||||
|
||||
@@ -582,6 +586,9 @@ class ReplayAdapter extends LlmAdapter {
|
||||
id: model,
|
||||
name: configuredModel?.name ?? model,
|
||||
...configuredModel?.description === undefined ? {} : { description: configuredModel.description },
|
||||
...configuredModel?.inputModalities === undefined
|
||||
? {}
|
||||
: { inputModalities: [...configuredModel.inputModalities] },
|
||||
...configuredModel?.contextWindow === undefined
|
||||
? {}
|
||||
: { context: { contextWindow: configuredModel.contextWindow } },
|
||||
|
||||
@@ -592,7 +592,7 @@ describe('installLlmReplay (through the real LlmService)', () => {
|
||||
backoff: { initialDelayMs: 1, maxDelayMs: 1, jitterRatio: 0 },
|
||||
},
|
||||
models: [
|
||||
{ id: 'flash', contextWindow: 128_000 },
|
||||
{ id: 'flash', contextWindow: 128_000, inputModalities: ['text', 'image'] },
|
||||
{ id: 'pro', name: 'Pro', description: 'Larger model' },
|
||||
],
|
||||
},
|
||||
@@ -605,13 +605,15 @@ describe('installLlmReplay (through the real LlmService)', () => {
|
||||
{ id: 'empty', name: 'empty' },
|
||||
])
|
||||
await expect(ctx.llm.listModels('deepseek')).resolves.toEqual([
|
||||
{ provider: 'deepseek', id: 'flash', name: 'flash' },
|
||||
{ provider: 'deepseek', id: 'flash', name: 'flash', inputModalities: ['text', 'image'] },
|
||||
{ provider: 'deepseek', id: 'pro', name: 'Pro', description: 'Larger model' },
|
||||
])
|
||||
await expect(ctx.llm.listModels('empty')).resolves.toEqual([])
|
||||
await expect(ctx.llm.resolveModelInfo('deepseek', 'flash')).resolves.toMatchObject({
|
||||
context: { contextWindow: 128_000 },
|
||||
inputModalities: ['text', 'image'],
|
||||
})
|
||||
await expect(ctx.llm.resolveModelInfo('deepseek', 'pro')).resolves.not.toHaveProperty('inputModalities')
|
||||
await expect(ctx.llm.resolveModelInfo('deepseek', 'pro')).resolves.not.toHaveProperty('context')
|
||||
await expect(ctx.llm.resolveModelInfo('deepseek', 'unlisted')).resolves.not.toHaveProperty('context')
|
||||
await expect(ctx.llm.resolveModelInfo('empty', 'unlisted')).resolves.not.toHaveProperty('context')
|
||||
|
||||
Reference in New Issue
Block a user