From 109b469a7e52a1a62e9355833001a0257bb7740d Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Mon, 27 Jul 2026 12:41:36 +0800 Subject: [PATCH] fix(tool-web): bound conversion depth and complete fetch output Two review findings on the turndown swap, both verified empirically: - Unclosed-tag nesting makes the synchronous turndown/domino walk superlinear (measured: depth 512 ~0.15s, 2k ~2s, 20k ~5s), during which the cooperative fetchTimeoutMs timer cannot fire. renderBody now preflights nesting depth with a linear tag scan and passes bodies past 512 levels through raw; the try/catch stays for markup the scan cannot see (comment-hidden tags), simulated in tests via a converter throw. - Markdown escaping can expand converted HTML ~2x (100k underscores render as 200k chars), so provider body caps no longer bounded the model-visible result. formatFetchOutput now caps the complete output (header + body + footer) under new fetchMaxOutputChars config (default 200000 = 2x the local provider's default body cap), reusing the truncation notice. README EN+ZH, config catalog, Agent Note EN+ZH updated; the new web-fetch fixture is migrated to the packed layout master now requires; tool-web coverage stays 100% per-file. --- ...ndown-for-tool-web-html-markdown.i18n.yaml | 4 +- ...-26-turndown-for-tool-web-html-markdown.md | 4 +- ...-turndown-for-tool-web-html-markdown.zh.md | 4 +- docs/config-catalog.md | 6 +- .../tests/snapshots/web-fetch/session.jsonl | 102 +----------------- packages/web/tool-web/README.i18n.yaml | 4 +- packages/web/tool-web/README.md | 5 +- packages/web/tool-web/README.zh.md | 5 +- packages/web/tool-web/src/fetch.ts | 86 ++++++++++++--- packages/web/tool-web/src/index.ts | 19 +++- packages/web/tool-web/tests/tool-web.spec.ts | 70 ++++++++++-- 11 files changed, 172 insertions(+), 137 deletions(-) diff --git a/.agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.i18n.yaml b/.agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.i18n.yaml index 60a5d9aca7..a114e32885 100644 --- a/.agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.i18n.yaml +++ b/.agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write -2026-07-26-turndown-for-tool-web-html-markdown.md: c72decc336055f3b78dafdf98f2be3771b833cdb -2026-07-26-turndown-for-tool-web-html-markdown.zh.md: 30667b62538ec50608cae461b5cdf651b48e2731 +2026-07-26-turndown-for-tool-web-html-markdown.md: c7ef4bf538cc949eec8463c8a2ac750685d1a715 +2026-07-26-turndown-for-tool-web-html-markdown.zh.md: 3104dac3cd6db516396b6773f5d2185f3da22ca3 diff --git a/.agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.md b/.agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.md index c72decc336..c7ef4bf538 100644 --- a/.agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.md +++ b/.agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.md @@ -10,7 +10,7 @@ English | [中文](2026-07-26-turndown-for-tool-web-html-markdown.zh.md) ## Decision -`packages/web/tool-web/src/fetch.ts` owns a module-level [`turndown`](https://github.com/mixmark-io/turndown) instance (`headingStyle: 'atx'`, `codeBlockStyle: 'fenced'`, `bulletListMarker: '-'` — fixed model-facing presentation, not deployment tunables) with `@joplin/turndown-plugin-gfm`'s composite `gfm` plugin for tables/strikethrough and `remove(['script', 'style', 'noscript'])` replacing the old wholesale drops. `renderBody`'s `html` arm calls it in a try/catch falling back to the raw HTML body: the regex version could never throw, while turndown/domino's recursive DOM walk overflows with a `RangeError` at a few thousand nesting levels (measured: 4k throws on the main thread, 8k in a worker thread), and a degraded page beats an error for a body the provider already decoded. `html.ts` and its conversion tests are deleted; the fallback and the status-header/truncation-footer formatting are tested in `tests/tool-web.spec.ts`, and the README's Known Limitations trades the regex-converter caveat for the pathological-nesting fallback. The gfm plugin ships no types; `src/turndown-plugin-gfm.d.ts` declares the one imported export over `@types/turndown` (a devDependency). +`packages/web/tool-web/src/fetch.ts` owns a module-level [`turndown`](https://github.com/mixmark-io/turndown) instance (`headingStyle: 'atx'`, `codeBlockStyle: 'fenced'`, `bulletListMarker: '-'` — fixed model-facing presentation, not deployment tunables) with `@joplin/turndown-plugin-gfm`'s composite `gfm` plugin for tables/strikethrough and `remove(['script', 'style', 'noscript'])` replacing the old wholesale drops. `renderBody`'s `html` arm guards the conversion twice: a linear tag-scan preflight passes bodies nested past 512 levels through raw (the synchronous walk is superlinear on unclosed nesting — measured seconds at 20k levels — during which the cooperative timeout cannot fire), and a try/catch falls back to the raw HTML when turndown still throws on markup the scan cannot see; a degraded page beats an error for a body the provider already decoded. `formatFetchOutput` bounds the complete output (`fetchMaxOutputChars` config, default 200,000) because markdown escaping can expand converted HTML to ~2× a provider's body cap. `html.ts` and its conversion tests are deleted; the fallback and the status-header/truncation-footer formatting are tested in `tests/tool-web.spec.ts`, and the README's Known Limitations trades the regex-converter caveat for the pathological-nesting fallback. The gfm plugin ships no types; `src/turndown-plugin-gfm.d.ts` declares the one imported export over `@types/turndown` (a devDependency). The dependency-weight question the proposal flagged resolves in favor of the swap: `@deepseek-ai/dsh-tool-web` is in the single-file-executable closure ([single-exe note](../architecture/2026-07-10-single-file-executable-sdk-runtime-distribution.md)), and the exe's asset globs would pack ~7.9 MB of the three packages as published — but ~6 MB of that is `@mixmark-io/domino`'s test corpus (`test/**`), with runtime `lib/` at ~550 KB against a ~174 MB artifact, under 0.5% either way. @@ -33,5 +33,5 @@ The previously-missing keyless `web_fetch` snapshot ships with the change as the ## Testing -- `packages/web/tool-web/tests/tool-web.spec.ts` covers the turndown conversion surface (entities, links, tables, nesting, script/style/noscript removal) through `renderBody`, and the raw-HTML fallback with a measured reliably-overflowing 20k-level nesting input; per-file coverage on the package src is 100%. +- `packages/web/tool-web/tests/tool-web.spec.ts` covers the turndown conversion surface (entities, links, tables, nesting, script/style/noscript removal) through `renderBody`, the fast raw-HTML passthrough for 20k-level nesting, the depth scan's void/self-closing/unbalanced cases, the residual converter-throw fallback, and the whole-output cap at expanding, exact, and tiny budgets; per-file coverage on the package src is 100%. - The `web-fetch` acp-agent snapshot pins the assembled behavior keylessly end to end (real Loader composition, real HTTP fetch, real conversion). diff --git a/.agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.zh.md b/.agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.zh.md index 30667b6253..3104dac3cd 100644 --- a/.agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.zh.md +++ b/.agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.zh.md @@ -10,7 +10,7 @@ Status: implemented ## 决策 -`packages/web/tool-web/src/fetch.ts` 持有一个模块级 [`turndown`](https://github.com/mixmark-io/turndown) 实例（`headingStyle: 'atx'`、`codeBlockStyle: 'fenced'`、`bulletListMarker: '-'`——固定的面向模型呈现方式，不是部署可调项），配合 `@joplin/turndown-plugin-gfm` 的组合 `gfm` 插件提供表格／删除线支持，并用 `remove(['script', 'style', 'noscript'])` 替代旧实现的整体剥离。`renderBody` 的 `html` 分支把调用包在 try/catch 中，失败时回退为原始 HTML 主体：正则版本从不可能抛异常，而 turndown/domino 的递归 DOM 遍历在数千层嵌套（实测：主线程 4k 层抛出，worker 线程 8k 层抛出）会以 `RangeError` 栈溢出，对提供方已经解码的主体来说，降级页面好过报错。`html.ts` 及其转换测试已删除；回退路径与状态头、截断页脚的格式化在 `tests/tool-web.spec.ts` 中有测试覆盖，README 的 Known Limitations 用病态嵌套回退条目替换了正则转换器的警示说明。gfm 插件不带类型声明；`src/turndown-plugin-gfm.d.ts` 基于 `@types/turndown`（devDependency）声明了唯一被导入的导出。 +`packages/web/tool-web/src/fetch.ts` 持有一个模块级 [`turndown`](https://github.com/mixmark-io/turndown) 实例（`headingStyle: 'atx'`、`codeBlockStyle: 'fenced'`、`bulletListMarker: '-'`——固定的面向模型呈现方式，不是部署可调项），配合 `@joplin/turndown-plugin-gfm` 的组合 `gfm` 插件提供表格／删除线支持，并用 `remove(['script', 'style', 'noscript'])` 替代旧实现的整体剥离。`renderBody` 的 `html` 分支对转换做了双重防护：一次线性标签扫描预检把嵌套超过 512 层的主体直接原样透传（同步遍历在未闭合嵌套上呈超线性——实测 2 万层需要数秒——期间协作式超时无法触发），扫描看不到的标记若仍让 turndown 抛异常，则由 try/catch 回退为原始 HTML；对提供方已经解码的主体来说，降级页面好过报错。`formatFetchOutput` 对完整输出设上限（`fetchMaxOutputChars` 配置，默认 200,000）：markdown 转义可能把转换后的 HTML 膨胀到提供方主体上限的约 2 倍。`html.ts` 及其转换测试已删除；透传、回退与整体输出上限，连同状态头、截断页脚的格式化，都在 `tests/tool-web.spec.ts` 中有测试覆盖，README 的 Known Limitations 用病态嵌套回退条目替换了正则转换器的警示说明。gfm 插件不带类型声明；`src/turndown-plugin-gfm.d.ts` 基于 `@types/turndown`（devDependency）声明了唯一被导入的导出。 提案标记的依赖体积问题的裁决结果支持替换：`@deepseek-ai/dsh-tool-web` 在单文件可执行文件闭包内（[single-exe 决策记录](../architecture/2026-07-10-single-file-executable-sdk-runtime-distribution.md)），可执行文件的资产 glob 会把这三个包按发布原样打入约 7.9 MB——但其中约 6 MB 是 `@mixmark-io/domino` 的测试语料（`test/**`），运行时 `lib/` 仅约 550 KB，相对约 174 MB 的产物，两种口径都不到 0.5%。 @@ -33,5 +33,5 @@ Status: implemented ## 测试 -- `packages/web/tool-web/tests/tool-web.spec.ts` 通过 `renderBody` 覆盖 turndown 转换面（实体、链接、表格、嵌套、script/style/noscript 移除），并用实测可稳定溢出的 2 万层嵌套输入覆盖原始 HTML 回退；该包 src 的逐文件覆盖率为 100%。 +- `packages/web/tool-web/tests/tool-web.spec.ts` 通过 `renderBody` 覆盖 turndown 转换面（实体、链接、表格、嵌套、script/style/noscript 移除）、2 万层嵌套的快速原样透传、深度扫描的空元素／自闭合／不平衡用例、残余的转换器抛错回退，以及在膨胀、恰好、极小预算下的整体输出上限；该包 src 的逐文件覆盖率为 100%。 - acp-agent 的 `web-fetch` 快照无密钥地端到端固定组装后的行为（真实 Loader 组合、真实 HTTP 抓取、真实转换）。 diff --git a/docs/config-catalog.md b/docs/config-catalog.md index 721c432f73..742d04dd71 100644 --- a/docs/config-catalog.md +++ b/docs/config-catalog.md @@ -1656,7 +1656,7 @@ Source: [`packages/tasks/tool-tasks/src/index.ts:23`](../packages/tasks/tool-tas Requires: `tools` · `web` · `systemPrompt` ```ts config-catalog -/** Plugin config: which web tools to register, the source cap, and per-tool budgets. */ +/** Plugin config: which web tools to register, the source cap, per-tool budgets, and the fetch output cap. */ export interface Config { /** Register `web_search`. Defaults to true. */ search?: boolean @@ -1668,10 +1668,12 @@ export interface Config { fetchTimeoutMs?: number /** Cooperative timeout budget (ms) for `web_search`. Defaults to 30000. */ searchTimeoutMs?: number + /** Cap on one `web_fetch` output's characters (header, rendered body, and footer). Defaults to 200000. */ + fetchMaxOutputChars?: number } ``` -Source: [`packages/web/tool-web/src/index.ts:28`](../packages/web/tool-web/src/index.ts) +Source: [`packages/web/tool-web/src/index.ts:37`](../packages/web/tool-web/src/index.ts) ## `@deepseek-ai/dsh-tool-workflow` diff --git a/examples/acp-agent/tests/snapshots/web-fetch/session.jsonl b/examples/acp-agent/tests/snapshots/web-fetch/session.jsonl index c6c34bc8e3..47c97b1cba 100644 --- a/examples/acp-agent/tests/snapshots/web-fetch/session.jsonl +++ b/examples/acp-agent/tests/snapshots/web-fetch/session.jsonl @@ -5,75 +5,9 @@ {"type":"step/start","seq":3,"time":1785078727730,"data":{"turn":1,"step":1}} {"type":"request/header","seq":4,"time":1785078727731,"data":{"header":{"config":{"provider":"deepseek","model":"deepseek-v4-pro"},"system":"{{system}}","tools":"{{tools}}"},"reason":"initial"}} {"type":"assistant/chunk","seq":5,"time":1785078728804,"data":{"turn":1,"step":1,"chunk":{"type":"block-start","index":0,"blockType":"reasoning"}}} -{"type":"assistant/chunk","seq":6,"time":1785078728805,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"The"}}} -{"type":"assistant/chunk","seq":7,"time":1785078728943,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" user"}}} -{"type":"assistant/chunk","seq":8,"time":1785078728989,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" wants"}}} -{"type":"assistant/chunk","seq":9,"time":1785078728989,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" me"}}} -{"type":"assistant/chunk","seq":10,"time":1785078728989,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" to"}}} -{"type":"assistant/chunk","seq":11,"time":1785078728990,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" use"}}} -{"type":"assistant/chunk","seq":12,"time":1785078728990,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" the"}}} -{"type":"assistant/chunk","seq":13,"time":1785078728990,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" web"}}} -{"type":"assistant/chunk","seq":14,"time":1785078729038,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"_f"}}} -{"type":"assistant/chunk","seq":15,"time":1785078729038,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"etch"}}} -{"type":"assistant/chunk","seq":16,"time":1785078729039,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" tool"}}} -{"type":"assistant/chunk","seq":17,"time":1785078729039,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" exactly"}}} -{"type":"assistant/chunk","seq":18,"time":1785078729085,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" once"}}} -{"type":"assistant/chunk","seq":19,"time":1785078729086,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" to"}}} -{"type":"assistant/chunk","seq":20,"time":1785078729086,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" fetch"}}} -{"type":"assistant/chunk","seq":21,"time":1785078729086,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" http"}}} -{"type":"assistant/chunk","seq":22,"time":1785078729086,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"://"}}} -{"type":"assistant/chunk","seq":23,"time":1785078729086,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"127"}}} -{"type":"assistant/chunk","seq":24,"time":1785078729132,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"."}}} -{"type":"assistant/chunk","seq":25,"time":1785078729133,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"0"}}} -{"type":"assistant/chunk","seq":26,"time":1785078729133,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"."}}} -{"type":"assistant/chunk","seq":27,"time":1785078729133,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"0"}}} -{"type":"assistant/chunk","seq":28,"time":1785078729133,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"."}}} -{"type":"assistant/chunk","seq":29,"time":1785078729133,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"1"}}} -{"type":"assistant/chunk","seq":30,"time":1785078729182,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":":"}}} -{"type":"assistant/chunk","seq":31,"time":1785078729182,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"431"}}} -{"type":"assistant/chunk","seq":32,"time":1785078729183,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"17"}}} -{"type":"assistant/chunk","seq":33,"time":1785078729183,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"/m"}}} -{"type":"assistant/chunk","seq":34,"time":1785078729183,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"enu"}}} -{"type":"assistant/chunk","seq":35,"time":1785078729183,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":".html"}}} -{"type":"assistant/chunk","seq":36,"time":1785078729230,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":","}}} -{"type":"assistant/chunk","seq":37,"time":1785078729230,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" then"}}} -{"type":"assistant/chunk","seq":38,"time":1785078729230,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" reply"}}} -{"type":"assistant/chunk","seq":39,"time":1785078729230,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" with"}}} -{"type":"assistant/chunk","seq":40,"time":1785078729230,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" exactly"}}} -{"type":"assistant/chunk","seq":41,"time":1785078729231,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" \""}}} -{"type":"assistant/chunk","seq":42,"time":1785078729276,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"D"}}} -{"type":"assistant/chunk","seq":43,"time":1785078729277,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"ONE"}}} -{"type":"assistant/chunk","seq":44,"time":1785078729277,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"\"."}}} -{"type":"assistant/chunk","seq":45,"time":1785078729277,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" Let"}}} -{"type":"assistant/chunk","seq":46,"time":1785078729322,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" me"}}} -{"type":"assistant/chunk","seq":47,"time":1785078729323,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" do"}}} -{"type":"assistant/chunk","seq":48,"time":1785078729323,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" that"}}} -{"type":"assistant/chunk","seq":49,"time":1785078729323,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"."}}} +{"type":"reasoning-chunks","seq0":6,"time0":1785078728805,"data":{"turn":1,"step":1,"index":0,"dt":[138,46,0,0,1,0,0,48,0,1,0,46,1,0,0,0,0,46,1,0,0,0,0,49,0,1,0,0,0,47,0,0,0,0,1,45,1,0,0,45,1,0,0],"texts":["The"," user"," wants"," me"," to"," use"," the"," web","_f","etch"," tool"," exactly"," once"," to"," fetch"," http","://","127",".","0",".","0",".","1",":","431","17","/m","enu",".html",","," then"," reply"," with"," exactly"," \"","D","ONE","\"."," Let"," me"," do"," that","."]}} {"type":"assistant/chunk","seq":50,"time":1785078729463,"data":{"turn":1,"step":1,"chunk":{"type":"block-start","index":1,"blockType":"tool-call"}}} -{"type":"assistant/chunk","seq":51,"time":1785078729464,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":""}}} -{"type":"assistant/chunk","seq":52,"time":1785078729511,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":"{"}}} -{"type":"assistant/chunk","seq":53,"time":1785078729511,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":"\""}}} -{"type":"assistant/chunk","seq":54,"time":1785078729511,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":"url"}}} -{"type":"assistant/chunk","seq":55,"time":1785078729511,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":"\""}}} -{"type":"assistant/chunk","seq":56,"time":1785078729511,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":": "}}} -{"type":"assistant/chunk","seq":57,"time":1785078729557,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":"\""}}} -{"type":"assistant/chunk","seq":58,"time":1785078729557,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":"http"}}} -{"type":"assistant/chunk","seq":59,"time":1785078729557,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":"://"}}} -{"type":"assistant/chunk","seq":60,"time":1785078729558,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":"127"}}} -{"type":"assistant/chunk","seq":61,"time":1785078729604,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":"."}}} -{"type":"assistant/chunk","seq":62,"time":1785078729604,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":"0"}}} -{"type":"assistant/chunk","seq":63,"time":1785078729604,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":"."}}} -{"type":"assistant/chunk","seq":64,"time":1785078729604,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":"0"}}} -{"type":"assistant/chunk","seq":65,"time":1785078729604,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":"."}}} -{"type":"assistant/chunk","seq":66,"time":1785078729605,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":"1"}}} -{"type":"assistant/chunk","seq":67,"time":1785078729651,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":":"}}} -{"type":"assistant/chunk","seq":68,"time":1785078729652,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":"431"}}} -{"type":"assistant/chunk","seq":69,"time":1785078729652,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":"17"}}} -{"type":"assistant/chunk","seq":70,"time":1785078729652,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":"/m"}}} -{"type":"assistant/chunk","seq":71,"time":1785078729652,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":"enu"}}} -{"type":"assistant/chunk","seq":72,"time":1785078729652,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":".html"}}} -{"type":"assistant/chunk","seq":73,"time":1785078729697,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":"\""}}} -{"type":"assistant/chunk","seq":74,"time":1785078729698,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":"}"}}} +{"type":"tool-call-chunks","seq0":51,"time0":1785078729464,"data":{"turn":1,"step":1,"index":1,"dt":[47,0,0,0,0,46,0,0,1,46,0,0,0,0,1,46,1,0,0,0,0,45,1],"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","args":["","{","\"","url","\"",": ","\"","http","://","127",".","0",".","0",".","1",":","431","17","/m","enu",".html","\"","}"]}} {"type":"assistant/chunk","seq":75,"time":1785078729803,"data":{"turn":1,"step":1,"chunk":{"type":"block-end","index":0,"block":{"type":"reasoning","text":"The user wants me to use the web_fetch tool exactly once to fetch http://127.0.0.1:43117/menu.html, then reply with exactly \"DONE\". Let me do that."}}}} {"type":"assistant/chunk","seq":76,"time":1785078729803,"data":{"turn":1,"step":1,"chunk":{"type":"block-end","index":1,"block":{"type":"tool-call","id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","arguments":"{\"url\": \"http://127.0.0.1:43117/menu.html\"}"}}}} {"type":"assistant/chunk","seq":77,"time":1785078729803,"data":{"turn":1,"step":1,"chunk":{"type":"usage","usage":{"inputTokens":5405,"outputTokens":103,"cacheReadTokens":0,"reasoningTokens":44}}}} @@ -84,37 +18,7 @@ {"type":"step/end","seq":82,"time":1785078729847,"data":{"turn":1,"step":1}} {"type":"step/start","seq":83,"time":1785078729848,"data":{"turn":1,"step":2}} {"type":"assistant/chunk","seq":84,"time":1785078730611,"data":{"turn":1,"step":2,"chunk":{"type":"block-start","index":0,"blockType":"reasoning"}}} -{"type":"assistant/chunk","seq":85,"time":1785078730612,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":"The"}}} -{"type":"assistant/chunk","seq":86,"time":1785078730770,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":" user"}}} -{"type":"assistant/chunk","seq":87,"time":1785078730824,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":" asked"}}} -{"type":"assistant/chunk","seq":88,"time":1785078730825,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":" me"}}} -{"type":"assistant/chunk","seq":89,"time":1785078730825,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":" to"}}} -{"type":"assistant/chunk","seq":90,"time":1785078730825,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":" fetch"}}} -{"type":"assistant/chunk","seq":91,"time":1785078730861,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":" the"}}} -{"type":"assistant/chunk","seq":92,"time":1785078730862,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":" URL"}}} -{"type":"assistant/chunk","seq":93,"time":1785078730909,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":","}}} -{"type":"assistant/chunk","seq":94,"time":1785078730956,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":" then"}}} -{"type":"assistant/chunk","seq":95,"time":1785078731002,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":" reply"}}} -{"type":"assistant/chunk","seq":96,"time":1785078731003,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":" with"}}} -{"type":"assistant/chunk","seq":97,"time":1785078731003,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":" exactly"}}} -{"type":"assistant/chunk","seq":98,"time":1785078731003,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":" \""}}} -{"type":"assistant/chunk","seq":99,"time":1785078731050,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":"D"}}} -{"type":"assistant/chunk","seq":100,"time":1785078731050,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":"ONE"}}} -{"type":"assistant/chunk","seq":101,"time":1785078731050,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":"\"."}}} -{"type":"assistant/chunk","seq":102,"time":1785078731050,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":" I"}}} -{"type":"assistant/chunk","seq":103,"time":1785078731050,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":"'ve"}}} -{"type":"assistant/chunk","seq":104,"time":1785078731051,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":" fetched"}}} -{"type":"assistant/chunk","seq":105,"time":1785078731097,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":" it"}}} -{"type":"assistant/chunk","seq":106,"time":1785078731140,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":"."}}} -{"type":"assistant/chunk","seq":107,"time":1785078731141,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":" Now"}}} -{"type":"assistant/chunk","seq":108,"time":1785078731141,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":" I"}}} -{"type":"assistant/chunk","seq":109,"time":1785078731141,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":" just"}}} -{"type":"assistant/chunk","seq":110,"time":1785078731189,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":" reply"}}} -{"type":"assistant/chunk","seq":111,"time":1785078731189,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":" with"}}} -{"type":"assistant/chunk","seq":112,"time":1785078731235,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":" \""}}} -{"type":"assistant/chunk","seq":113,"time":1785078731235,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":"D"}}} -{"type":"assistant/chunk","seq":114,"time":1785078731235,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":"ONE"}}} -{"type":"assistant/chunk","seq":115,"time":1785078731235,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":"\"."}}} +{"type":"reasoning-chunks","seq0":85,"time0":1785078730612,"data":{"turn":1,"step":2,"index":0,"dt":[158,54,1,0,0,36,1,47,47,46,1,0,0,47,0,0,0,0,1,46,43,1,0,0,48,0,46,0,0,0],"texts":["The"," user"," asked"," me"," to"," fetch"," the"," URL",","," then"," reply"," with"," exactly"," \"","D","ONE","\"."," I","'ve"," fetched"," it","."," Now"," I"," just"," reply"," with"," \"","D","ONE","\"."]}} {"type":"assistant/chunk","seq":116,"time":1785078731235,"data":{"turn":1,"step":2,"chunk":{"type":"block-start","index":1,"blockType":"text"}}} {"type":"assistant/chunk","seq":117,"time":1785078731236,"data":{"turn":1,"step":2,"chunk":{"type":"text-delta","index":1,"text":"D"}}} {"type":"assistant/chunk","seq":118,"time":1785078731282,"data":{"turn":1,"step":2,"chunk":{"type":"text-delta","index":1,"text":"ONE"}}} diff --git a/packages/web/tool-web/README.i18n.yaml b/packages/web/tool-web/README.i18n.yaml index 1e746ed566..44279a66be 100644 --- a/packages/web/tool-web/README.i18n.yaml +++ b/packages/web/tool-web/README.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write -README.md: 5fe48ced81a2cd02197cf8cc10a7d6567b17ffca -README.zh.md: 34ad08e290166ee6db2cd7b836746541d18aad52 +README.md: 44cb1ba2a2f4e1fba7e192d8b6645e0447ebf221 +README.zh.md: 35b390dd5407af16d84ab391dd8351f784c60035 diff --git a/packages/web/tool-web/README.md b/packages/web/tool-web/README.md index 5fe48ced81..44cb1ba2a2 100644 --- a/packages/web/tool-web/README.md +++ b/packages/web/tool-web/README.md @@ -26,8 +26,9 @@ The normalized seam results are also the canonical tool values: `WebSearchResult | `searchMaxResults` | `8` | Upper bound on sources returned by one `web_search` call (the seam truncates a longer provider list and flags it). | | `fetchTimeoutMs` | `30000` | Cooperative tool-call timeout budget (ms) for `web_fetch`. | | `searchTimeoutMs` | `30000` | Cooperative tool-call timeout budget (ms) for `web_search`. | +| `fetchMaxOutputChars` | `200000` | Cap on one `web_fetch` output's characters — header, rendered body, and footer together; a cut body gets the truncation notice. | -`fetchTimeoutMs`/`searchTimeoutMs` declare each tool's cooperative timeout budget (attached as `ToolDefinition.timeoutMs`), enforced by [`@deepseek-ai/dsh-timeout-policy`](../../timeout/timeout-policy/README.md); the model-facing schema exposes no timeout argument. +`fetchTimeoutMs`/`searchTimeoutMs` declare each tool's cooperative timeout budget (attached as `ToolDefinition.timeoutMs`), enforced by [`@deepseek-ai/dsh-timeout-policy`](../../timeout/timeout-policy/README.md); the model-facing schema exposes no timeout argument. `fetchMaxOutputChars` bounds the complete rendered output because markdown escaping can expand converted HTML past a provider's body cap (worst case ~2×); the default is 2× the local provider's default 100,000-character body cap, so it never cuts what that bound already admits. ```yaml - id: tool-web @@ -126,6 +127,6 @@ Append-only; newly visible content follows the reusable request prefix and does ## Known Limitations and Deferred Work -- **HTML→markdown conversion falls back to raw HTML on pathological input** — [turndown](https://github.com/mixmark-io/turndown) (with GFM tables/strikethrough) converts fetched HTML through a real DOM, but its recursive walk overflows on absurdly deep nesting (thousands of levels); such a body passes through unconverted rather than erroring ([Agent Note](../../../.agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.md)). +- **HTML→markdown conversion falls back to raw HTML on pathological input** — [turndown](https://github.com/mixmark-io/turndown) (with GFM tables/strikethrough) converts fetched HTML through a real DOM, but the synchronous walk is superlinear on deep unclosed nesting, so bodies nested past a fixed 512-level preflight bound pass through unconverted (as does anything that still makes turndown throw) rather than stalling the event loop or erroring ([Agent Note](../../../.agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.md)). - **The model-facing surface is minimal by design, with promotions deferred** — `max_results` stays a config bound (not a model argument), and `web_fetch` takes only `url` (no `format`/`prompt`/LLM-summarization mode); both are named later steps in [the seam Agent Note](../../../.agents/notes/implemented/architecture/2026-06-24-web-capability-seam.md). - **No web-specific permission policy** — both tools execute without requesting `ctx.approval`; a deployment that needs confirmation must add a `tools/pre-execute` policy, and the package does not define persistent URL/domain grants. diff --git a/packages/web/tool-web/README.zh.md b/packages/web/tool-web/README.zh.md index 34ad08e290..35b390dd54 100644 --- a/packages/web/tool-web/README.zh.md +++ b/packages/web/tool-web/README.zh.md @@ -26,8 +26,9 @@ | `searchMaxResults` | `8` | 一次 `web_search` 调用返回的源数量上限（seam 截断更长的提供方列表并标记）。 | | `fetchTimeoutMs` | `30000` | `web_fetch` 的协作式工具调用超时预算（ms）。 | | `searchTimeoutMs` | `30000` | `web_search` 的协作式工具调用超时预算（ms）。 | +| `fetchMaxOutputChars` | `200000` | 单次 `web_fetch` 输出的字符上限——状态头、渲染后的主体与页脚合并计算；被截断的主体带截断提示。 | -`fetchTimeoutMs`／`searchTimeoutMs` 声明每个工具的协作式超时预算（附加为 `ToolDefinition.timeoutMs`），由 [`@deepseek-ai/dsh-timeout-policy`](../../timeout/timeout-policy/README.md) 强制执行；面向模型的 schema 不公开超时参数。 +`fetchTimeoutMs`／`searchTimeoutMs` 声明每个工具的协作式超时预算（附加为 `ToolDefinition.timeoutMs`），由 [`@deepseek-ai/dsh-timeout-policy`](../../timeout/timeout-policy/README.md) 强制执行；面向模型的 schema 不公开超时参数。`fetchMaxOutputChars` 对完整渲染输出设上限：markdown 转义可能让转换后的 HTML 超出提供方的主体上限（最坏约 2 倍）；默认值取本地提供方默认 100,000 字符主体上限的 2 倍，因此绝不会削减该上限本已允许的内容。 ```yaml - id: tool-web @@ -126,6 +127,6 @@ Use the web_fetch tool to retrieve the content of a specific HTTP(S) URL (for ex ## 已知限制与暂缓事项 -- **HTML→markdown 转换在病态输入上回退为原始 HTML**：[turndown](https://github.com/mixmark-io/turndown)（带 GFM 表格／删除线）通过真实 DOM 转换抓取到的 HTML，但其递归遍历在极深嵌套（数千层）上会栈溢出；此类主体不经转换原样通过，而非报错（[决策记录](../../../.agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.md)）。 +- **HTML→markdown 转换在病态输入上回退为原始 HTML**：[turndown](https://github.com/mixmark-io/turndown)（带 GFM 表格／删除线）通过真实 DOM 转换抓取到的 HTML，但同步遍历在深层未闭合嵌套上呈超线性，因此嵌套超过固定 512 层预检上限的主体不经转换原样通过（仍让 turndown 抛异常的输入同样如此），而非阻塞事件循环或报错（[决策记录](../../../.agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.md)）。 - **面向模型的表层有意保持最小，提升项暂缓**：`max_results` 保持为配置上限（不是模型参数），`web_fetch` 只接受 `url`（没有 `format`／`prompt`／LLM 摘要模式）；两项都列为 [seam Agent Note](../../../.agents/notes/implemented/architecture/2026-06-24-web-capability-seam.md) 中的后续步骤。 - **没有 web 专用权限策略**：两个工具都不会请求 `ctx.approval` 就直接执行；需要确认的部署必须添加 `tools/pre-execute` 策略，该包不定义持久 URL／domain 授权。 diff --git a/packages/web/tool-web/src/fetch.ts b/packages/web/tool-web/src/fetch.ts index 60c0f33507..e909ea25be 100644 --- a/packages/web/tool-web/src/fetch.ts +++ b/packages/web/tool-web/src/fetch.ts @@ -44,23 +44,69 @@ export function parseFetchArgs(args: { url: string }): { url: string } { return { url: args.url } } +/** + * Nesting-depth ceiling above which HTML skips conversion and passes through + * raw. Conversion runs synchronously on the event loop, and unclosed-tag + * nesting makes domino's tree (and turndown's walk over it) superlinear — + * measured: depth 512 ≈ 0.15s, 2,000 ≈ 2s, 20,000 ≈ 5s — during which the + * cooperative `fetchTimeoutMs` timer cannot fire. Real pages nest a few dozen + * levels; 512 is far above content and far below weaponizable. A robustness + * invariant, not a tunable. + */ +const MAX_CONVERSION_DEPTH = 512 + +/** Elements that never take a closing tag, so they must not count toward nesting depth. */ +const VOID_ELEMENTS = new Set([ + 'area', 'base', 'br', 'col', 'embed', 'hr', 'img', 'input', + 'link', 'meta', 'param', 'source', 'track', 'wbr', +]) + +/** + * Estimate the maximum element nesting depth of an HTML string with one linear + * tag scan. Overestimates when markup-like text sits inside `script`/`style` + * bodies or comments (the scan does not parse those), which can only cause a + * spurious raw-HTML fallback, never a missed bound. + * + * @param html - the decoded HTML body. + * @returns the deepest open-element count the scan reaches. + */ +export function htmlNestingDepth(html: string): number { + let depth = 0 + let max = 0 + for (const tag of html.matchAll(/<(\/?)([a-zA-Z][a-zA-Z0-9-]*)[^>]*?(\/?)>/g)) { + const [, closing, rawName = '', selfClosing] = tag + const name = rawName.toLowerCase() + if (VOID_ELEMENTS.has(name) || selfClosing === '/') continue + if (closing === '/') { + if (depth > 0) depth -= 1 + } else { + depth += 1 + if (depth > max) max = depth + } + } + return max +} + /** * Render a fetched body to model-facing markdown text. * * @param body - the decoded body; `html` is converted via turndown, `text` - * passes through verbatim. When turndown throws (deeply pathological HTML - * overflows its recursive DOM walk), the raw HTML passes through instead — - * a degraded page beats an error for a body the provider already decoded. + * passes through verbatim. HTML nested beyond {@link MAX_CONVERSION_DEPTH} + * skips conversion up front (the synchronous walk over such trees is + * superlinear and blocks the event loop past the cooperative timeout), and + * when turndown itself throws the raw HTML passes through instead — a + * degraded page beats an error for a body the provider already decoded. * @returns the text for the tool's output block. */ export function renderBody(body: WebFetchBody): string { switch (body.kind) { case 'html': + if (htmlNestingDepth(body.content) > MAX_CONVERSION_DEPTH) return body.content try { return turndown.turndown(body.content) } catch { - // turndown's DOM walk recurses per element; pathological nesting (a - // few thousand levels) throws RangeError. Provider errors stay + // turndown's DOM walk recurses per element; malformed markup the depth + // scan cannot see can still throw RangeError. Provider errors stay // structured WebErrors upstream; conversion failure downgrades to raw HTML. return body.content } @@ -72,17 +118,28 @@ export function renderBody(body: WebFetchBody): string { } } +/** The truncation notice appended when the provider or the output cap cut content. */ +const TRUNCATION_FOOTER = '\n\n(Content truncated. Fetch a more specific URL or section for the full text.)' + /** - * Format a fetch result as one model-facing text block. + * Format a fetch result as one model-facing text block, bounded as a whole. + * Markdown escaping can expand converted HTML (worst case ~2× the provider's + * body cap), so the bound applies here, where the complete output — header, + * rendered body, and footer — is known. * * @param result - the seam's fetch outcome. + * @param maxOutputChars - cap on the complete returned string; a cut body gets + * the same fetch-something-narrower notice as provider-side truncation. * @returns a `Fetched (HTTP )` header, the rendered body, and a - * fetch-something-narrower notice when the provider truncated the content. + * truncation notice when the provider or the cap cut the content. */ -export function formatFetchOutput(result: WebFetchResult): string { - const header = `Fetched ${result.url} (HTTP ${result.statusCode})` - const footer = result.truncated ? '\n\n(Content truncated. Fetch a more specific URL or section for the full text.)' : '' - return `${header}\n\n${renderBody(result.body)}${footer}` +export function formatFetchOutput(result: WebFetchResult, maxOutputChars: number): string { + const header = `Fetched ${result.url} (HTTP ${result.statusCode})\n\n` + const body = renderBody(result.body) + const full = `${header}${body}${result.truncated ? TRUNCATION_FOOTER : ''}` + if (full.length <= maxOutputChars) return full + const budget = Math.max(0, maxOutputChars - header.length - TRUNCATION_FOOTER.length) + return `${header}${body.slice(0, budget)}${TRUNCATION_FOOTER}` } /** @@ -102,8 +159,11 @@ export function presentFetchCall(args: { url: string }): GenericCallView { * registrations; both are effect-scoped and unregister on plugin dispose. * @param timeoutMs - the cooperative tool-call budget (ms) attached as the tool's * `ToolDefinition.timeoutMs` for `@deepseek-ai/dsh-timeout-policy` to enforce. + * @param maxOutputChars - cap on the complete rendered tool output (see + * {@link formatFetchOutput}); markdown escaping can outgrow the provider's + * body cap, so the model-context bound is enforced on the rendered result. */ -export function applyWebFetchTool(ctx: Context, timeoutMs: number): void { +export function applyWebFetchTool(ctx: Context, timeoutMs: number, maxOutputChars: number): void { ctx.systemPrompt.section({ name: 'tool:web_fetch', order: 111, @@ -147,7 +207,7 @@ export function applyWebFetchTool(ctx: Context, timeoutMs: number): void { truncated: { type: 'boolean', required: true }, }, }, - render: (_args, value) => [{ type: 'text', text: formatFetchOutput(value) }], + render: (_args, value) => [{ type: 'text', text: formatFetchOutput(value, maxOutputChars) }], }, timeoutMs, // Provider reads do not mutate parent-agent state. diff --git a/packages/web/tool-web/src/index.ts b/packages/web/tool-web/src/index.ts index e7ac4b2453..4a0ea5202c 100644 --- a/packages/web/tool-web/src/index.ts +++ b/packages/web/tool-web/src/index.ts @@ -13,7 +13,7 @@ import { applyWebSearchTool, WEB_SEARCH_MAX_RESULTS } from './search.ts' import { applyWebFetchTool } from './fetch.ts' export { WEB_SEARCH_MAX_RESULTS, applyWebSearchTool, formatSearchOutput, parseSearchArgs, presentSearchCall } from './search.ts' -export { applyWebFetchTool, formatFetchOutput, parseFetchArgs, presentFetchCall, renderBody } from './fetch.ts' +export { applyWebFetchTool, formatFetchOutput, htmlNestingDepth, parseFetchArgs, presentFetchCall, renderBody } from './fetch.ts' /** Cordis plugin name used by loader diagnostics. */ export const name = 'tool-web' @@ -24,7 +24,16 @@ export const inject = ['tools', 'web', 'systemPrompt'] /** Default cooperative tool-call timeout budget (ms) for the web tools. */ export const DEFAULT_WEB_TOOL_TIMEOUT_MS = 30_000 -/** Plugin config: which web tools to register, the source cap, and per-tool budgets. */ +/** + * Default cap on one `web_fetch` output's characters. Markdown escaping can + * roughly double converted HTML, so this sits at 2× the local provider's + * default 100,000-char body cap: it never cuts what that composition's + * provider bound already admits, while restoring a model-context bound for + * providers with larger or absent body caps. + */ +export const DEFAULT_FETCH_MAX_OUTPUT_CHARS = 200_000 + +/** Plugin config: which web tools to register, the source cap, per-tool budgets, and the fetch output cap. */ export interface Config { /** Register `web_search`. Defaults to true. */ search?: boolean @@ -36,6 +45,8 @@ export interface Config { fetchTimeoutMs?: number /** Cooperative timeout budget (ms) for `web_search`. Defaults to 30000. */ searchTimeoutMs?: number + /** Cap on one `web_fetch` output's characters (header, rendered body, and footer). Defaults to 200000. */ + fetchMaxOutputChars?: number } export const Config: z = z.object({ @@ -44,6 +55,7 @@ export const Config: z = z.object({ searchMaxResults: z.number().default(WEB_SEARCH_MAX_RESULTS), fetchTimeoutMs: z.number().default(DEFAULT_WEB_TOOL_TIMEOUT_MS), searchTimeoutMs: z.number().default(DEFAULT_WEB_TOOL_TIMEOUT_MS), + fetchMaxOutputChars: z.number().default(DEFAULT_FETCH_MAX_OUTPUT_CHARS), }) /** The shape after schemastery applies its defaults to every field. */ @@ -71,6 +83,7 @@ export function apply(ctx: Context, config: Config): void { assertPositiveInteger('searchMaxResults', resolved.searchMaxResults) assertPositiveInteger('fetchTimeoutMs', resolved.fetchTimeoutMs) assertPositiveInteger('searchTimeoutMs', resolved.searchTimeoutMs) + assertPositiveInteger('fetchMaxOutputChars', resolved.fetchMaxOutputChars) if (resolved.search) applyWebSearchTool(ctx, resolved.searchMaxResults, resolved.searchTimeoutMs) - if (resolved.fetch) applyWebFetchTool(ctx, resolved.fetchTimeoutMs) + if (resolved.fetch) applyWebFetchTool(ctx, resolved.fetchTimeoutMs, resolved.fetchMaxOutputChars) } diff --git a/packages/web/tool-web/tests/tool-web.spec.ts b/packages/web/tool-web/tests/tool-web.spec.ts index f9ffb1b5c5..79af9cbd2a 100644 --- a/packages/web/tool-web/tests/tool-web.spec.ts +++ b/packages/web/tool-web/tests/tool-web.spec.ts @@ -1,5 +1,6 @@ -import { describe, expect, it } from 'vitest' +import { describe, expect, it, vi } from 'vitest' import { Context } from 'cordis' +import TurndownService from 'turndown' import { CallId } from '@deepseek-ai/dsh-llm' import SystemPrompt from '@deepseek-ai/dsh-system-prompt' import ToolRegistry, { type ToolExecutionResult } from '@deepseek-ai/dsh-tools' @@ -9,6 +10,7 @@ import * as ToolWeb from '@deepseek-ai/dsh-tool-web' import { formatSearchOutput, formatFetchOutput, + htmlNestingDepth, parseSearchArgs, parseFetchArgs, presentSearchCall, @@ -92,11 +94,13 @@ describe('search formatting', () => { }) describe('fetch formatting', () => { + const NO_CAP = 1_000_000 + it('renders an html body to markdown text with a status header', () => { const out = formatFetchOutput({ url: 'https://a.test', statusCode: 200, truncated: false, body: { kind: 'html', content: '

Title

Body text

' }, - }) + }, NO_CAP) expect(out).toContain('Fetched https://a.test (HTTP 200)') expect(out).toContain('# Title') expect(out).toContain('Body text') @@ -106,11 +110,37 @@ describe('fetch formatting', () => { const out = formatFetchOutput({ url: 'https://a.test', statusCode: 200, truncated: true, body: { kind: 'text', content: 'plain' }, - }) + }, NO_CAP) expect(out).toContain('plain') expect(out).toContain('Content truncated') }) + it('caps the complete output and notes truncation, even when markdown escaping expands the body', () => { + // 1,000 underscores render as 2,000 escaped characters — conversion can + // outgrow a provider-side body cap, so the bound applies to the output. + const out = formatFetchOutput({ + url: 'https://a.test', statusCode: 200, truncated: false, + body: { kind: 'html', content: `

${'_'.repeat(1000)}

` }, + }, 500) + expect(out.length).toBeLessThanOrEqual(500) + expect(out).toContain('Fetched https://a.test (HTTP 200)') + expect(out).toContain('\\_\\_') + expect(out).toContain('Content truncated') + // Exact and tiny caps: the complete result is bounded, header and footer included. + const exact = formatFetchOutput({ + url: 'https://a.test', statusCode: 200, truncated: false, + body: { kind: 'text', content: 'abc' }, + }, 'Fetched https://a.test (HTTP 200)\n\nabc'.length) + expect(exact).toBe('Fetched https://a.test (HTTP 200)\n\nabc') + const tiny = formatFetchOutput({ + url: 'https://a.test', statusCode: 200, truncated: true, + body: { kind: 'text', content: 'abcdef' }, + }, 10) + expect(tiny).toContain('Fetched https://a.test (HTTP 200)') + expect(tiny).toContain('Content truncated') + expect(tiny).not.toContain('abcdef') + }) + it('renderBody dispatches on kind', () => { expect(renderBody({ kind: 'text', content: 'x' })).toBe('x') expect(renderBody({ kind: 'html', content: '

y

' })).toBe('y') @@ -129,14 +159,38 @@ describe('fetch formatting', () => { .toBe('**bold _italic_**\n\n> quoted') }) - it('falls back to the raw html body when turndown throws on pathological nesting', { timeout: 60_000 }, () => { - // Nesting past V8's default stack overflows turndown/domino's recursive - // walk with a RangeError (measured: 4k levels throw on the main thread, - // 8k in a worker); 20k adds margin over either stack size. The raw body - // must pass through instead of throwing. + it('passes deeply nested html through raw without attempting conversion', () => { + // Unclosed-tag nesting makes the synchronous conversion superlinear + // (seconds at 20k levels, during which the cooperative timeout cannot + // fire), so the depth preflight skips conversion entirely; this must + // return fast, not merely not-throw. const depth = 20_000 const pathological = '
'.repeat(depth) + 'x' + '
'.repeat(depth) + const started = Date.now() expect(renderBody({ kind: 'html', content: pathological })).toBe(pathological) + expect(Date.now() - started).toBeLessThan(2_000) + }) + + it('htmlNestingDepth counts open elements, ignoring void and self-closing tags', () => { + expect(htmlNestingDepth('

x

')).toBe(2) + expect(htmlNestingDepth('

')).toBe(1) + expect(htmlNestingDepth('

x

')).toBe(1) + expect(htmlNestingDepth('plain text, no tags')).toBe(0) + expect(htmlNestingDepth('
'.repeat(600))).toBe(600) + }) + + it('falls back to the raw html when turndown throws despite a shallow depth scan', () => { + // Comments hide markup from the depth scan by design (it may only + // over-count, never under-count real elements); simulate the residual + // turndown failure path with a converter throw instead. + const spy = vi.spyOn(TurndownService.prototype, 'turndown').mockImplementation(() => { + throw new RangeError('Maximum call stack size exceeded') + }) + try { + expect(renderBody({ kind: 'html', content: '

x

' })).toBe('

x

') + } finally { + spy.mockRestore() + } }) it('validates url (non-empty), no timeout parameter', () => {