diff --git a/.agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.i18n.yaml b/.agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.i18n.yaml index a114e32885..41f59f644f 100644 --- a/.agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.i18n.yaml +++ b/.agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.i18n.yaml @@ -1,6 +1,6 @@ # Bilingual-pair consistency record (docs/i18n/README.md): the git blob hash of each # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: -# pnpm run verify-translation-pairing --write -2026-07-26-turndown-for-tool-web-html-markdown.md: c7ef4bf538cc949eec8463c8a2ac750685d1a715 -2026-07-26-turndown-for-tool-web-html-markdown.zh.md: 3104dac3cd6db516396b6773f5d2185f3da22ca3 +# pnpm run verify-translation-pairing --write .agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.md +2026-07-26-turndown-for-tool-web-html-markdown.md: 0e387021e3d3be3011cc0d64d37864b30aec4fdf +2026-07-26-turndown-for-tool-web-html-markdown.zh.md: 44e7d08c1db40a1203e8cda955b521774335e774 diff --git a/.agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.md b/.agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.md index c7ef4bf538..0e387021e3 100644 --- a/.agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.md +++ b/.agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.md @@ -10,7 +10,7 @@ English | [中文](2026-07-26-turndown-for-tool-web-html-markdown.zh.md) ## Decision -`packages/web/tool-web/src/fetch.ts` owns a module-level [`turndown`](https://github.com/mixmark-io/turndown) instance (`headingStyle: 'atx'`, `codeBlockStyle: 'fenced'`, `bulletListMarker: '-'` — fixed model-facing presentation, not deployment tunables) with `@joplin/turndown-plugin-gfm`'s composite `gfm` plugin for tables/strikethrough and `remove(['script', 'style', 'noscript'])` replacing the old wholesale drops. `renderBody`'s `html` arm guards the conversion twice: a linear tag-scan preflight passes bodies nested past 512 levels through raw (the synchronous walk is superlinear on unclosed nesting — measured seconds at 20k levels — during which the cooperative timeout cannot fire), and a try/catch falls back to the raw HTML when turndown still throws on markup the scan cannot see; a degraded page beats an error for a body the provider already decoded. `formatFetchOutput` bounds the complete output (`fetchMaxOutputChars` config, default 200,000) because markdown escaping can expand converted HTML to ~2× a provider's body cap. `html.ts` and its conversion tests are deleted; the fallback and the status-header/truncation-footer formatting are tested in `tests/tool-web.spec.ts`, and the README's Known Limitations trades the regex-converter caveat for the pathological-nesting fallback. The gfm plugin ships no types; `src/turndown-plugin-gfm.d.ts` declares the one imported export over `@types/turndown` (a devDependency). +`packages/web/tool-web/src/fetch.ts` owns a module-level [`turndown`](https://github.com/mixmark-io/turndown) instance (`headingStyle: 'atx'`, `codeBlockStyle: 'fenced'`, `bulletListMarker: '-'` — fixed model-facing presentation, not deployment tunables) with `@joplin/turndown-plugin-gfm`'s composite `gfm` plugin for tables/strikethrough and `remove(['script', 'style', 'noscript'])` replacing the old wholesale drops. `formatFetchOutput` limits both the source prefix converted synchronously and the complete rendered output with `fetchMaxOutputChars` (default 200,000), so a custom provider cannot make conversion work unbounded before the output cap applies. The HTML arm then guards conversion twice: a conservative linear lexical pass treats comment contents conservatively, skips raw-text bodies, honors quoted tag text, and passes a body through raw when its stack crosses 512 levels; a try/catch also falls back to raw HTML when turndown rejects markup the guard cannot model. The GFM cell rule is overridden to ignore `colspan`, which Markdown cannot represent, rather than letting an untrusted numeric attribute synthesize arbitrary empty cells. `html.ts` and its conversion tests are deleted; the source/output bounds, fallback, and status-header/truncation-footer formatting are tested in `tests/tool-web.spec.ts`, and the README's Known Limitations trades the regex-converter caveat for the bounded degradation cases. The gfm plugin ships no types; `src/turndown-plugin-gfm.d.ts` declares the one imported export over `@types/turndown` (a devDependency). The dependency-weight question the proposal flagged resolves in favor of the swap: `@deepseek-ai/dsh-tool-web` is in the single-file-executable closure ([single-exe note](../architecture/2026-07-10-single-file-executable-sdk-runtime-distribution.md)), and the exe's asset globs would pack ~7.9 MB of the three packages as published — but ~6 MB of that is `@mixmark-io/domino`'s test corpus (`test/**`), with runtime `lib/` at ~550 KB against a ~174 MB artifact, under 0.5% either way. @@ -27,11 +27,11 @@ The previously-missing keyless `web_fetch` snapshot ships with the change as the ## Consequences -- **Bought**: full-fidelity model-visible markdown — tables, images, strikethrough, nested emphasis, fenced code blocks, and the complete named-entity set — plus the deletion of the bespoke converter and its entity tables, with the README's regex-converter caveat narrowed to one degenerate case. -- **Paid**: two runtime dependencies (`turndown` → `@mixmark-io/domino`) enter tool-web and therefore the exe closure (~550 KB of runtime code as measured above), and a new failure mode — pathological nesting — is handled by falling back to raw HTML rather than converting. +- **Bought**: standards-based model-visible markdown — ordinary tables, images, strikethrough, nested emphasis, fenced code blocks, and the complete named-entity set — plus the deletion of the bespoke converter and its entity tables. +- **Paid**: two runtime dependencies (`turndown` → `@mixmark-io/domino`) enter tool-web and therefore the exe closure (~550 KB of runtime code as measured above); overlong input is converted only through a bounded prefix, pathological nesting falls back to raw HTML, and spanning table cells are flattened because GFM has no corresponding syntax. - Model-visible output changed on every fetched HTML page; nothing pinned the old output, and the new snapshot pins the new one. ## Testing -- `packages/web/tool-web/tests/tool-web.spec.ts` covers the turndown conversion surface (entities, links, tables, nesting, script/style/noscript removal) through `renderBody`, the fast raw-HTML passthrough for 20k-level nesting, the depth scan's void/self-closing/unbalanced cases, the residual converter-throw fallback, and the whole-output cap at expanding, exact, and tiny budgets; per-file coverage on the package src is 100%. +- `packages/web/tool-web/tests/tool-web.spec.ts` covers the turndown conversion surface (entities, links, tables, nesting, script/style/noscript removal), ignored table spans, source-prefix and complete-output bounds, fast raw-HTML passthrough for deep or deceptively closed nesting, linear handling of malformed tags, the residual converter-throw fallback, and exact and tiny output budgets; per-file coverage on the package src is 100%. - The `web-fetch` acp-agent snapshot pins the assembled behavior keylessly end to end (real Loader composition, real HTTP fetch, real conversion). diff --git a/.agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.zh.md b/.agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.zh.md index 3104dac3cd..44e7d08c1d 100644 --- a/.agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.zh.md +++ b/.agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.zh.md @@ -10,7 +10,7 @@ Status: implemented ## 决策 -`packages/web/tool-web/src/fetch.ts` 持有一个模块级 [`turndown`](https://github.com/mixmark-io/turndown) 实例（`headingStyle: 'atx'`、`codeBlockStyle: 'fenced'`、`bulletListMarker: '-'`——固定的面向模型呈现方式，不是部署可调项），配合 `@joplin/turndown-plugin-gfm` 的组合 `gfm` 插件提供表格／删除线支持，并用 `remove(['script', 'style', 'noscript'])` 替代旧实现的整体剥离。`renderBody` 的 `html` 分支对转换做了双重防护：一次线性标签扫描预检把嵌套超过 512 层的主体直接原样透传（同步遍历在未闭合嵌套上呈超线性——实测 2 万层需要数秒——期间协作式超时无法触发），扫描看不到的标记若仍让 turndown 抛异常，则由 try/catch 回退为原始 HTML；对提供方已经解码的主体来说，降级页面好过报错。`formatFetchOutput` 对完整输出设上限（`fetchMaxOutputChars` 配置，默认 200,000）：markdown 转义可能把转换后的 HTML 膨胀到提供方主体上限的约 2 倍。`html.ts` 及其转换测试已删除；透传、回退与整体输出上限，连同状态头、截断页脚的格式化，都在 `tests/tool-web.spec.ts` 中有测试覆盖，README 的 Known Limitations 用病态嵌套回退条目替换了正则转换器的警示说明。gfm 插件不带类型声明；`src/turndown-plugin-gfm.d.ts` 基于 `@types/turndown`（devDependency）声明了唯一被导入的导出。 +`packages/web/tool-web/src/fetch.ts` 持有一个模块级 [`turndown`](https://github.com/mixmark-io/turndown) 实例（`headingStyle: 'atx'`、`codeBlockStyle: 'fenced'`、`bulletListMarker: '-'`——固定的面向模型呈现方式，不是部署可调项），配合 `@joplin/turndown-plugin-gfm` 的组合 `gfm` 插件提供表格／删除线支持，并用 `remove(['script', 'style', 'noscript'])` 替代旧实现的整体剥离。`formatFetchOutput` 通过 `fetchMaxOutputChars`（默认 200,000）同时限制同步转换的源前缀和完整渲染输出，因此自定义提供方无法在输出上限生效前造成无界的转换工作。随后，HTML 分支对转换做双重防护：保守的线性词法扫描会保守处理注释内容，跳过原始文本元素的内容，正确处理标签内的引号文本，并在栈深超过 512 层时将主体作为原始 HTML 直接透传；当 turndown 拒绝守卫无法建模的标记时，try/catch 同样回退为原始 HTML。GFM 单元格规则被覆写为忽略 `colspan`；Markdown 无法表示它，这也避免了不受信任的数值属性凭空合成任意数量的空单元格。`html.ts` 及其转换测试已删除；源／输出上限、回退以及状态头／截断页脚格式化均在 `tests/tool-web.spec.ts` 中有测试覆盖，README 的 Known Limitations 用有界降级情形替换了正则转换器警示。gfm 插件不带类型声明；`src/turndown-plugin-gfm.d.ts` 基于 `@types/turndown`（devDependency）声明了唯一被导入的导出。 提案标记的依赖体积问题的裁决结果支持替换：`@deepseek-ai/dsh-tool-web` 在单文件可执行文件闭包内（[single-exe 决策记录](../architecture/2026-07-10-single-file-executable-sdk-runtime-distribution.md)），可执行文件的资产 glob 会把这三个包按发布原样打入约 7.9 MB——但其中约 6 MB 是 `@mixmark-io/domino` 的测试语料（`test/**`），运行时 `lib/` 仅约 550 KB，相对约 174 MB 的产物，两种口径都不到 0.5%。 @@ -27,11 +27,11 @@ Status: implemented ## 后果 -- **收益**：模型可见的完整保真 markdown——表格、图片、删除线、嵌套强调、围栏代码块以及完整的命名实体集——并删除了自制转换器及其实体表，README 中的正则转换器警示收窄为一个退化用例。 -- **代价**：两个运行时依赖（`turndown` → `@mixmark-io/domino`）进入 tool-web 进而进入可执行文件闭包（如上实测约 550 KB 运行时代码），并新增一种失败模式——病态嵌套改为回退原始 HTML 而非转换。 +- **收益**：基于标准的模型可见 markdown——普通表格、图片、删除线、嵌套强调、围栏代码块以及完整的命名实体集——并删除了自制转换器及其实体表。 +- **代价**：两个运行时依赖（`turndown` → `@mixmark-io/domino`）进入 tool-web 进而进入可执行文件闭包（如上实测约 550 KB 运行时代码）；超长输入只转换有界前缀，病态嵌套回退为原始 HTML，跨列表格单元格会被展平，因为 GFM 没有对应语法。 - 每个抓取到的 HTML 页面上模型可见的输出都已变化；旧输出本无任何固定，新快照固定了新输出。 ## 测试 -- `packages/web/tool-web/tests/tool-web.spec.ts` 通过 `renderBody` 覆盖 turndown 转换面（实体、链接、表格、嵌套、script/style/noscript 移除）、2 万层嵌套的快速原样透传、深度扫描的空元素／自闭合／不平衡用例、残余的转换器抛错回退，以及在膨胀、恰好、极小预算下的整体输出上限；该包 src 的逐文件覆盖率为 100%。 +- `packages/web/tool-web/tests/tool-web.spec.ts` 覆盖 turndown 转换面（实体、链接、表格、嵌套、script/style/noscript 移除）、被忽略的表格跨列、源前缀与完整输出上限、深层或带欺骗性闭合嵌套的快速原始 HTML 透传、畸形标签的线性处理、残余的转换器抛错回退，以及恰好达到上限和极小的输出预算；该包 src 的逐文件覆盖率为 100%。 - acp-agent 的 `web-fetch` 快照无密钥地端到端固定组装后的行为（真实 Loader 组合、真实 HTTP 抓取、真实转换）。 diff --git a/docs/config-catalog.md b/docs/config-catalog.md index 471a5dce96..fb96ff101a 100644 --- a/docs/config-catalog.md +++ b/docs/config-catalog.md @@ -1669,12 +1669,12 @@ export interface Config { fetchTimeoutMs?: number /** Cooperative timeout budget (ms) for `web_search`. Defaults to 30000. */ searchTimeoutMs?: number - /** Cap on one `web_fetch` output's characters (header, rendered body, and footer). Defaults to 200000. */ + /** Cap on source characters converted and complete `web_fetch` output characters. Defaults to 200000. */ fetchMaxOutputChars?: number } ``` -Source: [`packages/web/tool-web/src/index.ts:37`](../packages/web/tool-web/src/index.ts) +Source: [`packages/web/tool-web/src/index.ts:35`](../packages/web/tool-web/src/index.ts) ## `@deepseek-ai/dsh-tool-workflow` diff --git a/examples/acp-agent/web-fetch-fixture-server.mjs b/examples/acp-agent/web-fetch-fixture-server.mjs index 34a45fdd15..1667562faf 100644 --- a/examples/acp-agent/web-fetch-fixture-server.mjs +++ b/examples/acp-agent/web-fetch-fixture-server.mjs @@ -45,8 +45,11 @@ export async function apply(ctx) { }) // The fixture must never hold the process open past protocol shutdown. server.unref() - ctx.effect(() => () => { - server.close() - server.closeAllConnections() + ctx.effect(() => async () => { + await new Promise((resolve, reject) => { + server.close(error => error ? reject(error) : resolve(undefined)) + // Stop accepting first so a connection cannot arrive after the forced close. + server.closeAllConnections() + }) }, 'web-fetch-fixture-server') } diff --git a/packages/web/tool-web/README.i18n.yaml b/packages/web/tool-web/README.i18n.yaml index 44279a66be..58641384b9 100644 --- a/packages/web/tool-web/README.i18n.yaml +++ b/packages/web/tool-web/README.i18n.yaml @@ -1,6 +1,6 @@ # Bilingual-pair consistency record (docs/i18n/README.md): the git blob hash of each # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: -# pnpm run verify-translation-pairing --write -README.md: 44cb1ba2a2f4e1fba7e192d8b6645e0447ebf221 -README.zh.md: 35b390dd5407af16d84ab391dd8351f784c60035 +# pnpm run verify-translation-pairing --write packages/web/tool-web/README.md +README.md: 9b78920b1b6c611118294421dec1e75e381ed5d6 +README.zh.md: 2152c40f1ccac2272fa0b2681514a712417c0ad3 diff --git a/packages/web/tool-web/README.md b/packages/web/tool-web/README.md index 44cb1ba2a2..9b78920b1b 100644 --- a/packages/web/tool-web/README.md +++ b/packages/web/tool-web/README.md @@ -26,9 +26,9 @@ The normalized seam results are also the canonical tool values: `WebSearchResult | `searchMaxResults` | `8` | Upper bound on sources returned by one `web_search` call (the seam truncates a longer provider list and flags it). | | `fetchTimeoutMs` | `30000` | Cooperative tool-call timeout budget (ms) for `web_fetch`. | | `searchTimeoutMs` | `30000` | Cooperative tool-call timeout budget (ms) for `web_search`. | -| `fetchMaxOutputChars` | `200000` | Cap on one `web_fetch` output's characters — header, rendered body, and footer together; a cut body gets the truncation notice. | +| `fetchMaxOutputChars` | `200000` | Cap on source characters converted synchronously and on one complete `web_fetch` output (header, rendered body, and footer); a cut body gets the truncation notice when it fits. | -`fetchTimeoutMs`/`searchTimeoutMs` declare each tool's cooperative timeout budget (attached as `ToolDefinition.timeoutMs`), enforced by [`@deepseek-ai/dsh-timeout-policy`](../../timeout/timeout-policy/README.md); the model-facing schema exposes no timeout argument. `fetchMaxOutputChars` bounds the complete rendered output because markdown escaping can expand converted HTML past a provider's body cap (worst case ~2×); the default is 2× the local provider's default 100,000-character body cap, so it never cuts what that bound already admits. +`fetchTimeoutMs`/`searchTimeoutMs` declare each tool's cooperative timeout budget (attached as `ToolDefinition.timeoutMs`), enforced by [`@deepseek-ai/dsh-timeout-policy`](../../timeout/timeout-policy/README.md); the model-facing schema exposes no timeout argument. `fetchMaxOutputChars` bounds both synchronous conversion work and the complete rendered result: only that many source characters are converted, and the header, converted prefix, and truncation notice are then capped together. The default leaves headroom above the local provider's 100,000-character body cap, but rendered expansion can still make the final bound truncate the result. ```yaml - id: tool-web @@ -127,6 +127,6 @@ Append-only; newly visible content follows the reusable request prefix and does ## Known Limitations and Deferred Work -- **HTML→markdown conversion falls back to raw HTML on pathological input** — [turndown](https://github.com/mixmark-io/turndown) (with GFM tables/strikethrough) converts fetched HTML through a real DOM, but the synchronous walk is superlinear on deep unclosed nesting, so bodies nested past a fixed 512-level preflight bound pass through unconverted (as does anything that still makes turndown throw) rather than stalling the event loop or erroring ([Agent Note](../../../.agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.md)). +- **HTML→markdown conversion degrades on inputs GFM cannot safely represent** — [turndown](https://github.com/mixmark-io/turndown) (with GFM tables/strikethrough) converts at most `fetchMaxOutputChars` source characters through a real DOM. A conservative 512-level lexical guard passes deeply or ambiguously nested bodies through as raw HTML, conversion exceptions do the same, and table `colspan` is ignored because GFM has no spanning-cell representation; these bounds avoid blocking the event loop or expanding output from an untrusted numeric attribute ([Agent Note](../../../.agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.md)). - **The model-facing surface is minimal by design, with promotions deferred** — `max_results` stays a config bound (not a model argument), and `web_fetch` takes only `url` (no `format`/`prompt`/LLM-summarization mode); both are named later steps in [the seam Agent Note](../../../.agents/notes/implemented/architecture/2026-06-24-web-capability-seam.md). - **No web-specific permission policy** — both tools execute without requesting `ctx.approval`; a deployment that needs confirmation must add a `tools/pre-execute` policy, and the package does not define persistent URL/domain grants. diff --git a/packages/web/tool-web/README.zh.md b/packages/web/tool-web/README.zh.md index 35b390dd54..2152c40f1c 100644 --- a/packages/web/tool-web/README.zh.md +++ b/packages/web/tool-web/README.zh.md @@ -26,9 +26,9 @@ | `searchMaxResults` | `8` | 一次 `web_search` 调用返回的源数量上限（seam 截断更长的提供方列表并标记）。 | | `fetchTimeoutMs` | `30000` | `web_fetch` 的协作式工具调用超时预算（ms）。 | | `searchTimeoutMs` | `30000` | `web_search` 的协作式工具调用超时预算（ms）。 | -| `fetchMaxOutputChars` | `200000` | 单次 `web_fetch` 输出的字符上限——状态头、渲染后的主体与页脚合并计算；被截断的主体带截断提示。 | +| `fetchMaxOutputChars` | `200000` | 同步转换的源字符数与单次完整 `web_fetch` 输出的上限（状态头、渲染后的主体与页脚合并计算）；主体被截断时，在能容纳的情况下附带截断提示。 | -`fetchTimeoutMs`／`searchTimeoutMs` 声明每个工具的协作式超时预算（附加为 `ToolDefinition.timeoutMs`），由 [`@deepseek-ai/dsh-timeout-policy`](../../timeout/timeout-policy/README.md) 强制执行；面向模型的 schema 不公开超时参数。`fetchMaxOutputChars` 对完整渲染输出设上限：markdown 转义可能让转换后的 HTML 超出提供方的主体上限（最坏约 2 倍）；默认值取本地提供方默认 100,000 字符主体上限的 2 倍，因此绝不会削减该上限本已允许的内容。 +`fetchTimeoutMs`／`searchTimeoutMs` 声明每个工具的协作式超时预算（附加为 `ToolDefinition.timeoutMs`），由 [`@deepseek-ai/dsh-timeout-policy`](../../timeout/timeout-policy/README.md) 强制执行；面向模型的 schema 不公开超时参数。`fetchMaxOutputChars` 同时限制同步转换工作量和完整渲染结果：只转换至多该数量的源字符，随后对状态头、转换后的前缀和截断提示合并设限。默认值为本地提供方的 100,000 字符主体上限留出余量，但渲染膨胀仍可能使最终上限截断结果。 ```yaml - id: tool-web @@ -127,6 +127,6 @@ Use the web_fetch tool to retrieve the content of a specific HTTP(S) URL (for ex ## 已知限制与暂缓事项 -- **HTML→markdown 转换在病态输入上回退为原始 HTML**：[turndown](https://github.com/mixmark-io/turndown)（带 GFM 表格／删除线）通过真实 DOM 转换抓取到的 HTML，但同步遍历在深层未闭合嵌套上呈超线性，因此嵌套超过固定 512 层预检上限的主体不经转换原样通过（仍让 turndown 抛异常的输入同样如此），而非阻塞事件循环或报错（[决策记录](../../../.agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.md)）。 +- **HTML→markdown 转换会在 GFM 无法安全表示的输入上降级**：[turndown](https://github.com/mixmark-io/turndown)（带 GFM 表格／删除线）通过真实 DOM 转换至多 `fetchMaxOutputChars` 个源字符。保守的 512 层词法守卫会将深层或嵌套有歧义的主体作为原始 HTML 直接透传，转换异常也会如此处理；表格的 `colspan` 会被忽略，因为 GFM 无法表示跨列单元格。这些限制可避免阻塞事件循环，也避免不受信任的数值属性使输出膨胀（[决策记录](../../../.agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.md)）。 - **面向模型的表层有意保持最小，提升项暂缓**：`max_results` 保持为配置上限（不是模型参数），`web_fetch` 只接受 `url`（没有 `format`／`prompt`／LLM 摘要模式）；两项都列为 [seam Agent Note](../../../.agents/notes/implemented/architecture/2026-06-24-web-capability-seam.md) 中的后续步骤。 - **没有 web 专用权限策略**：两个工具都不会请求 `ctx.approval` 就直接执行；需要确认的部署必须添加 `tools/pre-execute` 策略，该包不定义持久 URL／domain 授权。 diff --git a/packages/web/tool-web/src/fetch.ts b/packages/web/tool-web/src/fetch.ts index e909ea25be..75108f663c 100644 --- a/packages/web/tool-web/src/fetch.ts +++ b/packages/web/tool-web/src/fetch.ts @@ -30,6 +30,52 @@ const turndown = new TurndownService({ turndown.use(gfm) turndown.remove(['script', 'style', 'noscript']) +/** Render one GFM table cell without interpreting HTML span counts. */ +function renderTableCell(content: string, index: number): string { + const prefix = index === 0 ? '| ' : ' ' + const escaped = content.trim().replace(/\n\r/g, '
').replace(/\n/g, '
').replace(/\|+/g, '\\|').padEnd(3, ' ') + return `${prefix}${escaped} |` +} + +/** Whether a row is the table's Markdown heading row. */ +function isTableHeadingRow(row: HTMLTableRowElement): boolean { + const cells = Array.from(row.cells) + const section = row.parentElement as HTMLTableSectionElement + const table = section.parentElement as HTMLTableElement + return (section.nodeName === 'THEAD' || table.rows[0] === row) + && cells.every(cell => cell.nodeName === 'TH') +} + +/** Map an HTML table-cell alignment to the GFM separator marker. */ +function tableBorder(cell: HTMLTableCellElement): string { + const alignment = (cell.getAttribute('align') || cell.style.textAlign || '').toLowerCase() + if (alignment === 'left') return ':---' + if (alignment === 'right') return '---:' + if (alignment === 'center') return ':---:' + return '---' +} + +turndown.addRule('tableCellWithoutSpanExpansion', { + filter: ['th', 'td'], + replacement(content, node) { + const cell = node as HTMLTableCellElement + const row = cell.parentNode as HTMLTableRowElement + // GFM cannot represent spanning cells. Ignoring colspan keeps conversion + // work and output proportional to the source instead of the numeric attribute. + return renderTableCell(content, Array.prototype.indexOf.call(row.childNodes, cell)) + }, +}) +turndown.addRule('tableRowWithoutSpanExpansion', { + filter: 'tr', + replacement(content, node) { + const row = node as HTMLTableRowElement + const border = isTableHeadingRow(row) + ? Array.from(row.cells, (cell, index) => renderTableCell(tableBorder(cell), index)).join('') + : '' + return `\n${content}${border.length > 0 ? `\n${border}` : ''}` + }, +}) + /** * Validate value constraints the schema DSL can't express: a non-blank `url`. * Throws a plain `Error` otherwise. No timeout parameter — the tool-call budget @@ -55,63 +101,142 @@ export function parseFetchArgs(args: { url: string }): { url: string } { */ const MAX_CONVERSION_DEPTH = 512 -/** Elements that never take a closing tag, so they must not count toward nesting depth. */ +/** Elements that never take a closing tag, so they do not grow the lexical stack. */ const VOID_ELEMENTS = new Set([ 'area', 'base', 'br', 'col', 'embed', 'hr', 'img', 'input', 'link', 'meta', 'param', 'source', 'track', 'wbr', ]) +/** Elements whose contents HTML parses as text until their matching end tag. */ +const RAW_TEXT_ELEMENTS = new Set(['script', 'style', 'noscript']) + +/** Whether a character can occur after a raw-text end-tag name. */ +function isTagBoundary(char: string | undefined): boolean { + return char === undefined || char === '>' || char === '/' || /\s/.test(char) +} + +/** Find the matching raw-text end tag without interpreting markup-like body text. */ +function findRawTextEnd(lowerHtml: string, name: string, from: number): number { + const prefix = `` characters, and only accepts a closing + * tag for the current element; malformed input therefore over-counts rather + * than hiding nesting. * * @param html - the decoded HTML body. - * @returns the deepest open-element count the scan reaches. + * @returns whether the body crosses {@link MAX_CONVERSION_DEPTH}. */ -export function htmlNestingDepth(html: string): number { - let depth = 0 - let max = 0 - for (const tag of html.matchAll(/<(\/?)([a-zA-Z][a-zA-Z0-9-]*)[^>]*?(\/?)>/g)) { - const [, closing, rawName = '', selfClosing] = tag - const name = rawName.toLowerCase() - if (VOID_ELEMENTS.has(name) || selfClosing === '/') continue - if (closing === '/') { - if (depth > 0) depth -= 1 - } else { - depth += 1 - if (depth > max) max = depth +function exceedsConversionDepth(html: string): boolean { + const lowerHtml = html.toLowerCase() + const openElements: string[] = [] + let offset = 0 + let inComment = false + + while (offset < html.length) { + const start = html.indexOf('<', offset) + if (inComment) { + const end = html.indexOf('-->', offset) + if (end !== -1 && (start === -1 || end < start)) { + inComment = false + offset = end + 3 + continue + } } + if (start === -1) break + if (!inComment && html.startsWith(''.repeat(600) + 'x' + expect(formatFetchOutput({ + url: 'https://a.test', statusCode: 200, truncated: false, + body: { kind: 'html', content: pathological }, + }, NO_CAP)).toBe(`${HEADER}${pathological}`) + const abruptlyClosedComments = '
'.repeat(600) + 'x' + expect(formatFetchOutput({ + url: 'https://a.test', statusCode: 200, truncated: false, + body: { kind: 'html', content: abruptlyClosedComments }, + }, NO_CAP)).toBe(`${HEADER}${abruptlyClosedComments}`) + }) + + it('the preflight accepts ordinary closed, void, self-closing, quoted, and raw-text markup', () => { + const paragraphs = '

\'>x

'.repeat(600) + const script = `` + expect(renderHtml(`<1bad>${paragraphs}${script}`)) + .not.toContain('x