Merge remote-tracking branch 'origin/master' into fix/pty-handoff-grace

packages/web/tool-web/README.i18n.yaml

@@ -1,6 +1,6 @@
 # Bilingual-pair consistency record (docs/i18n/README.md): the git blob hash of each
 # side as of the last confirmed-consistent state. Both languages carry equal authority;
 # after editing either side, bring the other along and re-record with:
-#   pnpm run verify-translation-pairing --write
-README.md: 5e567115c386d14b7e412ed2502e7290826a5e5e
-README.zh.md: b17fe4107908381806d4029481bbf03696c4f313
+#   pnpm run verify-translation-pairing --write packages/web/tool-web/README.md
+README.md: 9b78920b1b6c611118294421dec1e75e381ed5d6
+README.zh.md: 2152c40f1ccac2272fa0b2681514a712417c0ad3

packages/web/tool-web/README.md

@@ -11,7 +11,7 @@ Each tool is registered independently; a product that wants only one disables th
 | Tool | Args | Behavior |
 |---|---|---|
 | `web_search` | `query` (string) | Discovery. Returns an optional answer plus source URLs. `max_results` is **not** model-facing — the tool sets the bound (the `searchMaxResults` config, default 8) and passes it to the seam. |
-| `web_fetch` | `url` (string) | Retrieves a specific URL. HTML bodies are rendered to markdown-ish text; text bodies pass through. A non-2xx status is reported, not an error. The tool-call timeout is deployment policy (`dsh-timeout-policy`), not a model argument. |
+| `web_fetch` | `url` (string) | Retrieves a specific URL. HTML bodies are rendered to markdown (turndown with GFM tables/strikethrough); text bodies pass through. A non-2xx status is reported, not an error. The tool-call timeout is deployment policy (`dsh-timeout-policy`), not a model argument. |
 
 Both tools opt into concurrent scheduling because provider reads return content without mutating parent-agent state.
 
@@ -26,8 +26,9 @@ The normalized seam results are also the canonical tool values: `WebSearchResult
 | `searchMaxResults` | `8` | Upper bound on sources returned by one `web_search` call (the seam truncates a longer provider list and flags it). |
 | `fetchTimeoutMs` | `30000` | Cooperative tool-call timeout budget (ms) for `web_fetch`. |
 | `searchTimeoutMs` | `30000` | Cooperative tool-call timeout budget (ms) for `web_search`. |
+| `fetchMaxOutputChars` | `200000` | Cap on source characters converted synchronously and on one complete `web_fetch` output (header, rendered body, and footer); a cut body gets the truncation notice when it fits. |
 
-`fetchTimeoutMs`/`searchTimeoutMs` declare each tool's cooperative timeout budget (attached as `ToolDefinition.timeoutMs`), enforced by [`@deepseek-ai/dsh-timeout-policy`](../../timeout/timeout-policy/README.md); the model-facing schema exposes no timeout argument.
+`fetchTimeoutMs`/`searchTimeoutMs` declare each tool's cooperative timeout budget (attached as `ToolDefinition.timeoutMs`), enforced by [`@deepseek-ai/dsh-timeout-policy`](../../timeout/timeout-policy/README.md); the model-facing schema exposes no timeout argument. `fetchMaxOutputChars` bounds both synchronous conversion work and the complete rendered result: only that many source characters are converted, and the header, converted prefix, and truncation notice are then capped together. The default leaves headroom above the local provider's 100,000-character body cap, but rendered expansion can still make the final bound truncate the result.
 
 ```yaml
 - id: tool-web
@@ -126,6 +127,6 @@ Append-only; newly visible content follows the reusable request prefix and does
 
 ## Known Limitations and Deferred Work
 
-- **`htmlToMarkdown` is a minimal regex converter, not an HTML parser** — it strips script/style/noscript, keeps headings/bullets/links, and decodes about a dozen named entities; tables, images, and nested formatting are lost.
+- **HTML→markdown conversion degrades on inputs GFM cannot safely represent** — [turndown](https://github.com/mixmark-io/turndown) (with GFM tables/strikethrough) converts at most `fetchMaxOutputChars` source characters through a real DOM. A conservative 512-level lexical guard passes deeply or ambiguously nested bodies through as raw HTML, conversion exceptions do the same, and table `colspan` is ignored because GFM has no spanning-cell representation; these bounds avoid blocking the event loop or expanding output from an untrusted numeric attribute ([Agent Note](../../../.agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.md)).
 - **The model-facing surface is minimal by design, with promotions deferred** — `max_results` stays a config bound (not a model argument), and `web_fetch` takes only `url` (no `format`/`prompt`/LLM-summarization mode); both are named later steps in [the seam Agent Note](../../../.agents/notes/implemented/architecture/2026-06-24-web-capability-seam.md).
 - **No web-specific permission policy** — both tools execute without requesting `ctx.approval`; a deployment that needs confirmation must add a `tools/pre-execute` policy, and the package does not define persistent URL/domain grants.

packages/web/tool-web/README.zh.md

@@ -11,7 +11,7 @@
 | 工具 | 参数 | 行为 |
 |---|---|---|
 | `web_search` | `query`（string） | 发现。返回可选答案与源 URL。`max_results` **不** 面向模型：工具设置上限（`searchMaxResults` 配置，默认 8）并传给 seam。 |
-| `web_fetch` | `url`（string） | 获取特定 URL。HTML 主体渲染为近似 markdown 的文本；文本主体原样通过。非 2xx 状态会报告，而非报错。工具调用超时是部署策略（`dsh-timeout-policy`），不是模型参数。 |
+| `web_fetch` | `url`（string） | 获取特定 URL。HTML 主体渲染为 markdown（turndown，带 GFM 表格／删除线）；文本主体原样通过。非 2xx 状态会报告，而非报错。工具调用超时是部署策略（`dsh-timeout-policy`），不是模型参数。 |
 
 两个工具都选择并发调度，因为提供方读取会返回内容，不会修改父 agent 状态。
 
@@ -26,8 +26,9 @@
 | `searchMaxResults` | `8` | 一次 `web_search` 调用返回的源数量上限（seam 截断更长的提供方列表并标记）。 |
 | `fetchTimeoutMs` | `30000` | `web_fetch` 的协作式工具调用超时预算（ms）。 |
 | `searchTimeoutMs` | `30000` | `web_search` 的协作式工具调用超时预算（ms）。 |
+| `fetchMaxOutputChars` | `200000` | 同步转换的源字符数与单次完整 `web_fetch` 输出的上限（状态头、渲染后的主体与页脚合并计算）；主体被截断时，在能容纳的情况下附带截断提示。 |
 
-`fetchTimeoutMs`／`searchTimeoutMs` 声明每个工具的协作式超时预算（附加为 `ToolDefinition.timeoutMs`），由 [`@deepseek-ai/dsh-timeout-policy`](../../timeout/timeout-policy/README.md) 强制执行；面向模型的 schema 不公开超时参数。
+`fetchTimeoutMs`／`searchTimeoutMs` 声明每个工具的协作式超时预算（附加为 `ToolDefinition.timeoutMs`），由 [`@deepseek-ai/dsh-timeout-policy`](../../timeout/timeout-policy/README.md) 强制执行；面向模型的 schema 不公开超时参数。`fetchMaxOutputChars` 同时限制同步转换工作量和完整渲染结果：只转换至多该数量的源字符，随后对状态头、转换后的前缀和截断提示合并设限。默认值为本地提供方的 100,000 字符主体上限留出余量，但渲染膨胀仍可能使最终上限截断结果。
 
 ```yaml
 - id: tool-web
@@ -126,6 +127,6 @@ Use the web_fetch tool to retrieve the content of a specific HTTP(S) URL (for ex
 
 ## 已知限制与暂缓事项
 
-- **`htmlToMarkdown` 是最小正则转换器，不是 HTML parser**：它会移除 script/style/noscript，保留标题／项目符号／链接，并解码约十余个命名 entity；表格、图片与嵌套格式会丢失。
+- **HTML→markdown 转换会在 GFM 无法安全表示的输入上降级**：[turndown](https://github.com/mixmark-io/turndown)（带 GFM 表格／删除线）通过真实 DOM 转换至多 `fetchMaxOutputChars` 个源字符。保守的 512 层词法守卫会将深层或嵌套有歧义的主体作为原始 HTML 直接透传，转换异常也会如此处理；表格的 `colspan` 会被忽略，因为 GFM 无法表示跨列单元格。这些限制可避免阻塞事件循环，也避免不受信任的数值属性使输出膨胀（[决策记录](../../../.agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.md)）。
 - **面向模型的表层有意保持最小，提升项暂缓**：`max_results` 保持为配置上限（不是模型参数），`web_fetch` 只接受 `url`（没有 `format`／`prompt`／LLM 摘要模式）；两项都列为 [seam Agent Note](../../../.agents/notes/implemented/architecture/2026-06-24-web-capability-seam.md) 中的后续步骤。
 - **没有 web 专用权限策略**：两个工具都不会请求 `ctx.approval` 就直接执行；需要确认的部署必须添加 `tools/pre-execute` 策略，该包不定义持久 URL／domain 授权。

packages/web/tool-web/package.json

@@ -35,10 +35,13 @@
     "cordis": "^4.0.0-rc.7"
   },
   "dependencies": {
-    "schemastery": "^3.18.0"
+    "@joplin/turndown-plugin-gfm": "^1.0.67",
+    "schemastery": "^3.18.0",
+    "turndown": "^7.2.4"
   },
   "devDependencies": {
     "@deepseek-ai/dsh-agent": "workspace:^",
+    "@types/turndown": "^5.0.6",
     "@deepseek-ai/dsh-invariants": "workspace:^",
     "@deepseek-ai/dsh-llm": "workspace:^",
     "@deepseek-ai/dsh-session": "workspace:^",

packages/web/tool-web/src/fetch.ts

@@ -6,12 +6,75 @@
  */
 
 import type { Context } from 'cordis'
+import TurndownService from 'turndown'
+import { gfm } from '@joplin/turndown-plugin-gfm'
 import { defineTool } from '@deepseek-ai/dsh-tools'
 import type { GenericCallView } from '@deepseek-ai/dsh-tools'
 import type { WebFetchBody, WebFetchResult } from '@deepseek-ai/dsh-web'
 import { assertNever } from '@deepseek-ai/dsh-llm'
 import type {} from '@deepseek-ai/dsh-system-prompt'
-import { htmlToMarkdown } from './html.ts'
+
+/**
+ * The shared HTML→markdown converter: turndown over its bundled domino DOM,
+ * with GitHub-flavored tables/strikethrough (`@joplin/turndown-plugin-gfm`).
+ * The style options are fixed model-facing presentation (matching the repo's
+ * markdown conventions), not deployment tunables. `remove` drops non-content
+ * elements wholesale — turndown's default keeps their text. The instance is
+ * stateless across `turndown()` calls and safe to share.
+ */
+const turndown = new TurndownService({
+  headingStyle: 'atx',
+  codeBlockStyle: 'fenced',
+  bulletListMarker: '-',
+})
+turndown.use(gfm)
+turndown.remove(['script', 'style', 'noscript'])
+
+/** Render one GFM table cell without interpreting HTML span counts. */
+function renderTableCell(content: string, index: number): string {
+  const prefix = index === 0 ? '| ' : ' '
+  const escaped = content.trim().replace(/\n\r/g, '<br>').replace(/\n/g, '<br>').replace(/\|+/g, '\\|').padEnd(3, ' ')
+  return `${prefix}${escaped} |`
+}
+
+/** Whether a row is the table's Markdown heading row. */
+function isTableHeadingRow(row: HTMLTableRowElement): boolean {
+  const cells = Array.from(row.cells)
+  const section = row.parentElement as HTMLTableSectionElement
+  const table = section.parentElement as HTMLTableElement
+  return (section.nodeName === 'THEAD' || table.rows[0] === row)
+    && cells.every(cell => cell.nodeName === 'TH')
+}
+
+/** Map an HTML table-cell alignment to the GFM separator marker. */
+function tableBorder(cell: HTMLTableCellElement): string {
+  const alignment = (cell.getAttribute('align') || cell.style.textAlign || '').toLowerCase()
+  if (alignment === 'left') return ':---'
+  if (alignment === 'right') return '---:'
+  if (alignment === 'center') return ':---:'
+  return '---'
+}
+
+turndown.addRule('tableCellWithoutSpanExpansion', {
+  filter: ['th', 'td'],
+  replacement(content, node) {
+    const cell = node as HTMLTableCellElement
+    const row = cell.parentNode as HTMLTableRowElement
+    // GFM cannot represent spanning cells. Ignoring colspan keeps conversion
+    // work and output proportional to the source instead of the numeric attribute.
+    return renderTableCell(content, Array.prototype.indexOf.call(row.childNodes, cell))
+  },
+})
+turndown.addRule('tableRowWithoutSpanExpansion', {
+  filter: 'tr',
+  replacement(content, node) {
+    const row = node as HTMLTableRowElement
+    const border = isTableHeadingRow(row)
+      ? Array.from(row.cells, (cell, index) => renderTableCell(tableBorder(cell), index)).join('')
+      : ''
+    return `\n${content}${border.length > 0 ? `\n${border}` : ''}`
+  },
+})
 
 /**
  * Validate value constraints the schema DSL can't express: a non-blank `url`.
@@ -27,36 +90,182 @@ export function parseFetchArgs(args: { url: string }): { url: string } {
   return { url: args.url }
 }
 
+/**
+ * Nesting-depth ceiling above which HTML skips conversion and passes through
+ * raw. Conversion runs synchronously on the event loop, and unclosed-tag
+ * nesting makes domino's tree (and turndown's walk over it) superlinear —
+ * measured: depth 512 ≈ 0.15s, 2,000 ≈ 2s, 20,000 ≈ 5s — during which the
+ * cooperative `fetchTimeoutMs` timer cannot fire. Real pages nest a few dozen
+ * levels; 512 is far above content and far below weaponizable. A robustness
+ * invariant, not a tunable.
+ */
+const MAX_CONVERSION_DEPTH = 512
+
+/** Elements that never take a closing tag, so they do not grow the lexical stack. */
+const VOID_ELEMENTS = new Set([
+  'area', 'base', 'br', 'col', 'embed', 'hr', 'img', 'input',
+  'link', 'meta', 'param', 'source', 'track', 'wbr',
+])
+
+/** Elements whose contents HTML parses as text until their matching end tag. */
+const RAW_TEXT_ELEMENTS = new Set(['script', 'style', 'noscript'])
+
+/** Whether a character can occur after a raw-text end-tag name. */
+function isTagBoundary(char: string | undefined): boolean {
+  return char === undefined || char === '>' || char === '/' || /\s/.test(char)
+}
+
+/** Find the matching raw-text end tag without interpreting markup-like body text. */
+function findRawTextEnd(lowerHtml: string, name: string, from: number): number {
+  const prefix = `</${name}`
+  let candidate = lowerHtml.indexOf(prefix, from)
+  while (candidate !== -1 && !isTagBoundary(lowerHtml[candidate + prefix.length])) {
+    candidate = lowerHtml.indexOf(prefix, candidate + prefix.length)
+  }
+  return candidate
+}
+
+/**
+ * Conservatively reject HTML whose lexical element stack crosses the conversion
+ * depth ceiling. The single pass ignores closing tags inside comments, skips
+ * raw-text bodies, respects quoted `>` characters, and only accepts a closing
+ * tag for the current element; malformed input therefore over-counts rather
+ * than hiding nesting.
+ *
+ * @param html - the decoded HTML body.
+ * @returns whether the body crosses {@link MAX_CONVERSION_DEPTH}.
+ */
+function exceedsConversionDepth(html: string): boolean {
+  const lowerHtml = html.toLowerCase()
+  const openElements: string[] = []
+  let offset = 0
+  let inComment = false
+
+  while (offset < html.length) {
+    const start = html.indexOf('<', offset)
+    if (inComment) {
+      const end = html.indexOf('-->', offset)
+      if (end !== -1 && (start === -1 || end < start)) {
+        inComment = false
+        offset = end + 3
+        continue
+      }
+    }
+    if (start === -1) break
+    if (!inComment && html.startsWith('<!--', start)) {
+      inComment = true
+      offset = start + 4
+      continue
+    }
+
+    let cursor = start + 1
+    const closing = html[cursor] === '/'
+    if (closing) cursor += 1
+    const nameStart = cursor
+    while (/[a-zA-Z0-9-]/.test(html[cursor] ?? '')) cursor += 1
+    if (cursor === nameStart || !/[a-zA-Z]/.test(html.charAt(nameStart))) {
+      offset = start + 1
+      continue
+    }
+
+    const name = lowerHtml.slice(nameStart, cursor)
+    let quote: '"' | "'" | undefined
+    while (cursor < html.length) {
+      const char = html[cursor]
+      cursor += 1
+      if (quote !== undefined) {
+        if (char === quote) quote = undefined
+      } else if (char === '"' || char === "'") {
+        quote = char
+      } else if (char === '>') {
+        break
+      }
+    }
+    if (html[cursor - 1] !== '>') break
+
+    if (closing) {
+      if (!inComment && openElements.at(-1) === name) openElements.pop()
+    } else {
+      let last = cursor - 2
+      while (/\s/.test(html.charAt(last))) last -= 1
+      if (!VOID_ELEMENTS.has(name) && html[last] !== '/') {
+        openElements.push(name)
+        if (openElements.length > MAX_CONVERSION_DEPTH) return true
+        if (!inComment && RAW_TEXT_ELEMENTS.has(name)) {
+          const end = findRawTextEnd(lowerHtml, name, cursor)
+          if (end === -1) break
+          offset = end
+          continue
+        }
+      }
+    }
+    offset = cursor
+  }
+  return false
+}
+
+interface RenderedBody {
+  /** Converted text, or raw HTML when conversion is unsafe or fails. */
+  text: string
+  /** Whether the source was cut before conversion to bound synchronous work. */
+  sourceTruncated: boolean
+}
+
 /**
  * Render a fetched body to model-facing markdown text.
  *
- * @param body - the decoded body; `html` is converted via
- *   {@link htmlToMarkdown}, `text` passes through verbatim.
- * @returns the text for the tool's output block.
+ * @param body - the decoded body; `html` is converted via turndown, `text`
+ *   passes through verbatim.
+ * @param maxInputChars - maximum source characters processed synchronously.
+ * @returns the rendered prefix and whether the source was cut. HTML nested
+ *   beyond {@link MAX_CONVERSION_DEPTH} or rejected by turndown passes through
+ *   raw; a degraded page beats an error for a body the provider decoded.
  */
-export function renderBody(body: WebFetchBody): string {
+function renderBody(body: WebFetchBody, maxInputChars: number): RenderedBody {
+  const content = body.content.slice(0, maxInputChars)
+  const sourceTruncated = content.length !== body.content.length
   switch (body.kind) {
     case 'html':
-      return htmlToMarkdown(body.content)
+      if (exceedsConversionDepth(content)) return { text: content, sourceTruncated }
+      try {
+        return { text: turndown.turndown(content), sourceTruncated }
+      } catch {
+        // turndown's DOM walk recurses per element; malformed markup the lexical
+        // guard cannot model can still throw RangeError. Provider errors stay
+        // structured WebErrors upstream; conversion failure downgrades to raw HTML.
+        return { text: content, sourceTruncated }
+      }
     case 'text':
-      return body.content
+      return { text: content, sourceTruncated }
     /* v8 ignore next 2 -- WebFetchBody is a closed union; this arm is unreachable and only makes adding a kind a compile error. */
     default:
       return assertNever(body, 'unhandled web fetch body kind')
   }
 }
 
+/** The truncation notice appended when the provider or the output cap cut content. */
+const TRUNCATION_FOOTER = '\n\n(Content truncated. Fetch a more specific URL or section for the full text.)'
+
 /**
- * Format a fetch result as one model-facing text block.
+ * Format a fetch result as one model-facing text block, bounded as a whole.
+ * The same cap limits the source prefix processed synchronously, then applies
+ * again where the complete output — header, rendered body, and footer — is known.
  *
  * @param result - the seam's fetch outcome.
+ * @param maxOutputChars - cap on the complete returned string; a cut body gets
+ *   the same fetch-something-narrower notice as provider-side truncation.
  * @returns a `Fetched <url> (HTTP <status>)` header, the rendered body, and a
- *   fetch-something-narrower notice when the provider truncated the content.
+ *   truncation notice when the provider or the cap cut the content.
  */
-export function formatFetchOutput(result: WebFetchResult): string {
-  const header = `Fetched ${result.url} (HTTP ${result.statusCode})`
-  const footer = result.truncated ? '\n\n(Content truncated. Fetch a more specific URL or section for the full text.)' : ''
-  return `${header}\n\n${renderBody(result.body)}${footer}`
+export function formatFetchOutput(result: WebFetchResult, maxOutputChars: number): string {
+  const header = `Fetched ${result.url} (HTTP ${result.statusCode})\n\n`
+  const rendered = renderBody(result.body, maxOutputChars)
+  const prefix = `${header}${rendered.text}`
+  const truncated = result.truncated || rendered.sourceTruncated || prefix.length > maxOutputChars
+  const full = `${prefix}${truncated ? TRUNCATION_FOOTER : ''}`
+  if (full.length <= maxOutputChars) return full
+  if (maxOutputChars < TRUNCATION_FOOTER.length) return full.slice(0, maxOutputChars)
+  return `${prefix.slice(0, maxOutputChars - TRUNCATION_FOOTER.length)}${TRUNCATION_FOOTER}`
 }
 
 /**
@@ -76,8 +285,10 @@ export function presentFetchCall(args: { url: string }): GenericCallView {
  *   registrations; both are effect-scoped and unregister on plugin dispose.
  * @param timeoutMs - the cooperative tool-call budget (ms) attached as the tool's
  *   `ToolDefinition.timeoutMs` for `@deepseek-ai/dsh-timeout-policy` to enforce.
+ * @param maxOutputChars - cap on the complete rendered tool output (see
+ *   {@link formatFetchOutput}) and on source characters converted synchronously.
  */
-export function applyWebFetchTool(ctx: Context, timeoutMs: number): void {
+export function applyWebFetchTool(ctx: Context, timeoutMs: number, maxOutputChars: number): void {
   ctx.systemPrompt.section({
     name: 'tool:web_fetch',
     order: 111,
@@ -121,7 +332,7 @@ export function applyWebFetchTool(ctx: Context, timeoutMs: number): void {
           truncated: { type: 'boolean', required: true },
         },
       },
-      render: (_args, value) => [{ type: 'text', text: formatFetchOutput(value) }],
+      render: (_args, value) => [{ type: 'text', text: formatFetchOutput(value, maxOutputChars) }],
     },
     timeoutMs,
     // Provider reads do not mutate parent-agent state.

packages/web/tool-web/src/html.ts

@@ -1,86 +0,0 @@
-/**
- * Minimal dependency-free HTML-to-readable-text conversion for `web_fetch`, not a full parser. It
- * removes non-content elements and tags, decodes common entities, collapses whitespace, and keeps
- * basic headings, lists, and links. A richer converter can replace it without changing the seam or
- * tool schema.
- * @module @deepseek-ai/dsh-tool-web/html
- */
-
-/** Decode the handful of HTML entities common in textual content. */
-function decodeEntities(text: string): string {
-  return text
-    .replace(/&(#[xX][0-9a-fA-F]+|#[0-9]+|[a-zA-Z]+);/g, (match, entity: string) => {
-      if (entity.startsWith('#x') || entity.startsWith('#X')) {
-        const code = Number.parseInt(entity.slice(2), 16)
-        return safeFromCodePoint(code, match)
-      }
-      if (entity.startsWith('#')) {
-        const code = Number.parseInt(entity.slice(1), 10)
-        return safeFromCodePoint(code, match)
-      }
-      return NAMED_ENTITIES[entity] ?? match
-    })
-}
-
-const NAMED_ENTITIES: Record<string, string> = {
-  amp: '&', lt: '<', gt: '>', quot: '"', apos: "'", nbsp: ' ',
-  copy: '©', reg: '®', trade: '™', hellip: '…', mdash: '—', ndash: '–',
-}
-
-function safeFromCodePoint(code: number, fallback: string): string {
-  try {
-    return String.fromCodePoint(code)
-  } catch {
-    // An out-of-range code point (RangeError) is the only failure here; keep the
-    // original entity text rather than throwing out of pure presentation.
-    return fallback
-  }
-}
-
-/**
- * Convert an HTML document to a readable markdown-ish text approximation.
- * Best-effort and lossy by design — fidelity is the job of a future heavier
- * converter, not this fallback.
- *
- * @param html - the raw HTML source.
- * @returns plain text with markdown headings, list bullets, and links;
- *   whitespace collapsed to at most one blank line and trimmed.
- */
-export function htmlToMarkdown(html: string): string {
-  let text = html
-    // Drop non-content elements entirely (including their contents).
-    .replace(/<script\b[^>]*>[\s\S]*?<\/script>/gi, '')
-    .replace(/<style\b[^>]*>[\s\S]*?<\/style>/gi, '')
-    .replace(/<noscript\b[^>]*>[\s\S]*?<\/noscript>/gi, '')
-    .replace(/<!--[\s\S]*?-->/g, '')
-
-  // Convert links to markdown before stripping tags.
-  text = text.replace(/<a\b[^>]*\bhref\s*=\s*["']([^"']*)["'][^>]*>([\s\S]*?)<\/a>/gi, (_match, href: string, label: string) => {
-    const cleanLabel = label.replace(/<[^>]+>/g, '').trim()
-    return cleanLabel.length > 0 ? `[${cleanLabel}](${href})` : href
-  })
-
-  // Headings → markdown hashes.
-  text = text.replace(/<h([1-6])\b[^>]*>([\s\S]*?)<\/h\1>/gi, (_match, level: string, body: string) => {
-    const hashes = '#'.repeat(Number(level))
-    return `\n\n${hashes} ${body.replace(/<[^>]+>/g, '').trim()}\n\n`
-  })
-
-  // List items → bullets.
-  text = text.replace(/<li\b[^>]*>([\s\S]*?)<\/li>/gi, (_match, body: string) => `\n- ${body.replace(/<[^>]+>/g, '').trim()}`)
-
-  // Block-level breaks become paragraph breaks.
-  text = text
-    .replace(/<\/(p|div|section|article|header|footer|tr|table|ul|ol|blockquote)>/gi, '\n\n')
-    .replace(/<br\s*\/?>/gi, '\n')
-
-  // Drop all remaining tags, decode entities, collapse whitespace.
-  text = text.replace(/<[^>]+>/g, '')
-  text = decodeEntities(text)
-  text = text
-    .replace(/[ \t\f\v]+/g, ' ')
-    .replace(/ *\n */g, '\n')
-    .replace(/\n{3,}/g, '\n\n')
-    .trim()
-  return text
-}

packages/web/tool-web/src/index.ts

@@ -13,8 +13,7 @@ import { applyWebSearchTool, WEB_SEARCH_MAX_RESULTS } from './search.ts'
 import { applyWebFetchTool } from './fetch.ts'
 
 export { WEB_SEARCH_MAX_RESULTS, applyWebSearchTool, formatSearchOutput, parseSearchArgs, presentSearchCall } from './search.ts'
-export { applyWebFetchTool, formatFetchOutput, parseFetchArgs, presentFetchCall, renderBody } from './fetch.ts'
-export { htmlToMarkdown } from './html.ts'
+export { applyWebFetchTool, formatFetchOutput, parseFetchArgs, presentFetchCall } from './fetch.ts'
 
 /** Cordis plugin name used by loader diagnostics. */
 export const name = 'tool-web'
@@ -25,7 +24,14 @@ export const inject = ['tools', 'web', 'systemPrompt']
 /** Default cooperative tool-call timeout budget (ms) for the web tools. */
 export const DEFAULT_WEB_TOOL_TIMEOUT_MS = 30_000
 
-/** Plugin config: which web tools to register, the source cap, and per-tool budgets. */
+/**
+ * Default cap on one `web_fetch` output and on source characters converted
+ * synchronously. This leaves headroom above the local provider's default
+ * 100,000-character body cap while bounding custom providers and rendered output.
+ */
+export const DEFAULT_FETCH_MAX_OUTPUT_CHARS = 200_000
+
+/** Plugin config: which web tools to register, the source cap, per-tool budgets, and the fetch output cap. */
 export interface Config {
   /** Register `web_search`. Defaults to true. */
   search?: boolean
@@ -37,6 +43,8 @@ export interface Config {
   fetchTimeoutMs?: number
   /** Cooperative timeout budget (ms) for `web_search`. Defaults to 30000. */
   searchTimeoutMs?: number
+  /** Cap on source characters converted and complete `web_fetch` output characters. Defaults to 200000. */
+  fetchMaxOutputChars?: number
 }
 
 export const Config: z<Config> = z.object({
@@ -45,12 +53,13 @@ export const Config: z<Config> = z.object({
   searchMaxResults: z.number().default(WEB_SEARCH_MAX_RESULTS),
   fetchTimeoutMs: z.number().default(DEFAULT_WEB_TOOL_TIMEOUT_MS),
   searchTimeoutMs: z.number().default(DEFAULT_WEB_TOOL_TIMEOUT_MS),
+  fetchMaxOutputChars: z.number().default(DEFAULT_FETCH_MAX_OUTPUT_CHARS),
 })
 
 /** The shape after schemastery applies its defaults to every field. */
 type ResolvedConfig = Required<Config>
 
-/** The result cap must be a positive integer (it bounds a provider's source list). */
+/** Configured count, timeout, and character caps must be positive integers. */
 function assertPositiveInteger(name: string, value: number): void {
   if (!Number.isInteger(value) || value < 1) {
     throw new Error(`tool-web: ${name} must be a positive integer`)
@@ -72,6 +81,7 @@ export function apply(ctx: Context, config: Config): void {
   assertPositiveInteger('searchMaxResults', resolved.searchMaxResults)
   assertPositiveInteger('fetchTimeoutMs', resolved.fetchTimeoutMs)
   assertPositiveInteger('searchTimeoutMs', resolved.searchTimeoutMs)
+  assertPositiveInteger('fetchMaxOutputChars', resolved.fetchMaxOutputChars)
   if (resolved.search) applyWebSearchTool(ctx, resolved.searchMaxResults, resolved.searchTimeoutMs)
-  if (resolved.fetch) applyWebFetchTool(ctx, resolved.fetchTimeoutMs)
+  if (resolved.fetch) applyWebFetchTool(ctx, resolved.fetchTimeoutMs, resolved.fetchMaxOutputChars)
 }

packages/web/tool-web/src/turndown-plugin-gfm.d.ts

@@ -0,0 +1,12 @@
+/**
+ * Ambient module declaration for `@joplin/turndown-plugin-gfm`, which ships no
+ * types and has no DefinitelyTyped package. Only the composite `gfm` plugin is
+ * declared; the package's individual plugins (`tables`, `strikethrough`, …)
+ * stay undeclared until something imports them.
+ */
+declare module '@joplin/turndown-plugin-gfm' {
+  import type TurndownService from 'turndown'
+
+  /** The composite GitHub-flavored-markdown plugin (tables, strikethrough, task lists, highlighted code blocks). */
+  export const gfm: TurndownService.Plugin
+}

packages/web/tool-web/tests/tool-web.spec.ts

@@ -1,5 +1,6 @@
-import { describe, expect, it } from 'vitest'
+import { describe, expect, it, vi } from 'vitest'
 import { Context } from 'cordis'
+import TurndownService from 'turndown'
 import { CallId } from '@deepseek-ai/dsh-llm'
 import SystemPrompt from '@deepseek-ai/dsh-system-prompt'
 import ToolRegistry, { type ToolExecutionResult } from '@deepseek-ai/dsh-tools'
@@ -13,8 +14,6 @@ import {
   parseFetchArgs,
   presentSearchCall,
   presentFetchCall,
-  renderBody,
-  htmlToMarkdown,
   WEB_SEARCH_MAX_RESULTS,
 } from '@deepseek-ai/dsh-tool-web'
 
@@ -82,17 +81,29 @@ describe('search formatting', () => {
     expect(parseSearchArgs({ query: 'hi' })).toEqual({ query: 'hi' })
   })
 
+  it('falls back to the raw URL as a source label when the URL is unparseable', () => {
+    const out = formatSearchOutput({ truncated: false, sources: [{ url: 'not a url' }] })
+    expect(out).toContain('[not a url](not a url)')
+  })
+
   it('presents a search call as a search-kind card titled by the query', () => {
     expect(presentSearchCall({ query: 'find me' })).toEqual({ card: 'generic', title: 'find me', kind: 'search', rawInput: 'find me' })
   })
 })
 
 describe('fetch formatting', () => {
+  const NO_CAP = 1_000_000
+  const HEADER = 'Fetched https://a.test (HTTP 200)\n\n'
+  const renderHtml = (content: string) => formatFetchOutput({
+    url: 'https://a.test', statusCode: 200, truncated: false,
+    body: { kind: 'html', content },
+  }, NO_CAP).slice(HEADER.length)
+
   it('renders an html body to markdown text with a status header', () => {
     const out = formatFetchOutput({
       url: 'https://a.test', statusCode: 200, truncated: false,
       body: { kind: 'html', content: '<h1>Title</h1><p>Body text</p>' },
-    })
+    }, NO_CAP)
     expect(out).toContain('Fetched https://a.test (HTTP 200)')
     expect(out).toContain('# Title')
     expect(out).toContain('Body text')
@@ -102,14 +113,140 @@ describe('fetch formatting', () => {
     const out = formatFetchOutput({
       url: 'https://a.test', statusCode: 200, truncated: true,
       body: { kind: 'text', content: 'plain' },
-    })
+    }, NO_CAP)
     expect(out).toContain('plain')
     expect(out).toContain('Content truncated')
   })
 
-  it('renderBody dispatches on kind', () => {
-    expect(renderBody({ kind: 'text', content: 'x' })).toBe('x')
-    expect(renderBody({ kind: 'html', content: '<p>y</p>' })).toBe('y')
+  it('caps the complete output and notes truncation, even when markdown escaping expands the body', () => {
+    // 1,000 underscores render as 2,000 escaped characters — conversion can
+    // outgrow a provider-side body cap, so the bound applies to the output.
+    const out = formatFetchOutput({
+      url: 'https://a.test', statusCode: 200, truncated: false,
+      body: { kind: 'html', content: `<p>${'_'.repeat(1000)}</p>` },
+    }, 500)
+    expect(out.length).toBeLessThanOrEqual(500)
+    expect(out).toContain('Fetched https://a.test (HTTP 200)')
+    expect(out).toContain('\\_\\_')
+    expect(out).toContain('Content truncated')
+    // Exact and tiny caps: the complete result is bounded, header and footer included.
+    const exact = formatFetchOutput({
+      url: 'https://a.test', statusCode: 200, truncated: false,
+      body: { kind: 'text', content: 'abc' },
+    }, 'Fetched https://a.test (HTTP 200)\n\nabc'.length)
+    expect(exact).toBe('Fetched https://a.test (HTTP 200)\n\nabc')
+    const tiny = formatFetchOutput({
+      url: 'https://a.test', statusCode: 200, truncated: true,
+      body: { kind: 'text', content: 'abcdef' },
+    }, 10)
+    expect(tiny.length).toBeLessThanOrEqual(10)
+    expect(tiny).toBe('Fetched ht')
+  })
+
+  it('dispatches text and html bodies', () => {
+    expect(formatFetchOutput({
+      url: 'https://a.test', statusCode: 200, truncated: false,
+      body: { kind: 'text', content: 'x' },
+    }, NO_CAP)).toBe(`${HEADER}x`)
+    expect(renderHtml('<p>y</p>')).toBe('y')
+  })
+
+  it('converts html via turndown: entities, links, tables, nesting; drops script/style/noscript', () => {
+    expect(renderHtml('<style>.x{}</style><script>bad()</script><noscript>ns</noscript><p>Tom &amp; Jerry &copy; R&eacute;sum&eacute;</p><a href="https://a.test">link</a>'))
+      .toBe('Tom & Jerry © Résumé\n\n[link](https://a.test)')
+    expect(renderHtml('<h2>Heading</h2><ul><li>one</li><li>two</li></ul>'))
+      .toBe('## Heading\n\n-   one\n-   two')
+    expect(renderHtml('<table><tr><th>A</th><th>B</th></tr><tr><td>1</td><td>2</td></tr></table>'))
+      .toBe('| A   | B   |\n| --- | --- |\n| 1   | 2   |')
+    expect(renderHtml('<table><thead><tr><th align="left">L</th><th align="right">R</th><th style="text-align:center">C</th></tr></thead><tbody><tr><td>1</td><td>2</td><td>3</td></tr></tbody></table>'))
+      .toBe('| L   | R   | C   |\n| :--- | ---: | :---: |\n| 1   | 2   | 3   |')
+    expect(renderHtml('<p><strong>bold <em>italic</em></strong></p><blockquote><p>quoted</p></blockquote>'))
+      .toBe('**bold _italic_**\n\n> quoted')
+  })
+
+  it('does not expand numeric colspan attributes into unbounded output', () => {
+    const table = '<table><thead><tr><th colspan="1000000">A</th></tr></thead><tbody><tr><td>B</td></tr></tbody></table>'
+    expect(renderHtml(table)).toBe('| A   |\n| --- |\n| B   |')
+  })
+
+  it('passes deeply nested html through raw without attempting conversion', () => {
+    // Unclosed-tag nesting makes the synchronous conversion superlinear
+    // (seconds at 20k levels, during which the cooperative timeout cannot
+    // fire), so the depth preflight skips conversion entirely; this must
+    // return fast, not merely not-throw.
+    const depth = 20_000
+    const pathological = '<div>'.repeat(depth) + 'x' + '</div>'.repeat(depth)
+    const started = Date.now()
+    expect(formatFetchOutput({
+      url: 'https://a.test', statusCode: 200, truncated: false,
+      body: { kind: 'html', content: pathological },
+    }, NO_CAP)).toBe(`${HEADER}${pathological}`)
+    expect(Date.now() - started).toBeLessThan(2_000)
+  })
+
+  it('comments and mismatched closing tags cannot hide deep nesting from the preflight', () => {
+    const pathological = '<div><!-- </div> --></span>'.repeat(600) + 'x'
+    expect(formatFetchOutput({
+      url: 'https://a.test', statusCode: 200, truncated: false,
+      body: { kind: 'html', content: pathological },
+    }, NO_CAP)).toBe(`${HEADER}${pathological}`)
+    const abruptlyClosedComments = '<div><!-->'.repeat(600) + 'x'
+    expect(formatFetchOutput({
+      url: 'https://a.test', statusCode: 200, truncated: false,
+      body: { kind: 'html', content: abruptlyClosedComments },
+    }, NO_CAP)).toBe(`${HEADER}${abruptlyClosedComments}`)
+  })
+
+  it('the preflight accepts ordinary closed, void, self-closing, quoted, and raw-text markup', () => {
+    const paragraphs = '<p title=\'>\'>x<br   ><img src="x"><input/></p>'.repeat(600)
+    const script = `<script>const invalid = '</scriptx>'; const template = '${'<div>'.repeat(600)}'</script >`
+    expect(renderHtml(`<!doctype html><?pi><1bad>${paragraphs}${script}`))
+      .not.toContain('<p')
+    expect(renderHtml('plain text')).toBe('plain text')
+    expect(renderHtml('<p>x</p><!-- unfinished')).toBe('x')
+    expect(renderHtml('<script>unclosed')).toBe('')
+    expect(renderHtml('<script>closed by slash</script/>')).toBe('')
+    expect(renderHtml('<script>closed at end</script')).toBe('')
+  })
+
+  it('scans malformed unterminated tags in bounded time', () => {
+    const malformed = '<a'.repeat(100_000)
+    const started = Date.now()
+    const out = formatFetchOutput({
+      url: 'https://a.test', statusCode: 200, truncated: false,
+      body: { kind: 'html', content: malformed },
+    }, 200_000)
+    expect(out.length).toBeLessThanOrEqual(200_000)
+    expect(Date.now() - started).toBeLessThan(2_000)
+  })
+
+  it('falls back to the raw html when turndown throws despite a shallow depth scan', () => {
+    const spy = vi.spyOn(TurndownService.prototype, 'turndown').mockImplementation(() => {
+      throw new RangeError('Maximum call stack size exceeded')
+    })
+    try {
+      expect(formatFetchOutput({
+        url: 'https://a.test', statusCode: 200, truncated: false,
+        body: { kind: 'html', content: '<p>x</p>' },
+      }, NO_CAP)).toBe(`${HEADER}<p>x</p>`)
+    } finally {
+      spy.mockRestore()
+    }
+  })
+
+  it('bounds source conversion work before rendering a custom provider body', () => {
+    const spy = vi.spyOn(TurndownService.prototype, 'turndown').mockReturnValue('converted')
+    try {
+      const out = formatFetchOutput({
+        url: 'https://a.test', statusCode: 200, truncated: false,
+        body: { kind: 'html', content: `<p>${'x'.repeat(10_000)}</p>` },
+      }, 500)
+      expect(spy).toHaveBeenCalledWith(`<p>${'x'.repeat(497)}`)
+      expect(out.length).toBeLessThanOrEqual(500)
+      expect(out).toContain('Content truncated')
+    } finally {
+      spy.mockRestore()
+    }
   })
 
   it('validates url (non-empty), no timeout parameter', () => {
@@ -122,46 +259,6 @@ describe('fetch formatting', () => {
   })
 })
 
-describe('htmlToMarkdown', () => {
-  it('drops scripts/styles, keeps text, decodes entities, converts links', () => {
-    const md = htmlToMarkdown('<style>.x{}</style><script>bad()</script><p>Tom &amp; Jerry</p><a href="https://a.test">link</a>')
-    expect(md).not.toContain('bad()')
-    expect(md).not.toContain('.x{}')
-    expect(md).toContain('Tom & Jerry')
-    expect(md).toContain('[link](https://a.test)')
-  })
-
-  it('decodes numeric entities and collapses whitespace', () => {
-    expect(htmlToMarkdown('<p>a&#39;b</p>')).toBe("a'b")
-    expect(htmlToMarkdown('<div>x</div>\n\n\n<div>y</div>')).toBe('x\n\ny')
-  })
-
-  it('decodes hex entities and named entities, and leaves unknown/out-of-range ones intact', () => {
-    expect(htmlToMarkdown('<p>&#x41;&#X42;</p>')).toBe('AB')
-    expect(htmlToMarkdown('<p>&copy; &mdash;</p>')).toBe('© —')
-    expect(htmlToMarkdown('<p>&notareal;</p>')).toBe('&notareal;')
-    // An out-of-range code point keeps the original entity text (fromCodePoint fallback).
-    expect(htmlToMarkdown('<p>&#x110000;</p>')).toBe('&#x110000;')
-    expect(htmlToMarkdown('<p>&#1114112;</p>')).toBe('&#1114112;')
-  })
-
-  it('renders a link with an empty label as its bare href', () => {
-    expect(htmlToMarkdown('<a href="https://a.test"></a>')).toBe('https://a.test')
-  })
-
-  it('converts headings and list items to markdown', () => {
-    expect(htmlToMarkdown('<h2>Heading</h2><p>after</p>')).toContain('## Heading')
-    const list = htmlToMarkdown('<ul><li>one</li><li>two</li></ul>')
-    expect(list).toContain('- one')
-    expect(list).toContain('- two')
-  })
-
-  it('falls back to the raw URL as a source label when the URL is unparseable', () => {
-    const out = formatSearchOutput({ truncated: false, sources: [{ url: 'not a url' }] })
-    expect(out).toContain('[not a url](not a url)')
-  })
-})
-
 describe('tool-web registration', () => {
   it('registers both tools by default', async () => {
     const { fiber, ctx } = await mountTools()
@@ -395,3 +492,35 @@ describe('tool-call timeout budget is plugin config', () => {
       .rejects.toThrow(new RegExp(`tool-web: ${key} must be a positive integer`))
   })
 })
+
+describe('fetchMaxOutputChars is plugin config', () => {
+  it('bounds the rendered output of the registered web_fetch tool', async () => {
+    const fetchProvider = {
+      id: 'stub-fetch',
+      available: () => available,
+      fetch: (request: { url: string }) => Promise.resolve({
+        url: request.url,
+        statusCode: 200,
+        body: { kind: 'html' as const, content: `<p>${'_'.repeat(1_000)}</p>` },
+        truncated: false,
+      }),
+    }
+    const { fiber, call } = await mountTools({
+      config: { fetchMaxOutputChars: 100 },
+      webConfig: { fetchProvider: 'stub-fetch' },
+      fetchProvider,
+    })
+    const out = await call('web_fetch', { url: 'https://a.test' })
+    expect(out.content.map(block => block.type === 'text' ? block.text : '').join('')).toHaveLength(100)
+    await fiber.dispose()
+  })
+
+  it.each([0, -1, 1.5])('rejects an invalid fetchMaxOutputChars value %s at load', async (value) => {
+    const ctx = new Context()
+    await ctx.plugin(SystemPrompt)
+    await ctx.plugin(ToolRegistry)
+    await ctx.plugin(WebService, {})
+    await expect(ctx.plugin(ToolWeb, { fetchMaxOutputChars: value }))
+      .rejects.toThrow(/tool-web: fetchMaxOutputChars must be a positive integer/)
+  })
+})