feat(tool-web): replace the regex HTML-to-markdown converter with turndown
Implements the turndown Agent Note from the NIH dependency audit (full variant, not the minimal entities-only fallback): dsh-tool-web's fetch rendering now converts HTML through turndown + @joplin/turndown-plugin-gfm (atx headings, fenced code, dash bullets, GFM tables/strikethrough) over the real domino DOM, with script/style/noscript removed wholesale. The hand-rolled ~86-line regex converter html.ts and its entity tables are deleted; renderBody wraps the conversion in try/catch falling back to the raw HTML body, because turndown's recursive DOM walk overflows with a RangeError on pathological nesting (measured: 4k levels on the main thread, 8k in a worker) where the regex version could never throw. Closure weight, measured: tool-web IS in the single-exe runtime closure, and the exe asset globs would pack ~7.9 MB of the three new packages — but ~6 MB of that is domino's test corpus, with runtime lib/ at ~550 KB against a ~174 MB artifact (<0.5% either way), so the swap wins. Per testing policy the previously-missing keyless web_fetch snapshot ships in the same change: the acp-agent `web-fetch` scenario boots a new web.cordis.yml overlay (web seam + real dsh-web-fetch-local provider + tool-web fetch-only + a loopback HTTP fixture server on a fixed port serving deterministic HTML with entities, a GFM table, and nesting), so recording and keyless replay both drive the real HTTP fetch and real conversion end to end; the scenario pins the new `web` header class. The Agent Note moves proposed -> implemented and is rewritten per the lifecycle contract (Decision/Consequences/Testing, closure verdict and alternatives recorded); tool-web and acp-agent READMEs updated in both languages and pairs re-recorded.
This commit is contained in:
Tianyi Cui
@@ -6,12 +6,29 @@
|
||||
*/
|
||||
|
||||
import type { Context } from 'cordis'
|
||||
import TurndownService from 'turndown'
|
||||
import { gfm } from '@joplin/turndown-plugin-gfm'
|
||||
import { defineTool } from '@deepseek-ai/dsh-tools'
|
||||
import type { GenericCallView } from '@deepseek-ai/dsh-tools'
|
||||
import type { WebFetchBody, WebFetchResult } from '@deepseek-ai/dsh-web'
|
||||
import { assertNever } from '@deepseek-ai/dsh-llm'
|
||||
import type {} from '@deepseek-ai/dsh-system-prompt'
|
||||
import { htmlToMarkdown } from './html.ts'
|
||||
|
||||
/**
|
||||
* The shared HTML→markdown converter: turndown over its bundled domino DOM,
|
||||
* with GitHub-flavored tables/strikethrough (`@joplin/turndown-plugin-gfm`).
|
||||
* The style options are fixed model-facing presentation (matching the repo's
|
||||
* markdown conventions), not deployment tunables. `remove` drops non-content
|
||||
* elements wholesale — turndown's default keeps their text. The instance is
|
||||
* stateless across `turndown()` calls and safe to share.
|
||||
*/
|
||||
const turndown = new TurndownService({
|
||||
headingStyle: 'atx',
|
||||
codeBlockStyle: 'fenced',
|
||||
bulletListMarker: '-',
|
||||
})
|
||||
turndown.use(gfm)
|
||||
turndown.remove(['script', 'style', 'noscript'])
|
||||
|
||||
/**
|
||||
* Validate value constraints the schema DSL can't express: a non-blank `url`.
|
||||
@@ -30,14 +47,23 @@ export function parseFetchArgs(args: { url: string }): { url: string } {
|
||||
/**
|
||||
* Render a fetched body to model-facing markdown text.
|
||||
*
|
||||
* @param body - the decoded body; `html` is converted via
|
||||
* {@link htmlToMarkdown}, `text` passes through verbatim.
|
||||
* @param body - the decoded body; `html` is converted via turndown, `text`
|
||||
* passes through verbatim. When turndown throws (deeply pathological HTML
|
||||
* overflows its recursive DOM walk), the raw HTML passes through instead —
|
||||
* a degraded page beats an error for a body the provider already decoded.
|
||||
* @returns the text for the tool's output block.
|
||||
*/
|
||||
export function renderBody(body: WebFetchBody): string {
|
||||
switch (body.kind) {
|
||||
case 'html':
|
||||
return htmlToMarkdown(body.content)
|
||||
try {
|
||||
return turndown.turndown(body.content)
|
||||
} catch {
|
||||
// turndown's DOM walk recurses per element; pathological nesting (a
|
||||
// few thousand levels) throws RangeError. Provider errors stay
|
||||
// structured WebErrors upstream; conversion failure downgrades to raw HTML.
|
||||
return body.content
|
||||
}
|
||||
case 'text':
|
||||
return body.content
|
||||
/* v8 ignore next 2 -- WebFetchBody is a closed union; this arm is unreachable and only makes adding a kind a compile error. */
|
||||
|
||||
@@ -1,86 +0,0 @@
|
||||
/**
|
||||
* Minimal dependency-free HTML-to-readable-text conversion for `web_fetch`, not a full parser. It
|
||||
* removes non-content elements and tags, decodes common entities, collapses whitespace, and keeps
|
||||
* basic headings, lists, and links. A richer converter can replace it without changing the seam or
|
||||
* tool schema.
|
||||
* @module @deepseek-ai/dsh-tool-web/html
|
||||
*/
|
||||
|
||||
/** Decode the handful of HTML entities common in textual content. */
|
||||
function decodeEntities(text: string): string {
|
||||
return text
|
||||
.replace(/&(#[xX][0-9a-fA-F]+|#[0-9]+|[a-zA-Z]+);/g, (match, entity: string) => {
|
||||
if (entity.startsWith('#x') || entity.startsWith('#X')) {
|
||||
const code = Number.parseInt(entity.slice(2), 16)
|
||||
return safeFromCodePoint(code, match)
|
||||
}
|
||||
if (entity.startsWith('#')) {
|
||||
const code = Number.parseInt(entity.slice(1), 10)
|
||||
return safeFromCodePoint(code, match)
|
||||
}
|
||||
return NAMED_ENTITIES[entity] ?? match
|
||||
})
|
||||
}
|
||||
|
||||
const NAMED_ENTITIES: Record<string, string> = {
|
||||
amp: '&', lt: '<', gt: '>', quot: '"', apos: "'", nbsp: ' ',
|
||||
copy: '©', reg: '®', trade: '™', hellip: '…', mdash: '—', ndash: '–',
|
||||
}
|
||||
|
||||
function safeFromCodePoint(code: number, fallback: string): string {
|
||||
try {
|
||||
return String.fromCodePoint(code)
|
||||
} catch {
|
||||
// An out-of-range code point (RangeError) is the only failure here; keep the
|
||||
// original entity text rather than throwing out of pure presentation.
|
||||
return fallback
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert an HTML document to a readable markdown-ish text approximation.
|
||||
* Best-effort and lossy by design — fidelity is the job of a future heavier
|
||||
* converter, not this fallback.
|
||||
*
|
||||
* @param html - the raw HTML source.
|
||||
* @returns plain text with markdown headings, list bullets, and links;
|
||||
* whitespace collapsed to at most one blank line and trimmed.
|
||||
*/
|
||||
export function htmlToMarkdown(html: string): string {
|
||||
let text = html
|
||||
// Drop non-content elements entirely (including their contents).
|
||||
.replace(/<script\b[^>]*>[\s\S]*?<\/script>/gi, '')
|
||||
.replace(/<style\b[^>]*>[\s\S]*?<\/style>/gi, '')
|
||||
.replace(/<noscript\b[^>]*>[\s\S]*?<\/noscript>/gi, '')
|
||||
.replace(/<!--[\s\S]*?-->/g, '')
|
||||
|
||||
// Convert links to markdown before stripping tags.
|
||||
text = text.replace(/<a\b[^>]*\bhref\s*=\s*["']([^"']*)["'][^>]*>([\s\S]*?)<\/a>/gi, (_match, href: string, label: string) => {
|
||||
const cleanLabel = label.replace(/<[^>]+>/g, '').trim()
|
||||
return cleanLabel.length > 0 ? `[${cleanLabel}](${href})` : href
|
||||
})
|
||||
|
||||
// Headings → markdown hashes.
|
||||
text = text.replace(/<h([1-6])\b[^>]*>([\s\S]*?)<\/h\1>/gi, (_match, level: string, body: string) => {
|
||||
const hashes = '#'.repeat(Number(level))
|
||||
return `\n\n${hashes} ${body.replace(/<[^>]+>/g, '').trim()}\n\n`
|
||||
})
|
||||
|
||||
// List items → bullets.
|
||||
text = text.replace(/<li\b[^>]*>([\s\S]*?)<\/li>/gi, (_match, body: string) => `\n- ${body.replace(/<[^>]+>/g, '').trim()}`)
|
||||
|
||||
// Block-level breaks become paragraph breaks.
|
||||
text = text
|
||||
.replace(/<\/(p|div|section|article|header|footer|tr|table|ul|ol|blockquote)>/gi, '\n\n')
|
||||
.replace(/<br\s*\/?>/gi, '\n')
|
||||
|
||||
// Drop all remaining tags, decode entities, collapse whitespace.
|
||||
text = text.replace(/<[^>]+>/g, '')
|
||||
text = decodeEntities(text)
|
||||
text = text
|
||||
.replace(/[ \t\f\v]+/g, ' ')
|
||||
.replace(/ *\n */g, '\n')
|
||||
.replace(/\n{3,}/g, '\n\n')
|
||||
.trim()
|
||||
return text
|
||||
}
|
||||
@@ -14,7 +14,6 @@ import { applyWebFetchTool } from './fetch.ts'
|
||||
|
||||
export { WEB_SEARCH_MAX_RESULTS, applyWebSearchTool, formatSearchOutput, parseSearchArgs, presentSearchCall } from './search.ts'
|
||||
export { applyWebFetchTool, formatFetchOutput, parseFetchArgs, presentFetchCall, renderBody } from './fetch.ts'
|
||||
export { htmlToMarkdown } from './html.ts'
|
||||
|
||||
/** Cordis plugin name used by loader diagnostics. */
|
||||
export const name = 'tool-web'
|
||||
|
||||
12
packages/web/tool-web/src/turndown-plugin-gfm.d.ts
vendored
Normal file
12
packages/web/tool-web/src/turndown-plugin-gfm.d.ts
vendored
Normal file
@@ -0,0 +1,12 @@
|
||||
/**
|
||||
* Ambient module declaration for `@joplin/turndown-plugin-gfm`, which ships no
|
||||
* types and has no DefinitelyTyped package. Only the composite `gfm` plugin is
|
||||
* declared; the package's individual plugins (`tables`, `strikethrough`, …)
|
||||
* stay undeclared until something imports them.
|
||||
*/
|
||||
declare module '@joplin/turndown-plugin-gfm' {
|
||||
import type TurndownService from 'turndown'
|
||||
|
||||
/** The composite GitHub-flavored-markdown plugin (tables, strikethrough, task lists, highlighted code blocks). */
|
||||
export const gfm: TurndownService.Plugin
|
||||
}
|
||||
Reference in New Issue
Block a user