feat(tool-web): replace the regex HTML-to-markdown converter with turndown
Implements the turndown Agent Note from the NIH dependency audit (full variant, not the minimal entities-only fallback): dsh-tool-web's fetch rendering now converts HTML through turndown + @joplin/turndown-plugin-gfm (atx headings, fenced code, dash bullets, GFM tables/strikethrough) over the real domino DOM, with script/style/noscript removed wholesale. The hand-rolled ~86-line regex converter html.ts and its entity tables are deleted; renderBody wraps the conversion in try/catch falling back to the raw HTML body, because turndown's recursive DOM walk overflows with a RangeError on pathological nesting (measured: 4k levels on the main thread, 8k in a worker) where the regex version could never throw. Closure weight, measured: tool-web IS in the single-exe runtime closure, and the exe asset globs would pack ~7.9 MB of the three new packages — but ~6 MB of that is domino's test corpus, with runtime lib/ at ~550 KB against a ~174 MB artifact (<0.5% either way), so the swap wins. Per testing policy the previously-missing keyless web_fetch snapshot ships in the same change: the acp-agent `web-fetch` scenario boots a new web.cordis.yml overlay (web seam + real dsh-web-fetch-local provider + tool-web fetch-only + a loopback HTTP fixture server on a fixed port serving deterministic HTML with entities, a GFM table, and nesting), so recording and keyless replay both drive the real HTTP fetch and real conversion end to end; the scenario pins the new `web` header class. The Agent Note moves proposed -> implemented and is rewritten per the lifecycle contract (Decision/Consequences/Testing, closure verdict and alternatives recorded); tool-web and acp-agent READMEs updated in both languages and pairs re-recorded.
This commit is contained in:
@@ -14,7 +14,6 @@ import {
|
||||
presentSearchCall,
|
||||
presentFetchCall,
|
||||
renderBody,
|
||||
htmlToMarkdown,
|
||||
WEB_SEARCH_MAX_RESULTS,
|
||||
} from '@deepseek-ai/dsh-tool-web'
|
||||
|
||||
@@ -82,6 +81,11 @@ describe('search formatting', () => {
|
||||
expect(parseSearchArgs({ query: 'hi' })).toEqual({ query: 'hi' })
|
||||
})
|
||||
|
||||
it('falls back to the raw URL as a source label when the URL is unparseable', () => {
|
||||
const out = formatSearchOutput({ truncated: false, sources: [{ url: 'not a url' }] })
|
||||
expect(out).toContain('[not a url](not a url)')
|
||||
})
|
||||
|
||||
it('presents a search call as a search-kind card titled by the query', () => {
|
||||
expect(presentSearchCall({ query: 'find me' })).toEqual({ card: 'generic', title: 'find me', kind: 'search', rawInput: 'find me' })
|
||||
})
|
||||
@@ -112,6 +116,29 @@ describe('fetch formatting', () => {
|
||||
expect(renderBody({ kind: 'html', content: '<p>y</p>' })).toBe('y')
|
||||
})
|
||||
|
||||
it('converts html via turndown: entities, links, tables, nesting; drops script/style/noscript', () => {
|
||||
expect(renderBody({
|
||||
kind: 'html',
|
||||
content: '<style>.x{}</style><script>bad()</script><noscript>ns</noscript><p>Tom & Jerry © Résumé</p><a href="https://a.test">link</a>',
|
||||
})).toBe('Tom & Jerry © Résumé\n\n[link](https://a.test)')
|
||||
expect(renderBody({ kind: 'html', content: '<h2>Heading</h2><ul><li>one</li><li>two</li></ul>' }))
|
||||
.toBe('## Heading\n\n- one\n- two')
|
||||
expect(renderBody({ kind: 'html', content: '<table><tr><th>A</th><th>B</th></tr><tr><td>1</td><td>2</td></tr></table>' }))
|
||||
.toBe('| A | B |\n| --- | --- |\n| 1 | 2 |')
|
||||
expect(renderBody({ kind: 'html', content: '<p><strong>bold <em>italic</em></strong></p><blockquote><p>quoted</p></blockquote>' }))
|
||||
.toBe('**bold _italic_**\n\n> quoted')
|
||||
})
|
||||
|
||||
it('falls back to the raw html body when turndown throws on pathological nesting', { timeout: 60_000 }, () => {
|
||||
// Nesting past V8's default stack overflows turndown/domino's recursive
|
||||
// walk with a RangeError (measured: 4k levels throw on the main thread,
|
||||
// 8k in a worker); 20k adds margin over either stack size. The raw body
|
||||
// must pass through instead of throwing.
|
||||
const depth = 20_000
|
||||
const pathological = '<div>'.repeat(depth) + 'x' + '</div>'.repeat(depth)
|
||||
expect(renderBody({ kind: 'html', content: pathological })).toBe(pathological)
|
||||
})
|
||||
|
||||
it('validates url (non-empty), no timeout parameter', () => {
|
||||
expect(() => parseFetchArgs({ url: ' ' })).toThrow('non-empty')
|
||||
expect(parseFetchArgs({ url: 'https://a.test' })).toEqual({ url: 'https://a.test' })
|
||||
@@ -122,46 +149,6 @@ describe('fetch formatting', () => {
|
||||
})
|
||||
})
|
||||
|
||||
describe('htmlToMarkdown', () => {
|
||||
it('drops scripts/styles, keeps text, decodes entities, converts links', () => {
|
||||
const md = htmlToMarkdown('<style>.x{}</style><script>bad()</script><p>Tom & Jerry</p><a href="https://a.test">link</a>')
|
||||
expect(md).not.toContain('bad()')
|
||||
expect(md).not.toContain('.x{}')
|
||||
expect(md).toContain('Tom & Jerry')
|
||||
expect(md).toContain('[link](https://a.test)')
|
||||
})
|
||||
|
||||
it('decodes numeric entities and collapses whitespace', () => {
|
||||
expect(htmlToMarkdown('<p>a'b</p>')).toBe("a'b")
|
||||
expect(htmlToMarkdown('<div>x</div>\n\n\n<div>y</div>')).toBe('x\n\ny')
|
||||
})
|
||||
|
||||
it('decodes hex entities and named entities, and leaves unknown/out-of-range ones intact', () => {
|
||||
expect(htmlToMarkdown('<p>AB</p>')).toBe('AB')
|
||||
expect(htmlToMarkdown('<p>© —</p>')).toBe('© —')
|
||||
expect(htmlToMarkdown('<p>¬areal;</p>')).toBe('¬areal;')
|
||||
// An out-of-range code point keeps the original entity text (fromCodePoint fallback).
|
||||
expect(htmlToMarkdown('<p>�</p>')).toBe('�')
|
||||
expect(htmlToMarkdown('<p>�</p>')).toBe('�')
|
||||
})
|
||||
|
||||
it('renders a link with an empty label as its bare href', () => {
|
||||
expect(htmlToMarkdown('<a href="https://a.test"></a>')).toBe('https://a.test')
|
||||
})
|
||||
|
||||
it('converts headings and list items to markdown', () => {
|
||||
expect(htmlToMarkdown('<h2>Heading</h2><p>after</p>')).toContain('## Heading')
|
||||
const list = htmlToMarkdown('<ul><li>one</li><li>two</li></ul>')
|
||||
expect(list).toContain('- one')
|
||||
expect(list).toContain('- two')
|
||||
})
|
||||
|
||||
it('falls back to the raw URL as a source label when the URL is unparseable', () => {
|
||||
const out = formatSearchOutput({ truncated: false, sources: [{ url: 'not a url' }] })
|
||||
expect(out).toContain('[not a url](not a url)')
|
||||
})
|
||||
})
|
||||
|
||||
describe('tool-web registration', () => {
|
||||
it('registers both tools by default', async () => {
|
||||
const { fiber, ctx } = await mountTools()
|
||||
|
||||
Reference in New Issue
Block a user