From 7c90422f345688ca84c65acdd86e24decc55782b Mon Sep 17 00:00:00 2001 From: fz Date: Wed, 5 Aug 2026 15:06:38 +0800 Subject: [PATCH] fix(ui): harden TeX delimiter parsing --- ...026-07-23-web-assistant-markdown.i18n.yaml | 4 +- .../2026-07-23-web-assistant-markdown.md | 2 +- .../2026-07-23-web-assistant-markdown.zh.md | 2 +- .../client/ui-primitives/README.i18n.yaml | 4 +- packages/client/ui-primitives/README.md | 2 +- packages/client/ui-primitives/README.zh.md | 2 +- .../src/markdown/remarkMathCompatibility.ts | 84 +++++++++++++++++-- .../ui-primitives/tests/markdown.spec.tsx | 35 ++++++++ 8 files changed, 118 insertions(+), 17 deletions(-) diff --git a/.agents/notes/implemented/feature/2026-07-23-web-assistant-markdown.i18n.yaml b/.agents/notes/implemented/feature/2026-07-23-web-assistant-markdown.i18n.yaml index f8326927b9..2a56348913 100644 --- a/.agents/notes/implemented/feature/2026-07-23-web-assistant-markdown.i18n.yaml +++ b/.agents/notes/implemented/feature/2026-07-23-web-assistant-markdown.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write .agents/notes/implemented/feature/2026-07-23-web-assistant-markdown.md -2026-07-23-web-assistant-markdown.md: 347ab223c970a345ca5d7abea0edc0bf9cce6324 -2026-07-23-web-assistant-markdown.zh.md: 6898dd009373d893d3d049b83963234461f04959 +2026-07-23-web-assistant-markdown.md: 8a8778351911bcb3448366c718aa124c4a89de58 +2026-07-23-web-assistant-markdown.zh.md: 2ac024e24ff95b4eb296112562c93f343b832187 diff --git a/.agents/notes/implemented/feature/2026-07-23-web-assistant-markdown.md b/.agents/notes/implemented/feature/2026-07-23-web-assistant-markdown.md index 347ab223c9..8a87783519 100644 --- a/.agents/notes/implemented/feature/2026-07-23-web-assistant-markdown.md +++ b/.agents/notes/implemented/feature/2026-07-23-web-assistant-markdown.md @@ -14,7 +14,7 @@ The Web conversation preserves assistant Markdown source through session events, `MarkdownText` uses `react-markdown` with `remark-gfm` to build React elements from an AST. It covers CommonMark blocks plus GFM tables, task lists, strikethrough, and autolinks without raw-HTML parsing. Fenced code routes through the shared `CodeBlock`, which highlights registered grammars with the client's shiki singleton (`--shiki-*` tokens) and falls back to plain monospace otherwise. While a turn streams, fences stay on the plain arm so growing fences are not retokenized every chunk. -Visual spacing, tables, links, blockquotes, inline code, and code-block chrome follow deepsuite `@deepseek/md` (`markdown.css` / `code-block.css`) and the same `--dsw-alias-markdown-*`, `--dsw-font-markdown-*`, `--dsw-alias-border-l*`, and `--dsw-alias-label-*` tokens. Links use `--dsw-alias-state-business-primary` (deepsuite's sheet uses `--dsw-alias-brand-text`, which is blue only under newDesign; design-platform keeps brand-text near-black and is not retuned here). `CodeBlock` ships a language banner and a copy control (`复制` / `复制成功`). Finalized text renders KaTeX through `remark-math` and `rehype-katex`; `remarkMathCompatibility` maps `\\(...\\)`, `\\[...\\]`, and block-level same-line `$$...$$` to the same standard math AST nodes. This is a narrow parser compatibility layer, not a regex rewrite or malformed-model-output repair. Streaming stays literal until finalization so incomplete formulae do not flash errors. Citation pills, heading anchors, the thinking-small markdown variant, and custom □/☑ task markers remain out of scope; GFM task lists keep native checkboxes. +Visual spacing, tables, links, blockquotes, inline code, and code-block chrome follow deepsuite `@deepseek/md` (`markdown.css` / `code-block.css`) and the same `--dsw-alias-markdown-*`, `--dsw-font-markdown-*`, `--dsw-alias-border-l*`, and `--dsw-alias-label-*` tokens. Links use `--dsw-alias-state-business-primary` (deepsuite's sheet uses `--dsw-alias-brand-text`, which is blue only under newDesign; design-platform keeps brand-text near-black and is not retuned here). `CodeBlock` ships a language banner and a copy control (`复制` / `复制成功`). Finalized text renders KaTeX through `remark-math` and `rehype-katex`; `remarkMathCompatibility` maps `\(...\)`, `\[...\]`, and block-level same-line `$$...$$` to the same standard math AST nodes. This is a narrow parser compatibility layer, not a regex rewrite or malformed-model-output repair. Streaming stays literal until finalization so incomplete formulae do not flash errors. Citation pills, heading anchors, the thinking-small markdown variant, and custom □/☑ task markers remain out of scope; GFM task lists keep native checkboxes. The dependency is explicit in `ui-primitives`; because that pure library is seeded by the Web shell, the parser and highlighter are part of the initial browser bundle. diff --git a/.agents/notes/implemented/feature/2026-07-23-web-assistant-markdown.zh.md b/.agents/notes/implemented/feature/2026-07-23-web-assistant-markdown.zh.md index 6898dd0093..2ac024e24f 100644 --- a/.agents/notes/implemented/feature/2026-07-23-web-assistant-markdown.zh.md +++ b/.agents/notes/implemented/feature/2026-07-23-web-assistant-markdown.zh.md @@ -14,7 +14,7 @@ Web 对话通过会话事件、历史回放与流式累积保留 assistant Markd `MarkdownText` 使用 `react-markdown` 与 `remark-gfm`，从 AST 构建 React 元素。它覆盖 CommonMark 块，以及 GFM 表格、任务列表、删除线与自动链接，且不解析原始 HTML。围栏代码经共享的 `CodeBlock` 路由；该组件用客户端的 shiki 单例（`--shiki-*` token）高亮已注册语法，否则回退为纯等宽文本。轮次流式输出期间，围栏停留在纯文本分支，以免每收到一个分片就对增长中的围栏重新分词。 -视觉间距、表格、链接、引用块、行内代码与代码块外框遵循 deepsuite `@deepseek/md`（`markdown.css` / `code-block.css`），并使用同一套 `--dsw-alias-markdown-*`、`--dsw-font-markdown-*`、`--dsw-alias-border-l*` 与 `--dsw-alias-label-*` token。链接使用 `--dsw-alias-state-business-primary`（deepsuite 的样式表使用 `--dsw-alias-brand-text`，仅在 newDesign 下为蓝色；design-platform 将 brand-text 保持为近黑色，此处不做重新调色）。`CodeBlock` 提供语言横幅与复制控件（`复制` / `复制成功`）。已完成的文本通过 `remark-math` 和 `rehype-katex` 渲染 KaTeX；`remarkMathCompatibility` 将 `\\(...\\)`、`\\[...\\]` 和块级同一行 `$$...$$` 映射为同一套标准数学 AST 节点。这是一层小范围的解析器兼容层，不是正则重写，也不修复格式错误的模型输出。流式输出在完成前保持按字面渲染，避免不完整公式闪现错误。引用胶囊、标题锚点、thinking-small markdown 变体，以及自定义 □/☑ 任务标记仍不在范围内；GFM 任务列表继续使用原生复选框。 +视觉间距、表格、链接、引用块、行内代码与代码块外框遵循 deepsuite `@deepseek/md`（`markdown.css` / `code-block.css`），并使用同一套 `--dsw-alias-markdown-*`、`--dsw-font-markdown-*`、`--dsw-alias-border-l*` 与 `--dsw-alias-label-*` token。链接使用 `--dsw-alias-state-business-primary`（deepsuite 的样式表使用 `--dsw-alias-brand-text`，仅在 newDesign 下为蓝色；design-platform 将 brand-text 保持为近黑色，此处不做重新调色）。`CodeBlock` 提供语言横幅与复制控件（`复制` / `复制成功`）。已完成的文本通过 `remark-math` 和 `rehype-katex` 渲染 KaTeX；`remarkMathCompatibility` 将 `\(...\)`、`\[...\]` 和块级同一行 `$$...$$` 映射为同一套标准数学 AST 节点。这是一层小范围的解析器兼容层，不是正则重写，也不修复格式错误的模型输出。流式输出在完成前保持按字面渲染，避免不完整公式闪现错误。引用胶囊、标题锚点、thinking-small markdown 变体，以及自定义 □/☑ 任务标记仍不在范围内；GFM 任务列表继续使用原生复选框。 该依赖在 `ui-primitives` 中显式声明；由于这一纯库由 Web shell 预置，解析器与高亮器会成为初始浏览器 bundle 的一部分。 diff --git a/packages/client/ui-primitives/README.i18n.yaml b/packages/client/ui-primitives/README.i18n.yaml index 8c597efd5a..f6787f8fff 100644 --- a/packages/client/ui-primitives/README.i18n.yaml +++ b/packages/client/ui-primitives/README.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write packages/client/ui-primitives/README.md -README.md: 00e9560f43c83e1edc61c185a4fc562c6c923e8b -README.zh.md: 21226ab211106b7722139828762605cb71a4b498 +README.md: 3286e6da6020fa31ca13658e091a5b12dc57d28b +README.zh.md: e3d8b6ac869d33850945db088472d9c1958ab0b2 diff --git a/packages/client/ui-primitives/README.md b/packages/client/ui-primitives/README.md index 00e9560f43..3286e6da60 100644 --- a/packages/client/ui-primitives/README.md +++ b/packages/client/ui-primitives/README.md @@ -10,7 +10,7 @@ Pure React atoms (zero cordis): StateDot, ic_ds_* icons, Button/Pill/Menu/Modal/ ## Markdown rendering -`MarkdownText` renders GFM and `$…$` / `$$…$$` TeX math from untrusted assistant output through React elements, with math typeset by KaTeX and trusted commands disabled. It omits raw HTML, neutralizes relative and non-HTTP(S)/mailto links, opens HTTP(S) links with safe external-link attributes, and renders absolute HTTP(S) images without a referrer; relative paths, absolute local paths, `file:` URLs, and unsupported schemes retain their alt text. `MessageText` remains the literal-text primitive for user-authored content. `extractMarkdownPlainText` removes Markdown presentation markup for compact labels while preserving raw HTML as literal text. Element spacing, responsive images, tables, links, and inline code use the same `--dsw-alias-markdown-*` / `--dsw-font-markdown-*` tokens as deepsuite `@deepseek/md`. Fenced blocks render through `CodeBlock` (language banner, copy control, shiki for the registered grammars). +`MarkdownText` renders GFM and `$…$`, `$$…$$`, `\(…\)`, and `\[…\]` TeX math from untrusted assistant output through React elements, with math typeset by KaTeX and trusted commands disabled; block-level same-line `$$…$$` is display math, including `\tag{}`. It omits raw HTML, neutralizes relative and non-HTTP(S)/mailto links, opens HTTP(S) links with safe external-link attributes, and renders absolute HTTP(S) images without a referrer; relative paths, absolute local paths, `file:` URLs, and unsupported schemes retain their alt text. `MessageText` remains the literal-text primitive for user-authored content. `extractMarkdownPlainText` removes Markdown presentation markup for compact labels while preserving raw HTML as literal text. Element spacing, responsive images, tables, links, and inline code use the same `--dsw-alias-markdown-*` / `--dsw-font-markdown-*` tokens as deepsuite `@deepseek/md`. Fenced blocks render through `CodeBlock` (language banner, copy control, shiki for the registered grammars). ## Terminal output diff --git a/packages/client/ui-primitives/README.zh.md b/packages/client/ui-primitives/README.zh.md index 21226ab211..e3d8b6ac86 100644 --- a/packages/client/ui-primitives/README.zh.md +++ b/packages/client/ui-primitives/README.zh.md @@ -10,7 +10,7 @@ ## Markdown 渲染 -`MarkdownText` 通过 React 元素渲染来自不受信任 assistant 输出的 GFM 与 `$…$` / `$$…$$` TeX 公式，公式由 KaTeX 排版并禁用受信任命令。它会省略原始 HTML，使相对链接及非 HTTP(S)/mailto 链接失效，以安全的外部链接属性打开 HTTP(S) 链接，并在不发送 referrer 的情况下渲染采用绝对 HTTP(S) URL 的图片；相对路径、绝对本地路径、`file:` URL 与不受支持的 scheme 会保留其 alt 文本。`MessageText` 仍是用户创作内容使用的字面文本原语。`extractMarkdownPlainText` 会移除 Markdown 呈现标记以用于紧凑标签，同时将原始 HTML 保留为字面文本。元素间距、响应式图片、表格、链接与行内代码使用与 deepsuite `@deepseek/md` 相同的 `--dsw-alias-markdown-*` / `--dsw-font-markdown-*` token。围栏代码块通过 `CodeBlock` 渲染（语言横幅、复制控件，以及对已注册语法使用 shiki）。 +`MarkdownText` 通过 React 元素渲染来自不受信任 assistant 输出的 GFM 与 `$…$`、`$$…$$`、`\(…\)` 和 `\[…\]` TeX 公式，公式由 KaTeX 排版并禁用受信任命令；块级同一行 `$$…$$` 是显示公式并支持 `\tag{}`。它会省略原始 HTML，使相对链接及非 HTTP(S)/mailto 链接失效，以安全的外部链接属性打开 HTTP(S) 链接，并在不发送 referrer 的情况下渲染采用绝对 HTTP(S) URL 的图片；相对路径、绝对本地路径、`file:` URL 与不受支持的 scheme 会保留其 alt 文本。`MessageText` 仍是用户创作内容使用的字面文本原语。`extractMarkdownPlainText` 会移除 Markdown 呈现标记以用于紧凑标签，同时将原始 HTML 保留为字面文本。元素间距、响应式图片、表格、链接与行内代码使用与 deepsuite `@deepseek/md` 相同的 `--dsw-alias-markdown-*` / `--dsw-font-markdown-*` token。围栏代码块通过 `CodeBlock` 渲染（语言横幅、复制控件，以及对已注册语法使用 shiki）。 ## 终端输出 diff --git a/packages/client/ui-primitives/src/markdown/remarkMathCompatibility.ts b/packages/client/ui-primitives/src/markdown/remarkMathCompatibility.ts index 15e6c2b359..c9dfc9a5d3 100644 --- a/packages/client/ui-primitives/src/markdown/remarkMathCompatibility.ts +++ b/packages/client/ui-primitives/src/markdown/remarkMathCompatibility.ts @@ -16,8 +16,6 @@ const previousBackslash: Previous = function (code) { } const tokenizeBackslashMathText: Tokenizer = function (effects, ok, nok) { - const self = this - return start function start(code: number | null): State | undefined { @@ -38,8 +36,8 @@ const tokenizeBackslashMathText: Tokenizer = function (effects, ok, nok) { function between(code: number | null): State | undefined { if (code === codes.eof) return nok(code) - if (code === codes.backslash && self.previous !== codes.backslash) { - return effects.attempt({ partial: true, tokenize: tokenizeClose }, close, dataStart)(code) + if (code === codes.backslash) { + return effects.attempt({ partial: true, tokenize: tokenizeClose }, close, afterCloseAttempt)(code) } if (markdownLineEnding(code)) { effects.enter(types.lineEnding) @@ -50,10 +48,22 @@ const tokenizeBackslashMathText: Tokenizer = function (effects, ok, nok) { return dataStart(code) } + function afterCloseAttempt(code: number | null): State | undefined { + return effects.check({ partial: true, tokenize: tokenizeOpen }, nok, dataStart)(code) + } + function dataStart(code: number | null): State | undefined { effects.enter('mathTextData') effects.consume(code) - return data + return code === codes.backslash ? afterDataBackslash : data + } + + function afterDataBackslash(code: number | null): State | undefined { + if (code === codes.backslash) { + effects.consume(code) + return data + } + return data(code) } function data(code: number | null): State | undefined { @@ -88,11 +98,31 @@ const tokenizeBackslashMathText: Tokenizer = function (effects, ok, nok) { return closeOk } } + + function tokenizeOpen(openEffects: Parameters[0], openOk: State, openNok: State): State { + return slash + + function slash(code: number | null): State | undefined { + /* v8 ignore next -- the opening check follows a failed close attempt at a backslash. */ + if (code !== codes.backslash) return openNok(code) + openEffects.enter(types.chunkString) + openEffects.consume(code) + return parenthesis + } + + function parenthesis(code: number | null): State | undefined { + if (code !== codes.leftParenthesis) return openNok(code) + openEffects.consume(code) + openEffects.exit(types.chunkString) + return openOk + } + } } function createMathFlow(marker: number, openMarker: number, closeMarker: number, multiline: boolean): Construct { const tokenize: Tokenizer = function (effects, ok, nok) { const self = this + let oddBackslashRun = false const tail = self.events.at(-1) const initialSize = tail?.[1].type === types.linePrefix ? tail[2].sliceSerialize(tail[1], true).length @@ -124,11 +154,14 @@ function createMathFlow(marker: number, openMarker: number, closeMarker: number, function content(code: number | null): State | undefined { if (code === codes.eof) return nok(code) - if (code === marker && (marker !== codes.backslash || self.previous !== codes.backslash)) { - return effects.attempt({ partial: true, tokenize: tokenizeClosingFence }, closed, markerValueStart)(code) + if (code === marker && (marker !== codes.dollarSign || !oddBackslashRun)) { + return effects.attempt( + { partial: true, tokenize: tokenizeClosingFence }, + closed, + afterClosingFenceAttempt, + )(code) } if (markdownLineEnding(code)) { - /* v8 ignore next -- micromark gives same-line dollar flow to remark-math before this continuation branch. */ return multiline ? effects.attempt(nonLazyContinuation, afterContinuation, nok)(code) : nok(code) @@ -136,6 +169,12 @@ function createMathFlow(marker: number, openMarker: number, closeMarker: number, return valueStart(code) } + function afterClosingFenceAttempt(code: number | null): State | undefined { + return marker === codes.backslash + ? effects.check({ partial: true, tokenize: tokenizeOpeningFence }, nok, markerValueStart)(code) + : markerValueStart(code) + } + function afterContinuation(code: number | null): State | undefined { return effects.attempt( { partial: true, tokenize: tokenizeClosingFence }, @@ -148,12 +187,14 @@ function createMathFlow(marker: number, openMarker: number, closeMarker: number, function valueStart(code: number | null): State | undefined { effects.enter('mathFlowValue') + oddBackslashRun = code === codes.backslash effects.consume(code) return value } function markerValueStart(code: number | null): State | undefined { effects.enter('mathFlowValue') + oddBackslashRun = false effects.consume(code) return valueAfterMarker } @@ -171,6 +212,7 @@ function createMathFlow(marker: number, openMarker: number, closeMarker: number, effects.exit('mathFlowValue') return content(code) } + oddBackslashRun = code === codes.backslash ? !oddBackslashRun : false effects.consume(code) return value } @@ -208,6 +250,29 @@ function createMathFlow(marker: number, openMarker: number, closeMarker: number, return closeOk(code) } } + + function tokenizeOpeningFence( + openEffects: Parameters[0], + openOk: State, + openNok: State, + ): State { + return sequenceStart + + function sequenceStart(code: number | null): State | undefined { + /* v8 ignore next -- the opening check follows a failed close attempt at the marker. */ + if (code !== marker) return openNok(code) + openEffects.enter(types.chunkString) + openEffects.consume(code) + return sequenceEnd + } + + function sequenceEnd(code: number | null): State | undefined { + if (code !== openMarker) return openNok(code) + openEffects.consume(code) + openEffects.exit(types.chunkString) + return openOk + } + } } return { @@ -272,7 +337,8 @@ const backslashMath: Extension = { } /** - * Add TeX backslash delimiters and same-line display-dollar blocks to remark. + * Add TeX backslash delimiters and same-line display-dollar blocks for remark-math. + * The same processor must register remark-math to compile the emitted math tokens. * @returns Nothing. */ export function remarkMathCompatibility(this: RemarkProcessor): undefined { diff --git a/packages/client/ui-primitives/tests/markdown.spec.tsx b/packages/client/ui-primitives/tests/markdown.spec.tsx index c34f8227c3..057e807b1d 100644 --- a/packages/client/ui-primitives/tests/markdown.spec.tsx +++ b/packages/client/ui-primitives/tests/markdown.spec.tsx @@ -207,6 +207,13 @@ describe('MarkdownText', () => { source: '\\(\\frac{1}{5}\n+\\frac{1}{7}\\)', math: 1, display: 0, + value: '\\frac{1}{5}\n+\\frac{1}{7}', + }, + { + source: '\\[a\\\\\nb\\]', + math: 1, + display: 1, + value: 'a\\\\\nb', }, { source: '> \\[\n> \\frac{1}{5}\n> \\]', @@ -225,6 +232,9 @@ describe('MarkdownText', () => { expect(rendered.container.querySelectorAll('.katex')).toHaveLength(item.math) expect(rendered.container.querySelectorAll('.katex-display')).toHaveLength(item.display) expect(rendered.container.querySelector('.katex-error')).toBeNull() + if ('value' in item) { + expect(rendered.container.querySelector('annotation')?.textContent).toBe(item.value) + } rendered.unmount() } @@ -239,8 +249,10 @@ describe('MarkdownText', () => { const sources = [ '$$\n\\theta\n$$', '$$$\\theta$$$', + '$$a$b\nc', ' \\[\n \\theta\n \\]', '\\(\\theta', + '\\[\n\\[', '> \\[\nnot a quoted continuation\n\\]', ] @@ -251,6 +263,29 @@ describe('MarkdownText', () => { } }) + it('renders escaped dollars and even backslash pairs before closing fences', () => { + const source = [ + String.raw`$$100\$$$`, + '', + String.raw`\(a\\\)`, + '', + String.raw`\[b\\\]`, + ].join('\n') + const { container } = render() + const values = [...container.querySelectorAll('annotation')].map(node => node.textContent) + + expect(values).toEqual([String.raw`100\$`, String.raw`a\\`, String.raw`b\\`]) + expect(container.querySelector('.katex-error')).toBeNull() + }) + + it('bounds fallback work for repeated unclosed backslash delimiters', () => { + const startedAt = performance.now() + const { container } = render() + + expect(performance.now() - startedAt).toBeLessThan(1_000) + expect(container.querySelector('.katex')).toBeNull() + }) + it('leaves TeX-looking fenced code literal', () => { const source = '```tex\n\\[\\frac{1}{5}\\]\n$$x \\tag{1}$$\n```' const { container } = render()