Merge pull request #1732 from deepseek-harness/worktree/fix-1680-cjk-markdown
fix(web): render CJK emphasis and link code URLs
This commit is contained in:
Yichen Jiang
@@ -2,5 +2,5 @@
|
||||
# side as of the last confirmed-consistent state. Both languages carry equal authority;
|
||||
# after editing either side, bring the other along and re-record with:
|
||||
# pnpm run verify-translation-pairing --write packages/client/ui-primitives/README.md
|
||||
README.md: 03e7e3649fd0913fb48579aa87634153f67f5baf
|
||||
README.zh.md: 090ecc34e8d514e38853de8ed52e82d3bf019b43
|
||||
README.md: c64e86152737fd55c49ac39cc2e7b523e0323774
|
||||
README.zh.md: 29123c3570122bc0fe6a1808a75c5bc659315eaa
|
||||
|
||||
@@ -10,7 +10,7 @@ Pure React atoms (zero cordis): StateDot, ic_ds_* icons, Button/Pill/Menu/Modal/
|
||||
|
||||
## Markdown rendering
|
||||
|
||||
`MarkdownText` renders GFM and `$…$`, `$$…$$`, `\(…\)`, and `\[…\]` TeX math from untrusted assistant output through React elements, with math typeset by KaTeX and trusted commands disabled; block-level same-line `$$…$$` is display math, including `\tag{}`. It omits raw HTML, neutralizes relative and non-HTTP(S)/mailto links, opens HTTP(S) links with safe external-link attributes, and renders absolute HTTP(S) images without a referrer; relative paths, absolute local paths, `file:` URLs, and unsupported schemes retain their alt text. `MessageText` remains the literal-text primitive for user-authored content. `extractMarkdownPlainText` removes Markdown presentation markup for compact labels while preserving raw HTML as literal text. Element spacing, responsive images, tables, links, and inline code use the same `--dsw-alias-markdown-*` / `--dsw-font-markdown-*` tokens as deepsuite `@deepseek/md`. Fenced blocks render through `CodeBlock` (language banner, copy control, shiki for the registered grammars).
|
||||
`MarkdownText` renders GFM and `$…$`, `$$…$$`, `\(…\)`, and `\[…\]` TeX math from untrusted assistant output through React elements, with math typeset by KaTeX and trusted commands disabled; block-level same-line `$$…$$` is display math, including `\tag{}`. A narrow micromark extension lets asterisk strong emphasis ending in punctuation close before adjacent CJK text, where prose normally omits the whitespace CommonMark requires; single-asterisk emphasis, non-CJK adjacency, escapes, code, and math retain upstream parsing. It omits raw HTML, neutralizes relative and non-HTTP(S)/mailto links, opens HTTP(S) links with safe external-link attributes, and renders absolute HTTP(S) images without a referrer; relative paths, absolute local paths, `file:` URLs, and unsupported schemes retain their alt text. Inline code whose complete value is an absolute HTTP(S) URL keeps its code styling and gains the same safe external anchor; commands, partial URLs, other schemes, and fenced code remain inert. `MessageText` remains the literal-text primitive for user-authored content. `extractMarkdownPlainText` removes Markdown presentation markup for compact labels while preserving raw HTML as literal text. Element spacing, responsive images, tables, links, and inline code use the same `--dsw-alias-markdown-*` / `--dsw-font-markdown-*` tokens as deepsuite `@deepseek/md`. Fenced blocks render through `CodeBlock` (language banner, copy control, shiki for the registered grammars).
|
||||
|
||||
## Terminal output
|
||||
|
||||
|
||||
@@ -10,7 +10,7 @@
|
||||
|
||||
## Markdown 渲染
|
||||
|
||||
`MarkdownText` 通过 React 元素渲染来自不受信任 assistant 输出的 GFM 与 `$…$`、`$$…$$`、`\(…\)` 和 `\[…\]` TeX 公式，公式由 KaTeX 排版并禁用受信任命令；块级同一行 `$$…$$` 是显示公式并支持 `\tag{}`。它会省略原始 HTML，使相对链接及非 HTTP(S)/mailto 链接失效，以安全的外部链接属性打开 HTTP(S) 链接，并在不发送 referrer 的情况下渲染采用绝对 HTTP(S) URL 的图片；相对路径、绝对本地路径、`file:` URL 与不受支持的 scheme 会保留其 alt 文本。`MessageText` 仍是用户创作内容使用的字面文本原语。`extractMarkdownPlainText` 会移除 Markdown 呈现标记以用于紧凑标签，同时将原始 HTML 保留为字面文本。元素间距、响应式图片、表格、链接与行内代码使用与 deepsuite `@deepseek/md` 相同的 `--dsw-alias-markdown-*` / `--dsw-font-markdown-*` token。围栏代码块通过 `CodeBlock` 渲染（语言横幅、复制控件，以及对已注册语法使用 shiki）。
|
||||
`MarkdownText` 通过 React 元素渲染来自不受信任 assistant 输出的 GFM 与 `$…$`、`$$…$$`、`\(…\)` 和 `\[…\]` TeX 公式，公式由 KaTeX 排版并禁用受信任命令；块级同一行 `$$…$$` 是显示公式并支持 `\tag{}`。一个小范围的 micromark 扩展允许由星号标记、以标点结尾的粗体在紧邻的 CJK 文本前闭合，以适应 CJK 文本通常省略 CommonMark 所要求空格的写法；单星号强调、紧邻非 CJK 文本的情况、转义、代码与数学公式仍沿用上游解析行为。它会省略原始 HTML，使相对链接及非 HTTP(S)/mailto 链接失效，以安全的外部链接属性打开 HTTP(S) 链接，并在不发送 referrer 的情况下渲染采用绝对 HTTP(S) URL 的图片；相对路径、绝对本地路径、`file:` URL 与不受支持的 scheme 会保留其 alt 文本。完整内容为绝对 HTTP(S) URL 的行内代码会保留代码样式，并获得同样安全的外部链接；命令、非完整 URL、其他 scheme 与围栏代码仍不会成为链接。`MessageText` 仍是用户创作内容使用的字面文本原语。`extractMarkdownPlainText` 会移除 Markdown 呈现标记以用于紧凑标签，同时将原始 HTML 保留为字面文本。元素间距、响应式图片、表格、链接与行内代码使用与 deepsuite `@deepseek/md` 相同的 `--dsw-alias-markdown-*` / `--dsw-font-markdown-*` token。围栏代码块通过 `CodeBlock` 渲染（语言横幅、复制控件，以及对已注册语法使用 shiki）。
|
||||
|
||||
## 终端输出
|
||||
|
||||
|
||||
@@ -26,10 +26,12 @@
|
||||
"katex": "^0.16.47",
|
||||
"mdast-util-from-markdown": "^2.0.3",
|
||||
"mdast-util-gfm": "^3.1.0",
|
||||
"micromark-core-commonmark": "^2.0.3",
|
||||
"micromark-extension-gfm": "^3.0.0",
|
||||
"micromark-extension-math": "^3.1.0",
|
||||
"micromark-factory-space": "^2.0.1",
|
||||
"micromark-util-character": "^2.1.1",
|
||||
"micromark-util-classify-character": "^2.0.1",
|
||||
"micromark-util-symbol": "^2.0.1",
|
||||
"micromark-util-types": "^2.0.2",
|
||||
"react": "^18.2.0",
|
||||
|
||||
@@ -1,17 +1,19 @@
|
||||
import { isValidElement, useMemo } from 'react'
|
||||
import { isValidElement, useMemo, type ReactNode } from 'react'
|
||||
import ReactMarkdown from 'react-markdown'
|
||||
import type { Components, UrlTransform } from 'react-markdown'
|
||||
import rehypeKatex from 'rehype-katex'
|
||||
import remarkGfm from 'remark-gfm'
|
||||
import remarkMath from 'remark-math'
|
||||
import { CodeBlock } from './CodeBlock.tsx'
|
||||
import { remarkCjkFriendlyStrong } from './remarkCjkFriendlyStrong.ts'
|
||||
import { remarkMathCompatibility } from './remarkMathCompatibility.ts'
|
||||
import 'katex/dist/katex.min.css'
|
||||
import css from './MarkdownText.module.css'
|
||||
|
||||
const streamingRemarkPlugins = [remarkGfm]
|
||||
const streamingRemarkPlugins = [remarkGfm, remarkCjkFriendlyStrong]
|
||||
const settledRemarkPlugins = [
|
||||
remarkGfm,
|
||||
remarkCjkFriendlyStrong,
|
||||
remarkMathCompatibility,
|
||||
remarkMath,
|
||||
]
|
||||
@@ -34,6 +36,30 @@ function sanitizeUrl(url: string): string {
|
||||
|
||||
const safeUrl: UrlTransform = url => sanitizeUrl(url)
|
||||
|
||||
function renderSafeLink(href: string, children: ReactNode): ReactNode {
|
||||
const safeHref = sanitizeUrl(href)
|
||||
if (safeHref === '') return <>{children}</>
|
||||
const external = ['http:', 'https:'].includes(new URL(safeHref).protocol)
|
||||
return (
|
||||
<a
|
||||
href={safeHref}
|
||||
{...(external ? { target: '_blank', rel: 'noopener noreferrer' } : {})}
|
||||
>
|
||||
{children}
|
||||
</a>
|
||||
)
|
||||
}
|
||||
|
||||
function inlineCodeHttpUrl(value: string): string | undefined {
|
||||
if (value.trim() !== value) return undefined
|
||||
try {
|
||||
const protocol = new URL(value).protocol
|
||||
return protocol === 'http:' || protocol === 'https:' ? value : undefined
|
||||
} catch {
|
||||
return undefined
|
||||
}
|
||||
}
|
||||
|
||||
/** Copy-button labels forwarded to fence CodeBlocks (this package is cordis-free, so copy arrives via props). */
|
||||
export interface MarkdownCodeLabels {
|
||||
/** Copy-button idle label. */
|
||||
@@ -54,18 +80,10 @@ function remoteImageUrl(url: string): string | undefined {
|
||||
/** Build the component table; while `streaming`, fences render the plain arm (see CodeBlock). */
|
||||
function buildComponents(streaming: boolean, codeLabels?: MarkdownCodeLabels): Components {
|
||||
return {
|
||||
a: ({ href = '', children }) => {
|
||||
const safeHref = sanitizeUrl(href)
|
||||
if (safeHref === '') return <>{children}</>
|
||||
const external = ['http:', 'https:'].includes(new URL(safeHref).protocol)
|
||||
return (
|
||||
<a
|
||||
href={safeHref}
|
||||
{...(external ? { target: '_blank', rel: 'noopener noreferrer' } : {})}
|
||||
>
|
||||
{children}
|
||||
</a>
|
||||
)
|
||||
a: ({ href = '', children }) => renderSafeLink(href, children),
|
||||
code: ({ className, children }) => {
|
||||
const href = typeof children === 'string' ? inlineCodeHttpUrl(children) : undefined
|
||||
return <code className={className}>{href === undefined ? children : renderSafeLink(href, children)}</code>
|
||||
},
|
||||
img: ({ alt = '', src = '' }) => {
|
||||
const imageSrc = remoteImageUrl(src)
|
||||
@@ -88,10 +106,11 @@ function buildComponents(streaming: boolean, codeLabels?: MarkdownCodeLabels): C
|
||||
),
|
||||
// Fenced blocks route through the shared CodeBlock (shiki for registered
|
||||
// grammars, identical-geometry plain fallback for unknown/absent
|
||||
// languages); inline code keeps the default <code> path (the :not(pre)
|
||||
// rule styles it). While the message streams, the fence renders the
|
||||
// plain arm — retokenizing a growing fence on every chunk is quadratic
|
||||
// main-thread work; the finalize swap highlights it once.
|
||||
// languages); inline code keeps the <code> path (the :not(pre) rule
|
||||
// styles it), with a safe anchor only for complete HTTP(S) values. While
|
||||
// the message streams, the fence renders the plain arm — retokenizing a
|
||||
// growing fence on every chunk is quadratic main-thread work; the
|
||||
// finalize swap highlights it once.
|
||||
pre: ({ children }) => {
|
||||
// The markdown pipeline always hands `pre` its single `code` element;
|
||||
// the undefined arm guards a react-markdown representation change.
|
||||
@@ -126,7 +145,8 @@ const streamingComponents = buildComponents(true)
|
||||
* component table memoizes on its identity and a fresh literal per render
|
||||
* would rebuild it every streaming chunk.
|
||||
* @returns A GFM document with TeX math rendered through KaTeX; raw HTML,
|
||||
* relative links, and unsafe protocols are disabled, while absolute HTTP(S)
|
||||
* relative links, and unsafe protocols are disabled; complete HTTP(S)
|
||||
* inline-code values become safe external links, while absolute HTTP(S)
|
||||
* images render directly.
|
||||
*/
|
||||
export function MarkdownText({ text, streaming = false, codeLabels }: {
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
/** Let asterisk strong emphasis close after punctuation when CJK prose continues without whitespace. */
|
||||
|
||||
import { attention } from 'micromark-core-commonmark'
|
||||
import { unicodePunctuation } from 'micromark-util-character'
|
||||
import { classifyCharacter } from 'micromark-util-classify-character'
|
||||
import { codes, constants } from 'micromark-util-symbol'
|
||||
import type { Construct, Extension, State, Tokenizer } from 'micromark-util-types'
|
||||
|
||||
interface RemarkProcessor {
|
||||
data(): { micromarkExtensions?: Extension[] }
|
||||
}
|
||||
|
||||
const cjkCharacter = new RegExp([
|
||||
'\\p{Script_Extensions=Han}',
|
||||
'\\p{Script_Extensions=Hiragana}',
|
||||
'\\p{Script_Extensions=Katakana}',
|
||||
'\\p{Script_Extensions=Hangul}',
|
||||
'\\p{Script_Extensions=Bopomofo}',
|
||||
].join('|'), 'u')
|
||||
|
||||
function isCjkCharacter(code: number | null): boolean {
|
||||
return code !== null && code >= 0 && cjkCharacter.test(String.fromCodePoint(code))
|
||||
}
|
||||
|
||||
const tokenizeCjkFriendlyAttention: Tokenizer = function (effects, ok, nok) {
|
||||
const configuredAttentionMarkers = this.parser.constructs.attentionMarkers.null
|
||||
if (configuredAttentionMarkers === undefined) {
|
||||
throw new Error('micromark CommonMark attention markers are unavailable')
|
||||
}
|
||||
const attentionMarkers = configuredAttentionMarkers
|
||||
const previous = this.previous
|
||||
const before = classifyCharacter(previous)
|
||||
let marker: number | null = codes.eof
|
||||
|
||||
return start
|
||||
|
||||
function start(code: number | null): State | undefined {
|
||||
/* v8 ignore next -- this text construct is dispatched only for an asterisk. */
|
||||
if (code !== codes.asterisk) return nok(code)
|
||||
marker = code
|
||||
effects.enter('attentionSequence')
|
||||
return inside(code)
|
||||
}
|
||||
|
||||
function inside(code: number | null): State | undefined {
|
||||
if (code === marker) {
|
||||
effects.consume(code)
|
||||
return inside
|
||||
}
|
||||
|
||||
const token = effects.exit('attentionSequence')
|
||||
const after = classifyCharacter(code)
|
||||
const open = !after || (after === constants.characterGroupPunctuation && Boolean(before))
|
||||
|| attentionMarkers.includes(code)
|
||||
const commonMarkClose = !before
|
||||
|| (before === constants.characterGroupPunctuation && Boolean(after))
|
||||
|| attentionMarkers.includes(previous)
|
||||
const markerCount = token.end.offset - token.start.offset
|
||||
const cjkStrongClose = markerCount >= 2
|
||||
&& unicodePunctuation(previous)
|
||||
&& isCjkCharacter(code)
|
||||
const close = commonMarkClose || cjkStrongClose
|
||||
|
||||
token._open = open
|
||||
token._close = close
|
||||
return ok(code)
|
||||
}
|
||||
}
|
||||
|
||||
const cjkFriendlyAttention: Construct = {
|
||||
name: 'cjkFriendlyAttention',
|
||||
resolveAll: attention.resolveAll,
|
||||
tokenize: tokenizeCjkFriendlyAttention,
|
||||
}
|
||||
|
||||
const cjkFriendlyStrong: Extension = {
|
||||
text: { [codes.asterisk]: cjkFriendlyAttention },
|
||||
}
|
||||
|
||||
/**
|
||||
* Extend CommonMark asterisk strong emphasis for punctuation-delimited CJK prose.
|
||||
* @returns Nothing.
|
||||
*/
|
||||
export function remarkCjkFriendlyStrong(this: RemarkProcessor): undefined {
|
||||
const data = this.data()
|
||||
const extensions = data.micromarkExtensions ?? (data.micromarkExtensions = [])
|
||||
extensions.push(cjkFriendlyStrong)
|
||||
}
|
||||
@@ -3,6 +3,7 @@ import { cleanup, fireEvent, render, screen } from '@testing-library/react'
|
||||
import { afterEach, describe, expect, it } from 'vitest'
|
||||
import type { Extension } from 'micromark-util-types'
|
||||
import { JsonBlock, MarkdownText, MessageText } from '@deepseek-ai/dsh-client-ui-primitives'
|
||||
import { remarkCjkFriendlyStrong } from '../src/markdown/remarkCjkFriendlyStrong.ts'
|
||||
import { remarkMathCompatibility } from '../src/markdown/remarkMathCompatibility.ts'
|
||||
|
||||
afterEach(cleanup)
|
||||
@@ -68,6 +69,104 @@ describe('MarkdownText', () => {
|
||||
expect(screen.getByRole('link', { name: 'https://deepseek.com' })).toBeTruthy()
|
||||
})
|
||||
|
||||
it('closes punctuation-terminated strong emphasis before adjacent CJK text', () => {
|
||||
const cases = [
|
||||
['**注意：**内容', '注意：'],
|
||||
['**Notice:**内容', 'Notice:'],
|
||||
['**事件中间件（waterfall）**实现', '事件中间件（waterfall）'],
|
||||
['**事件中间件(waterfall)**实现', '事件中间件(waterfall)'],
|
||||
['**句号。**后续', '句号。'],
|
||||
['**Period.**后续', 'Period.'],
|
||||
['**提醒！**继续', '提醒！'],
|
||||
['**Warning!**继续', 'Warning!'],
|
||||
] as const
|
||||
const source = cases.map(([markdown]) => markdown).join('\n\n')
|
||||
|
||||
for (const streaming of [false, true]) {
|
||||
const rendered = render(<MarkdownText text={source} streaming={streaming} />)
|
||||
expect([...rendered.container.querySelectorAll('strong')].map(node => node.textContent))
|
||||
.toEqual(cases.map(([, strong]) => strong))
|
||||
rendered.unmount()
|
||||
}
|
||||
})
|
||||
|
||||
it('keeps the CJK strong extension out of escaped, code, math, and ASCII contexts', () => {
|
||||
const source = [
|
||||
String.raw`\**注意：**内容`,
|
||||
'`**注意：**内容`',
|
||||
'**Notice:**text',
|
||||
'*提醒！*继续',
|
||||
'$**注意：**内容$',
|
||||
'```md',
|
||||
'**注意：**内容',
|
||||
'```',
|
||||
'**普通**内容',
|
||||
'*普通*内容',
|
||||
].join('\n\n')
|
||||
const { container } = render(<MarkdownText text={source} />)
|
||||
|
||||
expect([...container.querySelectorAll('strong')].map(node => node.textContent)).toEqual(['普通'])
|
||||
expect([...container.querySelectorAll('em')].map(node => node.textContent)).toEqual(['普通'])
|
||||
expect(container.querySelector('code')?.textContent).toBe('**注意：**内容')
|
||||
expect(container.querySelector('.katex annotation')?.textContent).toBe('**注意：**内容')
|
||||
expect(container.querySelector('pre code')?.textContent).toContain('**注意：**内容')
|
||||
expect(container.textContent).toContain('**Notice:**text')
|
||||
expect(container.textContent).toContain('*提醒！*继续')
|
||||
expect(container.textContent).toContain('**注意：**内容')
|
||||
})
|
||||
|
||||
it('links complete HTTP(S) inline code without promoting commands, unsafe schemes, or fences', () => {
|
||||
const localUrl = 'http://127.0.0.1:3199/?demo=1'
|
||||
const remoteUrl = 'https://example.com/preview?q=one%20two#result'
|
||||
const source = [
|
||||
`\`${localUrl}\``,
|
||||
`\`${remoteUrl}\``,
|
||||
'`curl http://127.0.0.1:3199/?demo=1`',
|
||||
'`javascript:alert(1)`',
|
||||
'`mailto:dev@example.com`',
|
||||
`\` ${localUrl} \``,
|
||||
'```',
|
||||
localUrl,
|
||||
'```',
|
||||
].join('\n\n')
|
||||
const { container } = render(<MarkdownText text={source} />)
|
||||
|
||||
const links = screen.getAllByRole('link')
|
||||
expect(links.map(link => link.getAttribute('href'))).toEqual([localUrl, remoteUrl])
|
||||
for (const link of links) {
|
||||
expect(link.closest('code')).not.toBeNull()
|
||||
expect(link.getAttribute('target')).toBe('_blank')
|
||||
expect(link.getAttribute('rel')).toBe('noopener noreferrer')
|
||||
}
|
||||
links[0]?.focus()
|
||||
expect(document.activeElement).toBe(links[0])
|
||||
expect(screen.getByText('curl http://127.0.0.1:3199/?demo=1').closest('a')).toBeNull()
|
||||
expect(screen.getByText('javascript:alert(1)').closest('a')).toBeNull()
|
||||
expect(screen.getByText('mailto:dev@example.com').closest('a')).toBeNull()
|
||||
const paddedCode = [...container.querySelectorAll('code')]
|
||||
.find(code => code.textContent === ` ${localUrl} `)
|
||||
expect(paddedCode?.querySelector('a')).toBeNull()
|
||||
expect(container.querySelector('pre code a')).toBeNull()
|
||||
})
|
||||
|
||||
it('registers the CJK strong extension and rejects a parser without CommonMark attention markers', () => {
|
||||
const data: { micromarkExtensions?: Extension[] } = {}
|
||||
const processor = { data: () => data }
|
||||
remarkCjkFriendlyStrong.call(processor)
|
||||
remarkCjkFriendlyStrong.call(processor)
|
||||
|
||||
expect(data.micromarkExtensions).toHaveLength(2)
|
||||
const construct = data.micromarkExtensions?.[0]?.text?.[42]
|
||||
const tokenizer = Array.isArray(construct) ? construct[0]?.tokenize : construct?.tokenize
|
||||
expect(tokenizer).toBeTypeOf('function')
|
||||
expect(() => tokenizer?.call({
|
||||
parser: { constructs: { attentionMarkers: {} } },
|
||||
previous: null,
|
||||
} as never, {} as never, () => undefined, () => undefined)).toThrow(
|
||||
'micromark CommonMark attention markers are unavailable',
|
||||
)
|
||||
})
|
||||
|
||||
it('a fence labeled with an inherited object key renders plain, never crashing shiki', () => {
|
||||
for (const label of ['constructor', '__proto__', 'toString', 'hasOwnProperty']) {
|
||||
const { container, unmount } = render(<MarkdownText text={'```' + label + '\ncode body\n```'} />)
|
||||
|
||||
Reference in New Issue
Block a user