fix(web): render CJK-adjacent strong emphasis
This commit is contained in:
Yichen Jiang
@@ -2,5 +2,5 @@
|
||||
# side as of the last confirmed-consistent state. Both languages carry equal authority;
|
||||
# after editing either side, bring the other along and re-record with:
|
||||
# pnpm run verify-translation-pairing --write packages/client/ui-primitives/README.md
|
||||
README.md: 03e7e3649fd0913fb48579aa87634153f67f5baf
|
||||
README.zh.md: 090ecc34e8d514e38853de8ed52e82d3bf019b43
|
||||
README.md: aebcc06bf95b5bd843c2e8add6f89908d7bfa1fa
|
||||
README.zh.md: 7e141148cee23416e344f57089d425c272e86db4
|
||||
|
||||
@@ -10,7 +10,7 @@ Pure React atoms (zero cordis): StateDot, ic_ds_* icons, Button/Pill/Menu/Modal/
|
||||
|
||||
## Markdown rendering
|
||||
|
||||
`MarkdownText` renders GFM and `$…$`, `$$…$$`, `\(…\)`, and `\[…\]` TeX math from untrusted assistant output through React elements, with math typeset by KaTeX and trusted commands disabled; block-level same-line `$$…$$` is display math, including `\tag{}`. It omits raw HTML, neutralizes relative and non-HTTP(S)/mailto links, opens HTTP(S) links with safe external-link attributes, and renders absolute HTTP(S) images without a referrer; relative paths, absolute local paths, `file:` URLs, and unsupported schemes retain their alt text. `MessageText` remains the literal-text primitive for user-authored content. `extractMarkdownPlainText` removes Markdown presentation markup for compact labels while preserving raw HTML as literal text. Element spacing, responsive images, tables, links, and inline code use the same `--dsw-alias-markdown-*` / `--dsw-font-markdown-*` tokens as deepsuite `@deepseek/md`. Fenced blocks render through `CodeBlock` (language banner, copy control, shiki for the registered grammars).
|
||||
`MarkdownText` renders GFM and `$…$`, `$$…$$`, `\(…\)`, and `\[…\]` TeX math from untrusted assistant output through React elements, with math typeset by KaTeX and trusted commands disabled; block-level same-line `$$…$$` is display math, including `\tag{}`. A narrow micromark extension lets asterisk strong emphasis ending in punctuation close before adjacent CJK text, where prose normally omits the whitespace CommonMark requires; single-asterisk emphasis, non-CJK adjacency, escapes, code, and math retain upstream parsing. It omits raw HTML, neutralizes relative and non-HTTP(S)/mailto links, opens HTTP(S) links with safe external-link attributes, and renders absolute HTTP(S) images without a referrer; relative paths, absolute local paths, `file:` URLs, and unsupported schemes retain their alt text. `MessageText` remains the literal-text primitive for user-authored content. `extractMarkdownPlainText` removes Markdown presentation markup for compact labels while preserving raw HTML as literal text. Element spacing, responsive images, tables, links, and inline code use the same `--dsw-alias-markdown-*` / `--dsw-font-markdown-*` tokens as deepsuite `@deepseek/md`. Fenced blocks render through `CodeBlock` (language banner, copy control, shiki for the registered grammars).
|
||||
|
||||
## Terminal output
|
||||
|
||||
|
||||
@@ -10,7 +10,7 @@
|
||||
|
||||
## Markdown 渲染
|
||||
|
||||
`MarkdownText` 通过 React 元素渲染来自不受信任 assistant 输出的 GFM 与 `$…$`、`$$…$$`、`\(…\)` 和 `\[…\]` TeX 公式，公式由 KaTeX 排版并禁用受信任命令；块级同一行 `$$…$$` 是显示公式并支持 `\tag{}`。它会省略原始 HTML，使相对链接及非 HTTP(S)/mailto 链接失效，以安全的外部链接属性打开 HTTP(S) 链接，并在不发送 referrer 的情况下渲染采用绝对 HTTP(S) URL 的图片；相对路径、绝对本地路径、`file:` URL 与不受支持的 scheme 会保留其 alt 文本。`MessageText` 仍是用户创作内容使用的字面文本原语。`extractMarkdownPlainText` 会移除 Markdown 呈现标记以用于紧凑标签，同时将原始 HTML 保留为字面文本。元素间距、响应式图片、表格、链接与行内代码使用与 deepsuite `@deepseek/md` 相同的 `--dsw-alias-markdown-*` / `--dsw-font-markdown-*` token。围栏代码块通过 `CodeBlock` 渲染（语言横幅、复制控件，以及对已注册语法使用 shiki）。
|
||||
`MarkdownText` 通过 React 元素渲染来自不受信任 assistant 输出的 GFM 与 `$…$`、`$$…$$`、`\(…\)` 和 `\[…\]` TeX 公式，公式由 KaTeX 排版并禁用受信任命令；块级同一行 `$$…$$` 是显示公式并支持 `\tag{}`。一个小范围的 micromark 扩展允许由星号标记、以标点结尾的粗体在紧邻的 CJK 文本前闭合，以适应 CJK 文本通常省略 CommonMark 所要求空格的写法；单星号强调、紧邻非 CJK 文本的情况、转义、代码与数学公式仍沿用上游解析行为。它会省略原始 HTML，使相对链接及非 HTTP(S)/mailto 链接失效，以安全的外部链接属性打开 HTTP(S) 链接，并在不发送 referrer 的情况下渲染采用绝对 HTTP(S) URL 的图片；相对路径、绝对本地路径、`file:` URL 与不受支持的 scheme 会保留其 alt 文本。`MessageText` 仍是用户创作内容使用的字面文本原语。`extractMarkdownPlainText` 会移除 Markdown 呈现标记以用于紧凑标签，同时将原始 HTML 保留为字面文本。元素间距、响应式图片、表格、链接与行内代码使用与 deepsuite `@deepseek/md` 相同的 `--dsw-alias-markdown-*` / `--dsw-font-markdown-*` token。围栏代码块通过 `CodeBlock` 渲染（语言横幅、复制控件，以及对已注册语法使用 shiki）。
|
||||
|
||||
## 终端输出
|
||||
|
||||
|
||||
@@ -26,10 +26,12 @@
|
||||
"katex": "^0.16.47",
|
||||
"mdast-util-from-markdown": "^2.0.3",
|
||||
"mdast-util-gfm": "^3.1.0",
|
||||
"micromark-core-commonmark": "^2.0.3",
|
||||
"micromark-extension-gfm": "^3.0.0",
|
||||
"micromark-extension-math": "^3.1.0",
|
||||
"micromark-factory-space": "^2.0.1",
|
||||
"micromark-util-character": "^2.1.1",
|
||||
"micromark-util-classify-character": "^2.0.1",
|
||||
"micromark-util-symbol": "^2.0.1",
|
||||
"micromark-util-types": "^2.0.2",
|
||||
"react": "^18.2.0",
|
||||
|
||||
@@ -5,13 +5,15 @@ import rehypeKatex from 'rehype-katex'
|
||||
import remarkGfm from 'remark-gfm'
|
||||
import remarkMath from 'remark-math'
|
||||
import { CodeBlock } from './CodeBlock.tsx'
|
||||
import { remarkCjkFriendlyStrong } from './remarkCjkFriendlyStrong.ts'
|
||||
import { remarkMathCompatibility } from './remarkMathCompatibility.ts'
|
||||
import 'katex/dist/katex.min.css'
|
||||
import css from './MarkdownText.module.css'
|
||||
|
||||
const streamingRemarkPlugins = [remarkGfm]
|
||||
const streamingRemarkPlugins = [remarkGfm, remarkCjkFriendlyStrong]
|
||||
const settledRemarkPlugins = [
|
||||
remarkGfm,
|
||||
remarkCjkFriendlyStrong,
|
||||
remarkMathCompatibility,
|
||||
remarkMath,
|
||||
]
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
/** Let asterisk strong emphasis close after punctuation when CJK prose continues without whitespace. */
|
||||
|
||||
import { attention } from 'micromark-core-commonmark'
|
||||
import { unicodePunctuation } from 'micromark-util-character'
|
||||
import { classifyCharacter } from 'micromark-util-classify-character'
|
||||
import { codes, constants } from 'micromark-util-symbol'
|
||||
import type { Construct, Extension, State, Tokenizer } from 'micromark-util-types'
|
||||
|
||||
interface RemarkProcessor {
|
||||
data(): { micromarkExtensions?: Extension[] }
|
||||
}
|
||||
|
||||
const cjkCharacter = new RegExp([
|
||||
'\\p{Script_Extensions=Han}',
|
||||
'\\p{Script_Extensions=Hiragana}',
|
||||
'\\p{Script_Extensions=Katakana}',
|
||||
'\\p{Script_Extensions=Hangul}',
|
||||
'\\p{Script_Extensions=Bopomofo}',
|
||||
].join('|'), 'u')
|
||||
|
||||
function isCjkCharacter(code: number | null): boolean {
|
||||
return code !== null && code >= 0 && cjkCharacter.test(String.fromCodePoint(code))
|
||||
}
|
||||
|
||||
const tokenizeCjkFriendlyAttention: Tokenizer = function (effects, ok, nok) {
|
||||
const configuredAttentionMarkers = this.parser.constructs.attentionMarkers.null
|
||||
if (configuredAttentionMarkers === undefined) {
|
||||
throw new Error('micromark CommonMark attention markers are unavailable')
|
||||
}
|
||||
const attentionMarkers = configuredAttentionMarkers
|
||||
const previous = this.previous
|
||||
const before = classifyCharacter(previous)
|
||||
let marker: number | null = codes.eof
|
||||
|
||||
return start
|
||||
|
||||
function start(code: number | null): State | undefined {
|
||||
/* v8 ignore next -- this text construct is dispatched only for an asterisk. */
|
||||
if (code !== codes.asterisk) return nok(code)
|
||||
marker = code
|
||||
effects.enter('attentionSequence')
|
||||
return inside(code)
|
||||
}
|
||||
|
||||
function inside(code: number | null): State | undefined {
|
||||
if (code === marker) {
|
||||
effects.consume(code)
|
||||
return inside
|
||||
}
|
||||
|
||||
const token = effects.exit('attentionSequence')
|
||||
const after = classifyCharacter(code)
|
||||
const open = !after || (after === constants.characterGroupPunctuation && Boolean(before))
|
||||
|| attentionMarkers.includes(code)
|
||||
const commonMarkClose = !before
|
||||
|| (before === constants.characterGroupPunctuation && Boolean(after))
|
||||
|| attentionMarkers.includes(previous)
|
||||
const markerCount = token.end.offset - token.start.offset
|
||||
const cjkStrongClose = markerCount >= 2
|
||||
&& unicodePunctuation(previous)
|
||||
&& isCjkCharacter(code)
|
||||
const close = commonMarkClose || cjkStrongClose
|
||||
|
||||
token._open = open
|
||||
token._close = close
|
||||
return ok(code)
|
||||
}
|
||||
}
|
||||
|
||||
const cjkFriendlyAttention: Construct = {
|
||||
name: 'cjkFriendlyAttention',
|
||||
resolveAll: attention.resolveAll,
|
||||
tokenize: tokenizeCjkFriendlyAttention,
|
||||
}
|
||||
|
||||
const cjkFriendlyStrong: Extension = {
|
||||
text: { [codes.asterisk]: cjkFriendlyAttention },
|
||||
}
|
||||
|
||||
/**
|
||||
* Extend CommonMark asterisk strong emphasis for punctuation-delimited CJK prose.
|
||||
* @returns Nothing.
|
||||
*/
|
||||
export function remarkCjkFriendlyStrong(this: RemarkProcessor): undefined {
|
||||
const data = this.data()
|
||||
const extensions = data.micromarkExtensions ?? (data.micromarkExtensions = [])
|
||||
extensions.push(cjkFriendlyStrong)
|
||||
}
|
||||
@@ -3,6 +3,7 @@ import { cleanup, fireEvent, render, screen } from '@testing-library/react'
|
||||
import { afterEach, describe, expect, it } from 'vitest'
|
||||
import type { Extension } from 'micromark-util-types'
|
||||
import { JsonBlock, MarkdownText, MessageText } from '@deepseek-ai/dsh-client-ui-primitives'
|
||||
import { remarkCjkFriendlyStrong } from '../src/markdown/remarkCjkFriendlyStrong.ts'
|
||||
import { remarkMathCompatibility } from '../src/markdown/remarkMathCompatibility.ts'
|
||||
|
||||
afterEach(cleanup)
|
||||
@@ -68,6 +69,70 @@ describe('MarkdownText', () => {
|
||||
expect(screen.getByRole('link', { name: 'https://deepseek.com' })).toBeTruthy()
|
||||
})
|
||||
|
||||
it('closes punctuation-terminated strong emphasis before adjacent CJK text', () => {
|
||||
const cases = [
|
||||
['**注意：**内容', '注意：'],
|
||||
['**Notice:**内容', 'Notice:'],
|
||||
['**事件中间件（waterfall）**实现', '事件中间件（waterfall）'],
|
||||
['**事件中间件(waterfall)**实现', '事件中间件(waterfall)'],
|
||||
['**句号。**后续', '句号。'],
|
||||
['**Period.**后续', 'Period.'],
|
||||
['**提醒！**继续', '提醒！'],
|
||||
['**Warning!**继续', 'Warning!'],
|
||||
] as const
|
||||
const source = cases.map(([markdown]) => markdown).join('\n\n')
|
||||
|
||||
for (const streaming of [false, true]) {
|
||||
const rendered = render(<MarkdownText text={source} streaming={streaming} />)
|
||||
expect([...rendered.container.querySelectorAll('strong')].map(node => node.textContent))
|
||||
.toEqual(cases.map(([, strong]) => strong))
|
||||
rendered.unmount()
|
||||
}
|
||||
})
|
||||
|
||||
it('keeps the CJK strong extension out of escaped, code, math, and ASCII contexts', () => {
|
||||
const source = [
|
||||
String.raw`\**注意：**内容`,
|
||||
'`**注意：**内容`',
|
||||
'**Notice:**text',
|
||||
'*提醒！*继续',
|
||||
'$**注意：**内容$',
|
||||
'```md',
|
||||
'**注意：**内容',
|
||||
'```',
|
||||
'**普通**内容',
|
||||
'*普通*内容',
|
||||
].join('\n\n')
|
||||
const { container } = render(<MarkdownText text={source} />)
|
||||
|
||||
expect([...container.querySelectorAll('strong')].map(node => node.textContent)).toEqual(['普通'])
|
||||
expect([...container.querySelectorAll('em')].map(node => node.textContent)).toEqual(['普通'])
|
||||
expect(container.querySelector('code')?.textContent).toBe('**注意：**内容')
|
||||
expect(container.querySelector('.katex annotation')?.textContent).toBe('**注意：**内容')
|
||||
expect(container.querySelector('pre code')?.textContent).toContain('**注意：**内容')
|
||||
expect(container.textContent).toContain('**Notice:**text')
|
||||
expect(container.textContent).toContain('*提醒！*继续')
|
||||
expect(container.textContent).toContain('**注意：**内容')
|
||||
})
|
||||
|
||||
it('registers the CJK strong extension and rejects a parser without CommonMark attention markers', () => {
|
||||
const data: { micromarkExtensions?: Extension[] } = {}
|
||||
const processor = { data: () => data }
|
||||
remarkCjkFriendlyStrong.call(processor)
|
||||
remarkCjkFriendlyStrong.call(processor)
|
||||
|
||||
expect(data.micromarkExtensions).toHaveLength(2)
|
||||
const construct = data.micromarkExtensions?.[0]?.text?.[42]
|
||||
const tokenizer = Array.isArray(construct) ? construct[0]?.tokenize : construct?.tokenize
|
||||
expect(tokenizer).toBeTypeOf('function')
|
||||
expect(() => tokenizer?.call({
|
||||
parser: { constructs: { attentionMarkers: {} } },
|
||||
previous: null,
|
||||
} as never, {} as never, () => undefined, () => undefined)).toThrow(
|
||||
'micromark CommonMark attention markers are unavailable',
|
||||
)
|
||||
})
|
||||
|
||||
it('a fence labeled with an inherited object key renders plain, never crashing shiki', () => {
|
||||
for (const label of ['constructor', '__proto__', 'toString', 'hasOwnProperty']) {
|
||||
const { container, unmount } = render(<MarkdownText text={'```' + label + '\ncode body\n```'} />)
|
||||
|
||||
Reference in New Issue
Block a user