refactor: detect md hard-wraps via mdast AST, not regex
Per review feedback (use a real markdown parser with an AST linked to source positions), rewrite verify-md-wrap to parse each file with mdast-util-from-markdown (the CommonMark parser behind remark) + the GFM extension, then flag any `paragraph` node whose source span covers more than one line. Why a parser over the hand-rolled line scanner: - It is a checker, not a formatter — it reports and never rewrites, so zero cosmetic churn (no emphasis-marker or table-delimiter normalization, which is why Prettier was rejected for this). - The AST owns every structural exemption (fenced code of any fence length, tables, lists, blockquotes, HTML, headings, reference defs), fixing both bugs the regex version had: it now catches wrapped list-item / blockquote prose (a `paragraph` inside those nodes) and no longer false-positives on a longer ```` fence wrapping an inner ```. Also unwrap two pre-existing hard-wrapped blockquotes (architecture.md, adding-a-tool.md) that the stricter AST check correctly surfaced.
This commit is contained in:
Tianyi Cui
@@ -5,17 +5,22 @@
|
||||
* paragraph (a one-word edit reflows and re-diffs the whole block) is a defect
|
||||
* this script catches before review.
|
||||
*
|
||||
* Scope mirrors doc-typecheck plus the two AGENTS.md files that doc-sync does
|
||||
* NOT otherwise cover (the convention itself lives there): README.md,
|
||||
* docs/** /*.md, packages/* /README.md, AGENTS.md, packages/AGENTS.md. (The
|
||||
* root and packages/ CLAUDE.md are symlinks to the AGENTS.md files, so they are
|
||||
* skipped to avoid double-reporting.)
|
||||
* Detection is AST-based: we parse each file with mdast-util-from-markdown (the
|
||||
* CommonMark parser behind remark) plus the GFM extension, then flag any
|
||||
* `paragraph` node whose source span covers more than one line. The parser owns
|
||||
* all the structure that legitimately occupies multiple lines — fenced code
|
||||
* (any fence length), tables, list items, blockquotes, HTML blocks, headings,
|
||||
* thematic breaks, link-reference definitions — so a hard wrap is simply "a
|
||||
* paragraph node that starts and ends on different lines." This is checker, not
|
||||
* formatter: it reports and never rewrites, so it introduces zero cosmetic
|
||||
* churn (no emphasis-marker or table-delimiter normalization).
|
||||
*
|
||||
* A violation is two consecutive *prose* lines — a paragraph that spans
|
||||
* physical lines instead of soft-wrapping. Structure that legitimately occupies
|
||||
* multiple lines is exempt: fenced code blocks, tables, list items (and their
|
||||
* indented continuations), headings, blockquotes, HTML blocks/comments,
|
||||
* horizontal rules, and reference-link / footnote definitions.
|
||||
* A wrapped paragraph inside a list item or blockquote is still a `paragraph`
|
||||
* node, so those are caught too. Scope mirrors doc-typecheck plus the two
|
||||
* AGENTS.md files that doc-sync does NOT otherwise cover (the convention itself
|
||||
* lives there): README.md, docs/** /*.md, packages/* /README.md, AGENTS.md,
|
||||
* packages/AGENTS.md. The root and packages/ CLAUDE.md are symlinks to the
|
||||
* AGENTS.md files, so they are deduped by real path.
|
||||
*
|
||||
* Run: `tsx scripts/verify-md-wrap.ts`.
|
||||
*/
|
||||
@@ -23,124 +28,47 @@
|
||||
import { readFileSync, realpathSync } from 'node:fs'
|
||||
import { relative, resolve } from 'node:path'
|
||||
import { glob } from 'node:fs/promises'
|
||||
import { fromMarkdown } from 'mdast-util-from-markdown'
|
||||
import { gfmFromMarkdown } from 'mdast-util-gfm'
|
||||
import { gfm } from 'micromark-extension-gfm'
|
||||
import type { Nodes } from 'mdast'
|
||||
|
||||
const root = resolve(import.meta.dirname, '..')
|
||||
|
||||
/** Files to check: doc-typecheck's scope plus the AGENTS.md pair. */
|
||||
const PATTERNS = ['README.md', 'docs/**/*.md', 'packages/*/README.md', 'AGENTS.md', 'packages/AGENTS.md']
|
||||
|
||||
/** A located hard-wrap: the second line of a multi-line prose paragraph. */
|
||||
/** A located hard-wrap: a prose paragraph spanning more than one source line. */
|
||||
interface Violation {
|
||||
file: string
|
||||
/** 1-based line number of the offending continuation line. */
|
||||
/** 1-based line where the hard-wrapped paragraph starts. */
|
||||
line: number
|
||||
text: string
|
||||
}
|
||||
|
||||
/**
|
||||
* True when a line is *prose* — ordinary paragraph text, not markdown
|
||||
* structure. Structural lines (headings, lists, tables, blockquotes, HTML,
|
||||
* fences, hrs, reference defs) legitimately stand alone or stack, so they never
|
||||
* count toward a hard-wrap pair. Caller handles fenced-code and list-body state.
|
||||
*/
|
||||
function isProse(line: string): boolean {
|
||||
if (line.trim() === '') return false
|
||||
// Up to 3 leading spaces is still a "top-level" block in CommonMark; deeper
|
||||
// indentation is handled as list continuation by the caller.
|
||||
const s = line.replace(/^ {0,3}/, '')
|
||||
if (/^#{1,6}\s/.test(s)) return false // ATX heading
|
||||
if (/^([-*+])\s/.test(s)) return false // bullet list
|
||||
if (/^\d{1,9}[.)]\s/.test(s)) return false // ordered list
|
||||
if (/^>/.test(s)) return false // blockquote
|
||||
if (/^\|/.test(s)) return false // table row
|
||||
if (/^<!--/.test(s) || /-->\s*$/.test(s)) return false // HTML comment line
|
||||
if (/^</.test(s)) return false // HTML block line
|
||||
if (/^([-*_])( *\1){2,}\s*$/.test(s)) return false // thematic break (hr)
|
||||
if (/^\[[^\]]+\]:\s/.test(s)) return false // reference-link / footnote definition
|
||||
if (/^[=-]+\s*$/.test(s)) return false // setext heading underline
|
||||
return true
|
||||
}
|
||||
|
||||
/** Find every hard-wrapped prose paragraph in one Markdown file. */
|
||||
/** Find every hard-wrapped prose paragraph in one Markdown file via its AST. */
|
||||
function findViolations(absPath: string): Violation[] {
|
||||
const file = relative(root, absPath)
|
||||
const lines = readFileSync(absPath, 'utf8').split('\n')
|
||||
const source = readFileSync(absPath, 'utf8')
|
||||
const tree = fromMarkdown(source, { extensions: [gfm()], mdastExtensions: [gfmFromMarkdown()] })
|
||||
const out: Violation[] = []
|
||||
|
||||
let inFence = false
|
||||
let fenceMarker = '' // '```' or '~~~'
|
||||
let inComment = false // inside a multi-line <!-- … --> HTML comment
|
||||
let inListItem = false // inside a list item's body (its indented continuations)
|
||||
let prevWasProse = false
|
||||
|
||||
lines.forEach((raw, i) => {
|
||||
const trimmed = raw.trim()
|
||||
|
||||
// Fenced code blocks: everything between matching fences is exempt.
|
||||
const fence = /^ {0,3}(```+|~~~+)/.exec(raw)
|
||||
if (fence) {
|
||||
const marker = (fence[1] ?? '').startsWith('`') ? '```' : '~~~'
|
||||
if (!inFence) {
|
||||
inFence = true
|
||||
fenceMarker = marker
|
||||
} else if (marker === fenceMarker) {
|
||||
inFence = false
|
||||
const visit = (node: Nodes): void => {
|
||||
if (node.type === 'paragraph' && node.position) {
|
||||
const { start, end } = node.position
|
||||
if (end.line > start.line) {
|
||||
const firstLine = source.split('\n')[start.line - 1] ?? ''
|
||||
out.push({ file, line: start.line, text: firstLine.trim() })
|
||||
}
|
||||
prevWasProse = false
|
||||
// A paragraph's children are inline (text/emphasis/…); no nested
|
||||
// paragraphs to find, so don't descend.
|
||||
return
|
||||
}
|
||||
if (inFence) {
|
||||
prevWasProse = false
|
||||
return
|
||||
if ('children' in node) {
|
||||
for (const child of node.children) visit(child)
|
||||
}
|
||||
|
||||
// Multi-line HTML comments are exempt (e.g. generated-file headers). Track
|
||||
// open/close across lines so the body of a 3+ line comment isn't read as
|
||||
// hard-wrapped prose.
|
||||
if (inComment) {
|
||||
if (/-->/.test(raw)) inComment = false
|
||||
prevWasProse = false
|
||||
return
|
||||
}
|
||||
if (/^ {0,3}<!--/.test(raw) && !/-->/.test(raw)) {
|
||||
inComment = true
|
||||
prevWasProse = false
|
||||
return
|
||||
}
|
||||
|
||||
if (trimmed === '') {
|
||||
inListItem = false
|
||||
prevWasProse = false
|
||||
return
|
||||
}
|
||||
|
||||
// Track list context so an item's wrapped continuation lines (indented or
|
||||
// lazy) are treated as list structure, not a hard-wrapped prose paragraph.
|
||||
const isListMarker = /^ {0,3}([-*+]|\d{1,9}[.)])\s/.test(raw)
|
||||
if (isListMarker) {
|
||||
inListItem = true
|
||||
prevWasProse = false
|
||||
return
|
||||
}
|
||||
if (inListItem) {
|
||||
// Indented under the item, or lazy continuation — still the list item.
|
||||
prevWasProse = false
|
||||
return
|
||||
}
|
||||
|
||||
if (!isProse(raw)) {
|
||||
prevWasProse = false
|
||||
return
|
||||
}
|
||||
|
||||
// A prose line. If the line before it was also prose, the paragraph spans
|
||||
// physical lines — a hard wrap.
|
||||
if (prevWasProse) {
|
||||
out.push({ file, line: i + 1, text: trimmed })
|
||||
}
|
||||
prevWasProse = true
|
||||
})
|
||||
|
||||
}
|
||||
visit(tree)
|
||||
return out
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user