/** * The client's ONE syntax highlighter: a synchronous fine-grained shiki core * (JavaScript regex engine — no oniguruma WASM, bundle-friendly) with an * explicit grammar allowlist and a CSS-variables theme. Colors live in the * theme package's token sheets as `--shiki-*` custom properties (light and * dark blocks), never here — the repo's tokens-only styling rule. * * Only the three markdown-fence and `run_code` grammars (TypeScript, shell, * JSON) load into the singleton at boot — the set every session renders. The * read card's wider extension set (the file-extension language hints the read * tool's `langFromPath` emits — `packages/fs/tool-fs`: python, rust, yaml, * markup, …) is imported lazily and registered the first time such a language * is requested, so a session that never opens a read card in one of those * languages pays neither the ~1.6 MB of grammar modules nor their synchronous * init. The first render of a lazy language falls back to plain text while its * grammar loads, then {@link onGrammarLoaded} notifies subscribers to re-render * with highlighting. An unknown or absent language falls back to plain text (no * highlighting, still monospace) — never an error. */ import { createHighlighterCoreSync, createCssVariablesTheme } from 'shiki/core' import { createJavaScriptRegexEngine } from 'shiki/engine/javascript' import langTs from '@shikijs/langs/typescript' import langBash from '@shikijs/langs/shellscript' import langJson from '@shikijs/langs/json' import type { HighlighterCore } from 'shiki/core' import type { CSSProperties } from 'react' /** A shiki grammar module's default export (a `LanguageRegistration[]`), taken * from a boot grammar so no direct `@shikijs/types` dependency is needed. */ type LangModule = { default: typeof langTs } /** * Grammars the singleton loads at boot; each entry's own `name` is the id * `codeToTokens`/`codeToHtml` resolve. The JS-family aliases (js/jsx/ts/tsx) * resolve to the TypeScript grammar rather than a separate one: it tokenizes * plain TS/JS exactly, and JSX/TSX approximately (shiki's TS grammar is not the * dedicated TSX grammar, so JSX elements tokenize imperfectly) — an accepted * trade to keep the boot set to one JS-family grammar. The read card's wider * set loads lazily through {@link LAZY_GRAMMARS}. */ const LANGS = [langTs, langBash, langJson] /** * The read card's extension grammars, each behind a dynamic import so its * module stays out of the boot chunk until a read of that language renders. * Keyed by the grammar id (`LanguageRegistration.name`) the aliases resolve to. * `@shikijs/langs`' default export is a `LanguageRegistration[]`; the loader * hands the whole array to `loadLanguageSync`, which registers each entry * (including embedded sub-grammars). The three boot grammars are absent — * already loaded, so no alias value ever points at a missing entry here. */ const LAZY_GRAMMARS = new Map Promise>([ ['python', () => import('@shikijs/langs/python')], ['ruby', () => import('@shikijs/langs/ruby')], ['go', () => import('@shikijs/langs/go')], ['rust', () => import('@shikijs/langs/rust')], ['java', () => import('@shikijs/langs/java')], ['c', () => import('@shikijs/langs/c')], ['cpp', () => import('@shikijs/langs/cpp')], ['csharp', () => import('@shikijs/langs/csharp')], ['kotlin', () => import('@shikijs/langs/kotlin')], ['swift', () => import('@shikijs/langs/swift')], ['php', () => import('@shikijs/langs/php')], ['yaml', () => import('@shikijs/langs/yaml')], ['toml', () => import('@shikijs/langs/toml')], ['ini', () => import('@shikijs/langs/ini')], ['markdown', () => import('@shikijs/langs/markdown')], ['mdx', () => import('@shikijs/langs/mdx')], ['html', () => import('@shikijs/langs/html')], ['css', () => import('@shikijs/langs/css')], ['scss', () => import('@shikijs/langs/scss')], ['less', () => import('@shikijs/langs/less')], ['sql', () => import('@shikijs/langs/sql')], ['xml', () => import('@shikijs/langs/xml')], ['lua', () => import('@shikijs/langs/lua')], ]) /** * Language ids (and aliases) the highlighter accepts; everything else renders * plain. A Map, not an object: fence info strings are assistant-authored, so * a label like `constructor` or `__proto__` must miss instead of resolving an * inherited property and crashing the renderer inside shiki. Keys cover both * the markdown-fence aliases `CodeBlock` uses and the file-extension hint ids * the read tool's `langFromPath` emits, so both callers resolve the same * grammars. The JS family maps to the TypeScript grammar (see {@link LANGS} for * the JSX/TSX approximation), unchanged from when this was the only * non-shell/JSON grammar. A value not in {@link LANGS} names a * {@link LAZY_GRAMMARS} entry loaded on first use. */ const LANG_ALIASES = new Map([ ['typescript', 'typescript'], ['ts', 'typescript'], ['tsx', 'typescript'], ['javascript', 'typescript'], ['js', 'typescript'], ['jsx', 'typescript'], ['shellscript', 'shellscript'], ['bash', 'shellscript'], ['sh', 'shellscript'], ['shell', 'shellscript'], ['zsh', 'shellscript'], ['json', 'json'], ['jsonc', 'json'], ['py', 'python'], ['python', 'python'], ['rb', 'ruby'], ['ruby', 'ruby'], ['go', 'go'], ['rs', 'rust'], ['rust', 'rust'], ['java', 'java'], ['c', 'c'], ['cpp', 'cpp'], ['cs', 'csharp'], ['csharp', 'csharp'], ['kotlin', 'kotlin'], ['swift', 'swift'], ['php', 'php'], ['yaml', 'yaml'], ['yml', 'yaml'], ['toml', 'toml'], ['ini', 'ini'], ['md', 'markdown'], ['markdown', 'markdown'], ['mdx', 'mdx'], ['html', 'html'], ['css', 'css'], ['scss', 'scss'], ['less', 'less'], ['sql', 'sql'], ['xml', 'xml'], ['lua', 'lua'], ]) /** All token colors resolve through `--shiki-*` custom properties (theme package sheets). */ const cssVariablesTheme = createCssVariablesTheme({ name: 'css-variables', variablePrefix: '--shiki-', fontStyle: true, }) let singleton: HighlighterCore | undefined /** The synchronous highlighter (one instance per document); pre-warmed below, lazy as the fallback. */ function highlighter(): HighlighterCore { singleton ??= createHighlighterCoreSync({ themes: [cssVariablesTheme], langs: LANGS, engine: createJavaScriptRegexEngine({ forgiving: true }), }) return singleton } /** Grammar ids whose lazy import is in flight or done, so it is requested once. */ const requested = new Set() /** Subscribers re-rendered after a lazy grammar registers (React callers). */ const listeners = new Set<() => void>() /** Bumped on each lazy-grammar load; the `useSyncExternalStore` snapshot. */ let loadCount = 0 /** * Subscribe to lazy-grammar load completions; `listener` fires after a * {@link LAZY_GRAMMARS} grammar finishes registering on the singleton, so a * caller that rendered its plain fallback while the grammar loaded can * re-highlight. Shaped as a `useSyncExternalStore` subscribe: pair it with * {@link grammarLoadCount} as the snapshot. Returns an unsubscribe function. * @param listener - invoked (no args) on each grammar-load completion. * @returns a disposer that removes the listener. */ export function subscribeGrammarLoaded(listener: () => void): () => void { listeners.add(listener) return () => { listeners.delete(listener) } } /** * The lazy-grammar load counter — a value that changes on every load, so a * `useSyncExternalStore` snapshot re-renders the subscriber when a grammar * registers. Opaque: only its identity across renders matters. * @returns the current load count. */ export function grammarLoadCount(): number { return loadCount } /** * Ensure the grammar `resolved` names is registered. A boot grammar (not in * {@link LAZY_GRAMMARS}) and an already-loaded lazy grammar report ready * synchronously; a lazy grammar not yet loaded starts its import (once) and * reports not-ready, so the caller renders plain until a * {@link subscribeGrammarLoaded} listener fires. * @param resolved - the grammar id an alias resolved to. * @returns whether the grammar is registered and ready to tokenize now. */ function ensureGrammar(resolved: string): boolean { const load = LAZY_GRAMMARS.get(resolved) // A boot grammar (already registered) has no lazy loader; it is always ready. if (load === undefined) return true if (highlighter().getLoadedLanguages().includes(resolved)) return true if (!requested.has(resolved)) { requested.add(resolved) void load().then((mod) => { highlighter().loadLanguageSync(mod.default) loadCount += 1 for (const listener of listeners) listener() }) } return false } // Engine + grammar construction costs a long task (~120-175ms); building it // during the first finalized fence's render would jank exactly when a stream // completes. Warm the singleton in a deferred task at module load (= plugin // boot) instead; the lazy path above stays as the correctness fallback for a // fence that renders before the timer fires. `unref` (Node-only) keeps a // non-browser import from pinning the event loop. const warmupTimer = setTimeout(() => { highlighter() }, 0) ;(warmupTimer as { unref?: () => void }).unref?.() /** * Highlight `code` into shiki's HTML (a single `
` tree)
 * when `lang` maps to a registered grammar; `undefined` means the caller
 * renders its plain fallback. A lazy grammar not yet loaded returns `undefined`
 * for this call and loads in the background; subscribe with
 * {@link onGrammarLoaded} to re-highlight once it registers.
 * @param code - the source text.
 * @param lang - the language hint (a markdown fence info string or a fixed caller id).
 * @returns the highlighted HTML, or `undefined` for unknown or not-yet-loaded languages.
 */
export function highlightToHtml(code: string, lang: string | undefined): string | undefined {
  const resolved = lang === undefined ? undefined : LANG_ALIASES.get(lang.toLowerCase())
  if (resolved === undefined) return undefined
  if (!ensureGrammar(resolved)) return undefined
  return highlighter().codeToHtml(code, { lang: resolved, theme: 'css-variables' })
}

/**
 * One highlighted run of a line: the text and the inline style shiki assigned
 * it. The css-variables theme colors every run through a `--shiki-*` custom
 * property, so `style.color` is always present; it is held as a style object
 * rather than a bare color so a run spreads onto a `` uniformly.
 */
export interface HighlightSpan {
  text: string
  style: CSSProperties
}

/**
 * Tokenize `code` into per-line highlighted runs when `lang` maps to a
 * registered grammar; `undefined` means the caller renders its plain fallback.
 * A line-numbered view needs the token runs split per line (one gutter number
 * per line), which the single-`
` {@link highlightToHtml} does not expose,
 * so this returns shiki's own 2D line/token structure narrowed to what a run
 * renders. Each run's color is a `--shiki-*` custom property, keeping token
 * colors on the theme package's sheets exactly as the HTML path does; the
 * css-variables theme carries no font-style bits, matching that path's
 * color-only output. The trailing newline shiki appends as a final empty line
 * is dropped so the run count matches the caller's own line array.
 * @param code - the source text.
 * @param lang - the language hint (a file-extension-derived language id).
 * @returns one entry per source line (each an array of runs), or `undefined` for unknown or not-yet-loaded languages.
 */
export function highlightLines(code: string, lang: string | undefined): HighlightSpan[][] | undefined {
  const resolved = lang === undefined ? undefined : LANG_ALIASES.get(lang.toLowerCase())
  if (resolved === undefined) return undefined
  if (!ensureGrammar(resolved)) return undefined
  const { tokens } = highlighter().codeToTokens(code, { lang: resolved, theme: 'css-variables' })
  // shiki tokenizes `a\nb` into two lines; a trailing newline (`a\n`) adds a
  // third, empty line the caller's own line array does not carry. Drop that
  // one terminator line so the two structures stay in step. The explicit
  // `last !== undefined` (over `tokens[...]?.length`) keeps a single branch for
  // per-file coverage, matching TerminalBlock's terminator check.
  const last = tokens[tokens.length - 1]
  const lines = tokens.length > 1 && last !== undefined && last.length === 0
    ? tokens.slice(0, -1)
    : tokens
  return lines.map(line => line.map(token => ({ text: token.content, style: { color: token.color } })))
}