fix(tools): escape unpaired surrogates and state the Cf boundary by category

U+00AD is 0xAD, so "Cf cannot be addressed by \xNN" was false for the
first example in its own list. The real boundary is the category: one
\xNN form covers Cc exactly, and escaping the single addressable Cf
member would leave a rule that is neither category- nor
addressability-shaped.

A lone surrogate is the NUL case rather than the invisible-character
case -- Python source must be UTF-8-encodable, and compile() raises
UnicodeEncodeError for one in a string literal or a # comment alike
(measured on 3.9). JSON.parse on a wire "\ud800" escape produces them,
so escape them as \uNNNN; the regex's u flag keeps well-formed astral
pairs intact.

Pin the whitespace-plus-surviving-control boundary, which also pins
trim-after-escape.

packages/core/tools/src/py-types.ts

@@ -84,16 +84,32 @@ interface RenderState {
  * declaration of the tools. The rest are legal but invisible; escaping them
  * with the same rule keeps the emitted text readable and the treatment uniform.
  *
- * The set stops at `Cc` because the escape is `\xNN`, which addresses exactly
- * U+0000 to U+00FF: the whole `Cc` block fits, and the invisible `Cf`
- * formatting characters (U+00AD soft hyphen, U+200B ZWSP, U+200E/U+200F bidi
- * marks, U+2060 word joiner) do not. `Cf` therefore passes through by design —
- * covering it would need a second `\uNNNN` escape form, and it is legal in both
- * consumers, since only LF and CR terminate a Python string literal or a `#`
- * comment.
+ * The boundary is the category, not per-code-point addressability: `\xNN`
+ * addresses U+0000 to U+00FF, so one escape form covers `Cc` exactly. The
+ * invisible `Cf` formatting characters pass through by design — of them only
+ * U+00AD soft hyphen would fit `\xNN` at all, and escaping that one while
+ * U+200B ZWSP, U+200E/U+200F bidi marks, and U+2060 word joiner passed through
+ * would leave a rule that is neither category- nor addressability-shaped. The
+ * whole family is legal in both consumers, since only LF and CR terminate a
+ * Python string literal or a `#` comment.
  */
 const UNPRINTABLE = /[\u0000-\u0008\u000e-\u001f\u007f-\u009f]/g
 
+/**
+ * Unpaired surrogate code points, escaped by {@link describe} as `\uNNNN` —
+ * its own form, since `\xNN` stops at U+00FF. The `u` flag is what makes this
+ * the LONE ones: in Unicode mode a well-formed pair is a single astral code
+ * point outside D800 to DFFF, so an emoji in a description survives untouched.
+ *
+ * This is the NUL case from {@link UNPRINTABLE}, not the invisible-character
+ * case. Python source must be UTF-8-encodable and a lone surrogate is not, so
+ * `compile()` raises `UnicodeEncodeError: surrogates not allowed` for one
+ * anywhere in the text — measured on 3.9 for a string literal and for a `#`
+ * comment alike. A raw or MCP tool description reaches this: `JSON.parse` on a
+ * wire `"\ud800"` escape yields exactly such a code point.
+ */
+const LONE_SURROGATE = /[\ud800-\udfff]/gu
+
 /**
  * The collapsed one-line `description` of a schema node (byte-stable across
  * formatting churn), or `undefined` when the node carries none. Every caller
@@ -106,7 +122,8 @@ const UNPRINTABLE = /[\u0000-\u0008\u000e-\u001f\u007f-\u009f]/g
  * NOT absent: it collapses to that character's visible escape.
  *
  * Control characters left over after the whitespace collapse are rendered as
- * their `\xNN` escapes (see {@link UNPRINTABLE}); the escape's own backslash is
+ * their `\xNN` escapes (see {@link UNPRINTABLE}) and unpaired surrogates as
+ * their `\uNNNN` escapes (see {@link LONE_SURROGATE}); the escape's own backslash is
  * emitted literally by both consumers, since {@link docLines} doubles it into a
  * Python source escape and a `#` comment carries it verbatim.
  */
@@ -116,6 +133,7 @@ function describe(schema: object): string | undefined {
   const collapsed = description
     .replace(/\s+/g, ' ')
     .replace(UNPRINTABLE, char => `\\x${char.charCodeAt(0).toString(16).padStart(2, '0')}`)
+    .replace(LONE_SURROGATE, char => `\\u${char.charCodeAt(0).toString(16).padStart(4, '0')}`)
     .trim()
   return collapsed.length === 0 ? undefined : collapsed
 }

packages/core/tools/tests/py-types.spec.ts

@@ -94,6 +94,15 @@ describe('renderToolsSdkPy', () => {
     parameters: parameterSchemaSpecToJsonSchema({}) as unknown as Record<string, unknown>,
     output: { type: 'string' },
   }
+  /** One tool carrying `description` at both emission sites: the method docstring and the field comment. */
+  const described = (description: string): ToolSdkSchema => ({
+    name: 'weird',
+    description,
+    parameters: parameterSchemaSpecToJsonSchema({
+      field: { type: 'string', required: true, description },
+    }) as unknown as Record<string, unknown>,
+    output: { type: 'string' },
+  })
 
   it('declares identifier tools as async methods and lists exotic/reserved names as subscript comments', () => {
     const text = renderToolsSdkPy([exotic, bash, reserved])
@@ -694,17 +703,11 @@ describe('renderToolsSdkPy', () => {
     // A description ending in `"` or an odd backslash would otherwise merge
     // with (or escape) the closing triple quote — and this block is Code
     // Mode's only SDK, so it must always parse.
-    const make = (description: string): ToolSdkSchema => ({
-      name: 'weird',
-      description,
-      parameters: parameterSchemaSpecToJsonSchema({}) as unknown as Record<string, unknown>,
-      output: { type: 'string' },
-    })
-    const trailingQuote = renderToolsSdkPy([make('ends in a quote"')])
+    const trailingQuote = renderToolsSdkPy([described('ends in a quote"')])
     expect(trailingQuote).toContain(String.raw`"""ends in a quote\""""`)
-    const trailingBackslash = renderToolsSdkPy([make('ends in a backslash\\')])
+    const trailingBackslash = renderToolsSdkPy([described('ends in a backslash\\')])
     expect(trailingBackslash).toContain(String.raw`"""ends in a backslash\\"""`)
-    const tripleQuote = renderToolsSdkPy([make('contains """ triple quote')])
+    const tripleQuote = renderToolsSdkPy([described('contains """ triple quote')])
     expect(tripleQuote).toContain(String.raw`"""contains \"\"\" triple quote"""`)
   })
 
@@ -716,15 +719,7 @@ describe('renderToolsSdkPy', () => {
     // from parsing at all. The whitespace collapse does not remove it (a NUL is
     // not whitespace). Rendering it as a visible escape keeps the source
     // parseable and still shows the model what the schema said.
-    const make = (description: string): ToolSdkSchema => ({
-      name: 'weird',
-      description,
-      parameters: parameterSchemaSpecToJsonSchema({
-        field: { type: 'string', required: true, description },
-      }) as unknown as Record<string, unknown>,
-      output: { type: 'string' },
-    })
-    const nul = renderToolsSdkPy([make('before\u0000after')])
+    const nul = renderToolsSdkPy([described('before\u0000after')])
     // Both emission sites: the method docstring and the `#` field comment. The
     // docstring's backslash is doubled by the same escaping that keeps a literal
     // backslash from escaping the closing triple quote, so Python parses it back
@@ -735,23 +730,44 @@ describe('renderToolsSdkPy', () => {
     // The other C0 controls and DEL escape on the same path. Tab, newline and
     // carriage return never reach it: the whitespace collapse folds them to a
     // space first.
-    const others = renderToolsSdkPy([make('bell\u0007esc\u001bdel\u007f')])
+    const others = renderToolsSdkPy([described('bell\u0007esc\u001bdel\u007f')])
     expect(others).toContain(String.raw`bell\x07esc\x1bdel\x7f`)
-    expect(renderToolsSdkPy([make('tab\tnewline\ncr\r')])).toContain('"""tab newline cr"""')
+    expect(renderToolsSdkPy([described('tab\tnewline\ncr\r')])).toContain('"""tab newline cr"""')
     // No C1 control is ECMAScript whitespace (TAB/VT/FF/SP/NBSP/ZWNBSP/Zs plus
     // LF/CR/LS/PS), so the collapse folds none of U+0080 to U+009F and the
     // escape is what keeps them out of the docstring, where they would be
     // invisible. NBSP, which IS whitespace, folds instead. Windows-1252 bytes
     // 0x80 to 0x9F decoded as Latin-1 land exactly here.
-    const nel = renderToolsSdkPy([make('a\u0085b')])
+    const nel = renderToolsSdkPy([described('a\u0085b')])
     expect(nel).not.toContain('\u0085')
     expect(nel).toContain(String.raw`# a\x85b`)
-    const c1 = renderToolsSdkPy([make('csi\u009bst\u009cend\u009f')])
+    const c1 = renderToolsSdkPy([described('csi\u009bst\u009cend\u009f')])
     expect(c1).toContain(String.raw`csi\x9bst\x9cend\x9f`)
-    expect(renderToolsSdkPy([make('nb\u00a0sp')])).toContain('"""nb sp"""')
-    // `Cf` formatting characters pass through by design: `\xNN` cannot address
-    // them, and they terminate neither a Python string literal nor a `#`
-    // comment, so the block stays parseable with the code point intact.
-    expect(renderToolsSdkPy([make('zero\u200bwidth')])).toContain('"""zero\u200bwidth"""')
+    expect(renderToolsSdkPy([described('nb\u00a0sp')])).toContain('"""nb sp"""')
+    // `Cf` formatting characters pass through by category, not by
+    // addressability — U+00AD would fit `\xNN`, the rest would need a second
+    // form. They terminate neither a Python string literal nor a `#` comment,
+    // so the block stays parseable with the code point intact.
+    expect(renderToolsSdkPy([described('zero\u200bwidth')])).toContain('"""zero\u200bwidth"""')
+    // Whitespace around a surviving control character is not an absent
+    // description: the escape runs before the trim, so what is left is visible.
+    expect(renderToolsSdkPy([described(' \u0085 ')])).toContain(String.raw`# \x85`)
+  })
+
+  it('escapes unpaired surrogates, which make the source impossible to encode', () => {
+    // This is the NUL case, not the invisible-character case: Python source
+    // must be UTF-8-encodable, and `compile()` raises `UnicodeEncodeError:
+    // surrogates not allowed` for a lone surrogate in a string literal and in
+    // a `#` comment alike, so one would stop this block — Code Mode's only SDK
+    // — from parsing. A wire description reaches it: `JSON.parse` on a
+    // `"\ud800"` escape yields exactly this code point.
+    const high = renderToolsSdkPy([described('a\ud800b')])
+    expect(high).not.toContain('\ud800')
+    expect(high).toContain(String.raw`# a\ud800b`)
+    // A lone LOW surrogate is just as unencodable, and `\xNN` reaches neither.
+    expect(renderToolsSdkPy([described('a\udfffb')])).toContain(String.raw`# a\udfffb`)
+    // A well-formed pair is ONE astral code point, not two surrogates — the
+    // regex's `u` flag is what draws that line, so an emoji survives intact.
+    expect(renderToolsSdkPy([described('emoji \u{1f600} ok')])).toContain('"""emoji \u{1f600} ok"""')
   })
 })