diff --git a/packages/core/tools/src/py-types.ts b/packages/core/tools/src/py-types.ts index 952437eaa7..a74729d9ff 100644 --- a/packages/core/tools/src/py-types.ts +++ b/packages/core/tools/src/py-types.ts @@ -84,16 +84,32 @@ interface RenderState { * declaration of the tools. The rest are legal but invisible; escaping them * with the same rule keeps the emitted text readable and the treatment uniform. * - * The set stops at `Cc` because the escape is `\xNN`, which addresses exactly - * U+0000 to U+00FF: the whole `Cc` block fits, and the invisible `Cf` - * formatting characters (U+00AD soft hyphen, U+200B ZWSP, U+200E/U+200F bidi - * marks, U+2060 word joiner) do not. `Cf` therefore passes through by design — - * covering it would need a second `\uNNNN` escape form, and it is legal in both - * consumers, since only LF and CR terminate a Python string literal or a `#` - * comment. + * The boundary is the category, not per-code-point addressability: `\xNN` + * addresses U+0000 to U+00FF, so one escape form covers `Cc` exactly. The + * invisible `Cf` formatting characters pass through by design — of them only + * U+00AD soft hyphen would fit `\xNN` at all, and escaping that one while + * U+200B ZWSP, U+200E/U+200F bidi marks, and U+2060 word joiner passed through + * would leave a rule that is neither category- nor addressability-shaped. The + * whole family is legal in both consumers, since only LF and CR terminate a + * Python string literal or a `#` comment. */ const UNPRINTABLE = /[\u0000-\u0008\u000e-\u001f\u007f-\u009f]/g +/** + * Unpaired surrogate code points, escaped by {@link describe} as `\uNNNN` — + * its own form, since `\xNN` stops at U+00FF. The `u` flag is what makes this + * the LONE ones: in Unicode mode a well-formed pair is a single astral code + * point outside D800 to DFFF, so an emoji in a description survives untouched. + * + * This is the NUL case from {@link UNPRINTABLE}, not the invisible-character + * case. Python source must be UTF-8-encodable and a lone surrogate is not, so + * `compile()` raises `UnicodeEncodeError: surrogates not allowed` for one + * anywhere in the text — measured on 3.9 for a string literal and for a `#` + * comment alike. A raw or MCP tool description reaches this: `JSON.parse` on a + * wire `"\ud800"` escape yields exactly such a code point. + */ +const LONE_SURROGATE = /[\ud800-\udfff]/gu + /** * The collapsed one-line `description` of a schema node (byte-stable across * formatting churn), or `undefined` when the node carries none. Every caller @@ -106,7 +122,8 @@ const UNPRINTABLE = /[\u0000-\u0008\u000e-\u001f\u007f-\u009f]/g * NOT absent: it collapses to that character's visible escape. * * Control characters left over after the whitespace collapse are rendered as - * their `\xNN` escapes (see {@link UNPRINTABLE}); the escape's own backslash is + * their `\xNN` escapes (see {@link UNPRINTABLE}) and unpaired surrogates as + * their `\uNNNN` escapes (see {@link LONE_SURROGATE}); the escape's own backslash is * emitted literally by both consumers, since {@link docLines} doubles it into a * Python source escape and a `#` comment carries it verbatim. */ @@ -116,6 +133,7 @@ function describe(schema: object): string | undefined { const collapsed = description .replace(/\s+/g, ' ') .replace(UNPRINTABLE, char => `\\x${char.charCodeAt(0).toString(16).padStart(2, '0')}`) + .replace(LONE_SURROGATE, char => `\\u${char.charCodeAt(0).toString(16).padStart(4, '0')}`) .trim() return collapsed.length === 0 ? undefined : collapsed } diff --git a/packages/core/tools/tests/py-types.spec.ts b/packages/core/tools/tests/py-types.spec.ts index 78bcab6d23..61379d22ce 100644 --- a/packages/core/tools/tests/py-types.spec.ts +++ b/packages/core/tools/tests/py-types.spec.ts @@ -94,6 +94,15 @@ describe('renderToolsSdkPy', () => { parameters: parameterSchemaSpecToJsonSchema({}) as unknown as Record, output: { type: 'string' }, } + /** One tool carrying `description` at both emission sites: the method docstring and the field comment. */ + const described = (description: string): ToolSdkSchema => ({ + name: 'weird', + description, + parameters: parameterSchemaSpecToJsonSchema({ + field: { type: 'string', required: true, description }, + }) as unknown as Record, + output: { type: 'string' }, + }) it('declares identifier tools as async methods and lists exotic/reserved names as subscript comments', () => { const text = renderToolsSdkPy([exotic, bash, reserved]) @@ -694,17 +703,11 @@ describe('renderToolsSdkPy', () => { // A description ending in `"` or an odd backslash would otherwise merge // with (or escape) the closing triple quote — and this block is Code // Mode's only SDK, so it must always parse. - const make = (description: string): ToolSdkSchema => ({ - name: 'weird', - description, - parameters: parameterSchemaSpecToJsonSchema({}) as unknown as Record, - output: { type: 'string' }, - }) - const trailingQuote = renderToolsSdkPy([make('ends in a quote"')]) + const trailingQuote = renderToolsSdkPy([described('ends in a quote"')]) expect(trailingQuote).toContain(String.raw`"""ends in a quote\""""`) - const trailingBackslash = renderToolsSdkPy([make('ends in a backslash\\')]) + const trailingBackslash = renderToolsSdkPy([described('ends in a backslash\\')]) expect(trailingBackslash).toContain(String.raw`"""ends in a backslash\\"""`) - const tripleQuote = renderToolsSdkPy([make('contains """ triple quote')]) + const tripleQuote = renderToolsSdkPy([described('contains """ triple quote')]) expect(tripleQuote).toContain(String.raw`"""contains \"\"\" triple quote"""`) }) @@ -716,15 +719,7 @@ describe('renderToolsSdkPy', () => { // from parsing at all. The whitespace collapse does not remove it (a NUL is // not whitespace). Rendering it as a visible escape keeps the source // parseable and still shows the model what the schema said. - const make = (description: string): ToolSdkSchema => ({ - name: 'weird', - description, - parameters: parameterSchemaSpecToJsonSchema({ - field: { type: 'string', required: true, description }, - }) as unknown as Record, - output: { type: 'string' }, - }) - const nul = renderToolsSdkPy([make('before\u0000after')]) + const nul = renderToolsSdkPy([described('before\u0000after')]) // Both emission sites: the method docstring and the `#` field comment. The // docstring's backslash is doubled by the same escaping that keeps a literal // backslash from escaping the closing triple quote, so Python parses it back @@ -735,23 +730,44 @@ describe('renderToolsSdkPy', () => { // The other C0 controls and DEL escape on the same path. Tab, newline and // carriage return never reach it: the whitespace collapse folds them to a // space first. - const others = renderToolsSdkPy([make('bell\u0007esc\u001bdel\u007f')]) + const others = renderToolsSdkPy([described('bell\u0007esc\u001bdel\u007f')]) expect(others).toContain(String.raw`bell\x07esc\x1bdel\x7f`) - expect(renderToolsSdkPy([make('tab\tnewline\ncr\r')])).toContain('"""tab newline cr"""') + expect(renderToolsSdkPy([described('tab\tnewline\ncr\r')])).toContain('"""tab newline cr"""') // No C1 control is ECMAScript whitespace (TAB/VT/FF/SP/NBSP/ZWNBSP/Zs plus // LF/CR/LS/PS), so the collapse folds none of U+0080 to U+009F and the // escape is what keeps them out of the docstring, where they would be // invisible. NBSP, which IS whitespace, folds instead. Windows-1252 bytes // 0x80 to 0x9F decoded as Latin-1 land exactly here. - const nel = renderToolsSdkPy([make('a\u0085b')]) + const nel = renderToolsSdkPy([described('a\u0085b')]) expect(nel).not.toContain('\u0085') expect(nel).toContain(String.raw`# a\x85b`) - const c1 = renderToolsSdkPy([make('csi\u009bst\u009cend\u009f')]) + const c1 = renderToolsSdkPy([described('csi\u009bst\u009cend\u009f')]) expect(c1).toContain(String.raw`csi\x9bst\x9cend\x9f`) - expect(renderToolsSdkPy([make('nb\u00a0sp')])).toContain('"""nb sp"""') - // `Cf` formatting characters pass through by design: `\xNN` cannot address - // them, and they terminate neither a Python string literal nor a `#` - // comment, so the block stays parseable with the code point intact. - expect(renderToolsSdkPy([make('zero\u200bwidth')])).toContain('"""zero\u200bwidth"""') + expect(renderToolsSdkPy([described('nb\u00a0sp')])).toContain('"""nb sp"""') + // `Cf` formatting characters pass through by category, not by + // addressability — U+00AD would fit `\xNN`, the rest would need a second + // form. They terminate neither a Python string literal nor a `#` comment, + // so the block stays parseable with the code point intact. + expect(renderToolsSdkPy([described('zero\u200bwidth')])).toContain('"""zero\u200bwidth"""') + // Whitespace around a surviving control character is not an absent + // description: the escape runs before the trim, so what is left is visible. + expect(renderToolsSdkPy([described(' \u0085 ')])).toContain(String.raw`# \x85`) + }) + + it('escapes unpaired surrogates, which make the source impossible to encode', () => { + // This is the NUL case, not the invisible-character case: Python source + // must be UTF-8-encodable, and `compile()` raises `UnicodeEncodeError: + // surrogates not allowed` for a lone surrogate in a string literal and in + // a `#` comment alike, so one would stop this block — Code Mode's only SDK + // — from parsing. A wire description reaches it: `JSON.parse` on a + // `"\ud800"` escape yields exactly this code point. + const high = renderToolsSdkPy([described('a\ud800b')]) + expect(high).not.toContain('\ud800') + expect(high).toContain(String.raw`# a\ud800b`) + // A lone LOW surrogate is just as unencodable, and `\xNN` reaches neither. + expect(renderToolsSdkPy([described('a\udfffb')])).toContain(String.raw`# a\udfffb`) + // A well-formed pair is ONE astral code point, not two surrogates — the + // regex's `u` flag is what draws that line, so an emoji survives intact. + expect(renderToolsSdkPy([described('emoji \u{1f600} ok')])).toContain('"""emoji \u{1f600} ok"""') }) })