From 308f5ae0f35548946e4db8f2f05de1822394876f Mon Sep 17 00:00:00 2001 From: Chinesezjc Date: Wed, 5 Aug 2026 17:59:48 +0800 Subject: [PATCH] fix(tools): escape the whole C1 control range, not just NEL Unicode Cc is U+0000-U+001F plus U+007F-U+009F, and no C1 code point is ECMAScript whitespace, so U+0080-U+009F all survived the collapse and reached the docstring raw and invisible -- the gap the previous commit closed for NEL alone. \xNN addresses the whole block, which is the same reason the set stops at Cc, so widen the class to U+009F and pin U+009B/U+009C/U+009F. Windows-1252 bytes 0x80-0x9F decoded as Latin-1 produce exactly these. Also: required TypedDict fields share the optional fields' listDepth start, and a description of whitespace plus a surviving control character is not absent. --- packages/core/tools/src/py-types.ts | 32 ++++++++++++++-------- packages/core/tools/tests/py-types.spec.ts | 11 +++++--- 2 files changed, 27 insertions(+), 16 deletions(-) diff --git a/packages/core/tools/src/py-types.ts b/packages/core/tools/src/py-types.ts index d85660ee86..952437eaa7 100644 --- a/packages/core/tools/src/py-types.ts +++ b/packages/core/tools/src/py-types.ts @@ -71,10 +71,13 @@ interface RenderState { /** * The `Cc` code points that survive the whitespace collapse in {@link describe} - * and have no printable form: the C0 controls, DEL, and NEL. U+0009 to U+000D - * are absent because ECMAScript `\s` already collapsed them; U+0085 is `Cc` but - * NOT in `\s` (TAB/VT/FF/SP/NBSP/ZWNBSP/Zs plus LF/CR/LS/PS), so it survives and - * is escaped here. CPython rejects source containing a NUL outright + * and have no printable form: the C0 controls, DEL, and the C1 controls. Only + * U+0009 to U+000D are absent, because ECMAScript `\s` already collapsed them — + * `\s` is TAB/VT/FF/SP/NBSP/ZWNBSP/Zs plus LF/CR/LS/PS, so no C1 code point is + * in it and the whole U+0080 to U+009F block reaches this rule intact. Those + * are not hypothetical input: they are what Windows-1252 bytes 0x80 to 0x9F + * (smart quotes, em dash) become when decoded as Latin-1. + * CPython rejects source containing a NUL outright * (`SyntaxError: source code string cannot contain null bytes`), whether it * sits in a docstring or in a comment, so one such byte anywhere in a schema * description would make the whole generated SDK unparseable — the model's only @@ -82,13 +85,14 @@ interface RenderState { * with the same rule keeps the emitted text readable and the treatment uniform. * * The set stops at `Cc` because the escape is `\xNN`, which addresses exactly - * U+0000 to U+00FF. The invisible `Cf` formatting characters (U+00AD soft - * hyphen, U+200B ZWSP, U+200E/U+200F bidi marks, U+2060 word joiner) pass - * through by design: covering them would need a second `\uNNNN` escape form, - * and they are legal in both consumers — only LF and CR terminate a Python - * string literal or a `#` comment. + * U+0000 to U+00FF: the whole `Cc` block fits, and the invisible `Cf` + * formatting characters (U+00AD soft hyphen, U+200B ZWSP, U+200E/U+200F bidi + * marks, U+2060 word joiner) do not. `Cf` therefore passes through by design — + * covering it would need a second `\uNNNN` escape form, and it is legal in both + * consumers, since only LF and CR terminate a Python string literal or a `#` + * comment. */ -const UNPRINTABLE = /[\u0000-\u0008\u000e-\u001f\u007f\u0085]/g +const UNPRINTABLE = /[\u0000-\u0008\u000e-\u001f\u007f-\u009f]/g /** * The collapsed one-line `description` of a schema node (byte-stable across @@ -97,7 +101,9 @@ const UNPRINTABLE = /[\u0000-\u0008\u000e-\u001f\u007f\u0085]/g * so only the description field needs guarding. A description that collapses * to nothing (empty, or whitespace only) is `undefined` too: it documents the * node no better than an absent one, and emitting it would leave an empty - * `"""` docstring or a bare `# ` line in the SDK. + * `"""` docstring or a bare `# ` line in the SDK. Only ECMAScript whitespace + * folds, so a description of whitespace plus one surviving control character is + * NOT absent: it collapses to that character's visible escape. * * Control characters left over after the whitespace collapse are rendered as * their `\xNN` escapes (see {@link UNPRINTABLE}); the escape's own backslash is @@ -157,7 +163,9 @@ const MAX_CLASS_NAME_BASE = 120 * before the `->`, so it is NOT open here: 181. * - TypedDict field, `field: NotRequired[chain]` — a class-body line with no * other open bracket, and its children start at `listDepth: 1` to reserve - * the `NotRequired[`, so 179 `list[` plus `Literal[`: 181. + * the `NotRequired[`, so 179 `list[` plus `Literal[`: 181. Required fields + * share that start for uniformity, spending one level of representable depth + * on a bracket they never emit. * - Argument annotation, `async def f(self, args: chain) -> Y:` — the `(` IS * still open around it: 180 `list[` plus `Literal[` plus the paren, 182, the * worst case. Reachable only through a raw `register()` whose `parameters` diff --git a/packages/core/tools/tests/py-types.spec.ts b/packages/core/tools/tests/py-types.spec.ts index 158c043909..78bcab6d23 100644 --- a/packages/core/tools/tests/py-types.spec.ts +++ b/packages/core/tools/tests/py-types.spec.ts @@ -738,13 +738,16 @@ describe('renderToolsSdkPy', () => { const others = renderToolsSdkPy([make('bell\u0007esc\u001bdel\u007f')]) expect(others).toContain(String.raw`bell\x07esc\x1bdel\x7f`) expect(renderToolsSdkPy([make('tab\tnewline\ncr\r')])).toContain('"""tab newline cr"""') - // NEL is the one `Cc` code point the collapse does NOT fold: ECMAScript - // whitespace is TAB/VT/FF/SP/NBSP/ZWNBSP/Zs plus LF/CR/LS/PS, and U+0085 is - // in none of them, so without the escape it would reach the docstring raw - // and be invisible there. NBSP, which IS whitespace, folds instead. + // No C1 control is ECMAScript whitespace (TAB/VT/FF/SP/NBSP/ZWNBSP/Zs plus + // LF/CR/LS/PS), so the collapse folds none of U+0080 to U+009F and the + // escape is what keeps them out of the docstring, where they would be + // invisible. NBSP, which IS whitespace, folds instead. Windows-1252 bytes + // 0x80 to 0x9F decoded as Latin-1 land exactly here. const nel = renderToolsSdkPy([make('a\u0085b')]) expect(nel).not.toContain('\u0085') expect(nel).toContain(String.raw`# a\x85b`) + const c1 = renderToolsSdkPy([make('csi\u009bst\u009cend\u009f')]) + expect(c1).toContain(String.raw`csi\x9bst\x9cend\x9f`) expect(renderToolsSdkPy([make('nb\u00a0sp')])).toContain('"""nb sp"""') // `Cf` formatting characters pass through by design: `\xNN` cannot address // them, and they terminate neither a Python string literal nor a `#`