From c5b09c108f4d19c6ceb05b0302b8a979ab0d51d4 Mon Sep 17 00:00:00 2001
From: Chinesezjc <jczhai@deepseek.com>
Date: Wed, 5 Aug 2026 22:25:45 +0800
Subject: [PATCH] docs(tools): record LS/PS as tokenizer non-terminators, with
 a test

Review read `JSON.stringify`'s raw pass-through of U+0085/U+2028/U+2029 as a
parse hazard: an LS in a `Literal[...]` value or in a `# tools["..."]` comment
would end the physical line and take the SDK block down. Measured on CPython
3.9.6 (Unicode 13.0) and 3.12.13 (15.0): all three are accepted in both a
string literal and a `#` comment, value round-tripping, and only LF and CR
terminate either. The set is the tokenizer's, not `str.splitlines()`'.

Both existing claims were accurate, so nothing changes behaviorally. Name the
distinction where it was assumed: `UNPRINTABLE`'s terminator sentence now says
which set it means, and `pyScalar`'s raw-pass-through list, previously "DEL and
the C1 controls", now also names LS/PS, which are neither. A test pins the raw
form for `const` and `enum` so escaping them later cannot land as a silent
divergence from the TypeScript flavor.
---
 packages/core/tools/src/py-types.ts        | 16 ++++++++++++----
 packages/core/tools/tests/py-types.spec.ts | 13 +++++++++++++
 2 files changed, 25 insertions(+), 4 deletions(-)

diff --git a/packages/core/tools/src/py-types.ts b/packages/core/tools/src/py-types.ts
index 9c4aeabafc..0664fe1443 100644
--- a/packages/core/tools/src/py-types.ts
+++ b/packages/core/tools/src/py-types.ts
@@ -167,7 +167,12 @@ interface RenderState {
  * U+200B ZWSP, U+200E/U+200F bidi marks, and U+2060 word joiner passed through
  * would leave a rule that is neither category- nor addressability-shaped. The
  * whole family is legal in both consumers, since only LF and CR terminate a
- * Python string literal or a `#` comment.
+ * Python string literal or a `#` comment. That set is the tokenizer's, not
+ * `str.splitlines()`': NEL (U+0085), LS (U+2028), and PS (U+2029) split a
+ * string at run time but do not end a physical line in source — measured on
+ * CPython 3.9.6 and 3.12.13, each accepted in both positions with the value
+ * round-tripping — so they are safe raw wherever they reach emitted text
+ * unescaped, which for LS and PS is {@link pyScalar}'s `JSON.stringify`.
  */
 const UNPRINTABLE = /[\u0000-\u0008\u000e-\u001f\u007f-\u009f]/g
 
@@ -408,9 +413,12 @@ function childClassName(base: string, segment: string): string {
  * That leans on a coincidence worth naming: every escape `JSON.stringify` can
  * emit (`\"`, `\\`, `\b`, `\f`, `\n`, `\r`, `\t`, `\uXXXX`) is also a Python
  * escape denoting the same character, so the emitted `Literal[...]` both
- * parses and decodes back to the value the schema declared. DEL and the C1
- * controls do reach it raw — legal but invisible, byte-for-byte as in the TS
- * flavor; escaping them is a both-flavors change. The subscript tool-name
+ * parses and decodes back to the value the schema declared. DEL, the C1
+ * controls, and LS/PS (U+2028/U+2029) do reach it raw — legal but invisible,
+ * byte-for-byte as in the TS flavor; escaping them is a both-flavors change.
+ * LS and PS are legal here for the reason {@link UNPRINTABLE} records: they
+ * are `str.splitlines()` boundaries, not tokenizer line terminators. The
+ * subscript tool-name
  * comment quotes its name through its own call to the same `JSON.stringify`,
  * never through this function, and inherits both halves — escapes and
  * pass-throughs alike.
diff --git a/packages/core/tools/tests/py-types.spec.ts b/packages/core/tools/tests/py-types.spec.ts
index b584058765..259a1ec7f9 100644
--- a/packages/core/tools/tests/py-types.spec.ts
+++ b/packages/core/tools/tests/py-types.spec.ts
@@ -67,6 +67,19 @@ describe('jsonSchemaToPy', () => {
     expect(jsonSchemaToPy({ type: 'string', const: 'ends\\' })).toBe(String.raw`Literal["ends\\"]`)
   })
 
+  it('passes the paragraph separators through raw, which CPython does not treat as line terminators', () => {
+    // `JSON.stringify` escapes LF and CR but not LS/PS (U+2028/U+2029), which
+    // is safe here and not by accident: they are `str.splitlines()` boundaries,
+    // not tokenizer line terminators, so they end neither a string literal nor
+    // a `#` comment — measured on CPython 3.9.6 and 3.12.13. Pinning the raw
+    // form keeps a later "escape them for symmetry with LF" change from
+    // landing as a silent both-flavors divergence from `ts-types`.
+    // Escapes below — the two forms denote the same bytes, and neither
+    // character has a visible width.
+    expect(jsonSchemaToPy({ type: 'string', const: 'a\u2028b' })).toBe('Literal["a\u2028b"]')
+    expect(jsonSchemaToPy({ type: 'string', enum: ['a\u2029b'] })).toBe('Literal["a\u2029b"]')
+  })
+
   it('emits exact digits for a beyond-safe-range integer literal', () => {
     // Python integers are arbitrary-precision, so the emitted digits ARE the
     // value the model programs against. `String(2 ** 60)` prints the rounded