feat(tools): accept Unicode Python identifiers in the Python SDK renderer

The identifier test was ASCII-only, so an object with a `路径` field
degraded to dict[str, Any] -- dropping every sibling field's name,
requiredness and type, with no native schema behind it in Code Mode to
carry them. Python identifiers are `xid_start xid_continue*`, so match
that instead, and widen camelCase's split and head check to the same
sets (naming `_` explicitly in the split, since it is XID_Continue).

NFKC stability is a second and separate condition. CPython normalizes
identifiers at compile time while a JSON key is compared as written, so
a U+FB01 ligature key would be declared and reachable under its ASCII
expansion, a key the tool never accepts, and two keys that normalize
together would collapse
into one declaration. Those names take the subscript path. Generated
class names are normalized instead of rejected -- they are never matched
against a key. Astral characters can now reach the class-name cap, whose
slice counts UTF-16 code units, so drop a split surrogate half.

Also fix two comment claims. The note said one projection reads the
runtime twice per tool; the language-aware getters are installed on
run_code's own definition, so it is twice, both for that schema. And the
182-bracket site's reachability is an array reached from the root
through oneOf arms alone -- a union spine of any depth, not just one
root union; an object ancestor restarts the chain at the 181 site.
This commit is contained in:
Chinesezjc
2026-08-05 20:03:39 +08:00
parent ba634896e0
commit 5d65686c33
5 changed files with 187 additions and 26 deletions

View File

@@ -398,6 +398,106 @@ describe('renderToolsSdkPy', () => {
expect(text).not.toContain('dict[str, Any]')
})
it('keeps a non-ASCII field name as a TypedDict field and derives its class name from it', () => {
// `路径` satisfies `xid_start xid_continue*`, so CPython accepts it as an
// attribute and as the `TypedDict` key. Rejecting it would degrade the
// whole object, dropping every SIBLING field's name, requiredness and type
// too — and Code Mode omits the native schemas, so nothing else carries
// them. The nested class name is derived from the field, so `camelCase`
// has to pass the same characters through instead of splitting on them.
const tool: ToolSdkSchema = {
name: '搜索',
description: 'Unicode identifiers.',
parameters: {
type: 'object',
additionalProperties: false,
properties: {
路径: { type: 'string' },
opts: { type: 'object', additionalProperties: false, properties: { 深度: { type: 'number' } } },
},
required: ['路径'],
},
output: { type: 'string' },
}
const text = renderToolsSdkPy([tool])
expect(text).toContain('async def 搜索(self, args: 搜索Args) -> str:')
expect(text).toContain('class 搜索Args(TypedDict):')
expect(text).toContain(' 路径: str')
expect(text).toContain('class 搜索ArgsOpts(TypedDict):')
expect(text).toContain(' 深度: NotRequired[float]')
expect(text).not.toContain('dict[str, Any]')
})
it('degrades a field name that NFKC-normalizes to something else, which would be declared under another spelling', () => {
// U+FB01 LATIN SMALL LIGATURE FI passes the identifier grammar, but CPython
// normalizes identifiers at compile time while the harness compares the
// JSON key as written: `field: str` would declare and be reachable as
// `field`, a key the tool never accepts. Two keys that normalize together
// would additionally collapse into one declaration. The subscript path
// carries the exact bytes instead.
const text = renderToolsSdkPy([
{
name: 'ligature',
description: 'Normalizing field name.',
parameters: { type: 'object', additionalProperties: false, properties: { field: { type: 'string' } } },
output: { type: 'string' },
},
])
expect(text).toContain('async def ligature(self, args: dict[str, Any]) -> str:')
expect(text).not.toContain('field:')
expect(text).not.toContain('field:')
})
it('subscripts a tool name that NFKC-normalizes to something else, while declaring a plain Unicode one', () => {
// Same split at the tool-name site: `路径` becomes an `async def`, the
// ligature name cannot, because `async def find` would define `find`. The
// subscript comment quotes the name, so its exact bytes survive, and its
// TypedDict is still named and referenced — the name is only unusable as a
// method, not as a class-name source (`camelCase` normalizes what it
// derives, since a generated name is never matched against a JSON key).
const of = (name: string): ToolSdkSchema => ({
name,
description: `Tool ${name}.`,
parameters: { type: 'object', additionalProperties: false, properties: { q: { type: 'string' } }, required: ['q'] },
output: { type: 'string' },
})
const text = renderToolsSdkPy([of('路径'), of('find')])
expect(text).toContain('async def 路径(self, args: 路径Args) -> str:')
expect(text).toContain('# tools["find"](args: FIndArgs) -> str')
expect(text).toContain('class FIndArgs(TypedDict):')
expect(text).not.toContain('async def find')
expect(text).not.toContain('async def find')
})
it('drops a surrogate half rather than cutting a pair when capping an astral class-name base', () => {
// Class-name bases are capped by `slice`, which counts UTF-16 code units,
// so a boundary landing inside an astral pair would leave a lone high
// surrogate — not an identifier character, and not encodable text. Padding
// with one ASCII character shifts the boundary onto the pair.
// U+10330 GOTHIC LETTER AHSA: XID_Start and NFKC-stable, unlike `𝕏`, which
// NFKC-folds to ASCII `X` and so never reaches the boundary at all.
const AHSA = String.fromCodePoint(0x10330)
const className = (pad: string): string => {
const text = renderToolsSdkPy([
{
name: `${pad}${AHSA.repeat(200)}`,
description: 'Astral name.',
parameters: { type: 'object', additionalProperties: false, properties: { a: { type: 'string' } } },
output: { type: 'string' },
},
])
// The base is `${camelCase(name)}Args` capped to 120 code units, so the
// `Args` suffix itself is cut off here; match the declaration instead.
return /^class (.+)\(TypedDict\):$/mu.exec(text)![1]!
}
// Each character is 2 code units, so an unpadded name fills the cap with 60
// whole characters; one ASCII character of padding puts the boundary inside
// the 60th pair, and that half is dropped rather than emitted.
expect(className('')).toBe(AHSA.repeat(60))
expect(className('x')).toBe(`X${AHSA.repeat(59)}`)
expect(className('x')).toHaveLength(119)
})
it('declares a closed empty object with omitted properties as an empty TypedDict, not dict[str, Any]', () => {
// `{ type: 'object', additionalProperties: false }` with no `properties`
// is a closed empty object — no key accepted — exactly as the validator
@@ -580,8 +680,10 @@ describe('renderToolsSdkPy', () => {
// The worst of the three emission sites: the parameter list's `(` is still
// open around this annotation, so 180 `list[` plus the innermost bracket
// plus that paren is 182 of CPython's 200. Only a raw `register()` whose
// `parameters` root opens an array chain reaches it — rooted at the array,
// or at an array branch of a root `oneOf`, since a union adds no brackets.
// `parameters` is an array reached from the root through `oneOf` arms
// alone gets there — the root array itself, or one under any depth of
// unions, since an arm inherits the enclosing depth unchanged. An object
// ancestor takes it out of this case: its fields restart at the 181 site.
// `defineTool` compiles an object root, whose annotation is a bare
// TypedDict name or a one-bracket `dict[str, Any]`, never a chain.
const rooted = (depth: number): ToolSdkSchema => {
@@ -602,13 +704,25 @@ describe('renderToolsSdkPy', () => {
// rather than on another `list[`, so the count cannot grow past that.
expect(renderToolsSdkPy([rooted(181)]))
.toContain(`async def rooted(self, args: ${'list['.repeat(180)}Any${']'.repeat(180)}) -> str:`)
// A root union reaches the same 182: its branches inherit the enclosing
// depth because `A | B` opens nothing, so the chain under one of them
// starts at 0 exactly as the array-rooted case does.
const union = { ...rooted(180), parameters: { oneOf: [rooted(180).parameters, { type: 'string' }] } }
const text = renderToolsSdkPy([union])
expect(text).toContain(`args: ${'list['.repeat(180)}Literal["x"]${']'.repeat(180)} | str) -> str:`)
// A union spine reaches the same 182, at any number of arms deep: each arm
// inherits the enclosing depth because `A | B` opens nothing, so the chain
// under the innermost one still starts at 0. Three unions here, to pin that
// it is the whole `oneOf`-only path and not just a single root union.
let spine: Record<string, unknown> = rooted(180).parameters
for (let i = 0; i < 3; i++) spine = { oneOf: [spine, { type: 'string' }] }
const text = renderToolsSdkPy([{ ...rooted(180), parameters: spine }])
const chain = `${'list['.repeat(180)}Literal["x"]${']'.repeat(180)}`
expect(text).toContain(`args: ${chain} | str | str | str) -> str:`)
expect(text.split('async def rooted(self, args: ')[1]!.split(') -> str:')[0]!.split('[').length - 1).toBe(181)
// An object ancestor is the boundary of that path: the field it declares is
// a class-body line, so the same chain lands on the 181 site instead.
const boxed = renderToolsSdkPy([
{
...rooted(180),
parameters: { type: 'object', properties: { rows: rooted(180).parameters }, required: ['rows'] },
},
])
expect(boxed).toContain(` rows: ${'list['.repeat(179)}Any${']'.repeat(179)}`)
})
it('renders a deeply nested oneOf chain in linear time (no per-level re-materialization)', () => {