Coder/deepseek-harness
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(build): stage platform-specific PTY artifacts

Browse Source
This commit is contained in:
Tianyi Cui
2026-07-29 22:44:19 +08:00
parent 7fdde06cf9
commit c0bd88430a
14 changed files with 141 additions and 105 deletions
Download Patch File Download Diff File Expand all files Collapse all files

75
scripts/build-exe-for-python-sdk.ts
View File

@@ -8,8 +8,8 @@
import { spawn } from 'node:child_process'
import { existsSync, mkdirSync, readFileSync, statSync } from 'node:fs'
import { chmod, copyFile, readFile, rm, writeFile } from 'node:fs/promises'
import { basename, join, resolve, sep } from 'node:path'
import { chmod, copyFile, mkdir, readFile, rm, writeFile } from 'node:fs/promises'
import { basename, dirname, join, resolve, sep } from 'node:path'
import { parseArgs } from 'node:util'
const root = resolve(import.meta.dirname, '..')
@@ -55,19 +55,11 @@ type Arch = (typeof ARCHES)[number]
interface RuntimeProduct {
executable: string
spawnHelper: string
spawnHelper?: string
}
function spawnHelperBinaryTarget(path: string): string | undefined {
const header = readFileSync(path).subarray(0, 20)
if (header.length >= 20
&& header.subarray(0, 4).equals(Buffer.from([0x7f, 0x45, 0x4c, 0x46]))
&& header[4] === 2
&& header[5] === 1) {
const machine = header.readUInt16LE(18)
if (machine === 62) return 'linux-x64'
if (machine === 183) return 'linux-arm64'
}
const header = readFileSync(path).subarray(0, 8)
if (header.length >= 8 && header.readUInt32LE(0) === 0xfeedfacf) {
const cpuType = header.readUInt32LE(4)
if (cpuType === 0x01000007) return 'macos-x64'
@@ -76,6 +68,10 @@ function spawnHelperBinaryTarget(path: string): string | undefined {
return undefined
}
function runtimeProductFiles(product: RuntimeProduct): string[] {
return [product.executable, ...(product.spawnHelper === undefined ? [] : [product.spawnHelper])]
}
function isPlatform(value: string): value is Platform {
return (PLATFORMS as readonly string[]).includes(value)
}
@@ -317,7 +313,7 @@ class SingleExeBuild {
*/
async pack(target: Target): Promise<RuntimeProduct> {
const product = join(this.outDir, `${OUTPUT_BASENAME}-${target.platform}-${target.arch}`)
const spawnHelper = `${product}${SPAWN_HELPER_SUFFIX}`
await this.prepareNativePty(target)
if (!this.cli.dryRun) mkdirSync(this.outDir, { recursive: true })
await this.run(`pkg ${target.spec}`, pnpmBin(), [
'dlx',
@@ -332,6 +328,8 @@ class SingleExeBuild {
if (!this.cli.dryRun && !existsSync(product)) {
throw new Error(`build-exe-for-python-sdk: product ${product} is missing after the pkg run; inspect ${this.outDir}.`)
}
if (target.platform !== 'macos') return { executable: product }
const spawnHelper = `${product}${SPAWN_HELPER_SUFFIX}`
if (this.cli.dryRun) {
console.log(`build-exe-for-python-sdk: [dry-run] copy target node-pty spawn-helper to ${spawnHelper}`)
} else {
@@ -342,6 +340,38 @@ class SingleExeBuild {
return { executable: product, spawnHelper }
}
/**
* Put the target node-pty addon in the staged closure. Linux npm installs
* build it from source, but legacy deploy omits that side-effect directory.
* @param target - the pkg target whose native addon is being staged.
*/
private async prepareNativePty(target: Target): Promise<void> {
const stagedRoot = join(this.staging, 'node_modules', 'node-pty')
const stagedBuild = join(stagedRoot, 'build')
if (this.cli.dryRun) console.log(`build-exe-for-python-sdk: [dry-run] rm -rf ${stagedBuild}`)
else await rm(stagedBuild, { recursive: true, force: true })
const nativePlatform = target.platform === 'macos' ? 'darwin' : 'linux'
const prebuilt = join(stagedRoot, 'prebuilds', `${nativePlatform}-${target.arch}`, 'pty.node')
const source = join(root, 'packages', 'pty', 'pty-local', 'node_modules', 'node-pty', 'build', 'Release', 'pty.node')
const destination = join(stagedBuild, 'Release', 'pty.node')
if (this.cli.dryRun) {
if (target.platform === 'linux') console.log(`build-exe-for-python-sdk: [dry-run] cp ${source} ${destination}`)
return
}
if (existsSync(prebuilt)) return
const host = Target.host()
if (target.platform !== host.platform || target.arch !== host.arch || !existsSync(source)) {
throw new Error(
`build-exe-for-python-sdk: node-pty native addon for ${target.platform}-${target.arch} is missing; `
+ `checked ${prebuilt}, ${source}. Build the Linux runtime on its target architecture.`,
)
}
await mkdir(dirname(destination), { recursive: true })
await copyFile(source, destination)
}
/**
* Resolve the node-pty helper that matches a pkg target.
* @param target - the pkg target whose helper must be shipped.
@@ -349,14 +379,12 @@ class SingleExeBuild {
*/
private resolveSpawnHelper(target: Target): string {
const nodePtyRoot = join(this.staging, 'node_modules', 'node-pty')
const nativePlatform = target.platform === 'macos' ? 'darwin' : 'linux'
const candidates = [
join(nodePtyRoot, 'prebuilds', `${nativePlatform}-${target.arch}`, 'spawn-helper'),
join(nodePtyRoot, 'prebuilds', `darwin-${target.arch}`, 'spawn-helper'),
]
const hostPlatform = process.platform === 'darwin' ? 'macos' : process.platform
const hostArch = process.arch === 'x64' || process.arch === 'arm64' ? process.arch : undefined
if (target.platform === hostPlatform && target.arch === hostArch) {
candidates.push(join(nodePtyRoot, 'build', 'Release', 'spawn-helper'))
const host = Target.host()
if (target.platform === host.platform && target.arch === host.arch) {
candidates.push(join(root, 'packages', 'pty', 'pty-local', 'node_modules', 'node-pty', 'build', 'Release', 'spawn-helper'))
}
const helper = candidates.find(candidate => existsSync(candidate))
if (helper === undefined) {
@@ -387,11 +415,10 @@ class SingleExeBuild {
console.log(this.cli.dryRun ? 'build-exe-for-python-sdk: [dry-run] would produce:' : 'build-exe-for-python-sdk: products:')
for (const product of products) {
if (this.cli.dryRun) {
console.log(` ${product.executable}`)
console.log(` ${product.spawnHelper}`)
for (const path of runtimeProductFiles(product)) console.log(` ${path}`)
continue
}
for (const path of [product.executable, product.spawnHelper]) {
for (const path of runtimeProductFiles(product)) {
const megabytes = statSync(path).size / (1024 * 1024)
console.log(` ${path} (${megabytes.toFixed(1)} MB)`)
}
@@ -407,7 +434,7 @@ class SingleExeBuild {
const destDir = resolve(root, PYTHON_RUNTIME_DIR)
if (this.cli.dryRun) {
for (const product of products) {
for (const path of [product.executable, product.spawnHelper]) {
for (const path of runtimeProductFiles(product)) {
console.log(`build-exe-for-python-sdk: [dry-run] cp ${path} ${join(destDir, basename(path))}`)
}
}
@@ -415,7 +442,7 @@ class SingleExeBuild {
}
mkdirSync(destDir, { recursive: true })
for (const product of products) {
for (const path of [product.executable, product.spawnHelper]) {
for (const path of runtimeProductFiles(product)) {
const destination = join(destDir, basename(path))
await copyFile(path, destination)
await chmod(destination, statSync(path).mode & 0o777)

64
scripts/build-python-release.py
View File

@@ -27,26 +27,18 @@ EXECUTABLE_TARGETS = {value[1]: key for key, value in PLATFORMS.items()}
def spawn_helper_binary_target(header: bytes) -> str | None:
if (
len(header) >= 20
and header[:4] == b"\x7fELF"
and header[4] == 2
and header[5] == 1
):
machine = int.from_bytes(header[18:20], "little")
if machine == 62:
return "linux-x64"
if machine == 183:
return "linux-arm64"
if len(header) >= 8 and header[:4] == b"\xcf\xfa\xed\xfe":
if int.from_bytes(header[4:8], "little") == 0x0100000C:
cpu_type = int.from_bytes(header[4:8], "little")
if cpu_type == 0x01000007:
return "macos-x64"
if cpu_type == 0x0100000C:
return "macos-arm64"
return None
def validate_spawn_helper(path: Path, expected_target: str) -> None:
with path.open("rb") as helper:
actual_target = spawn_helper_binary_target(helper.read(20))
actual_target = spawn_helper_binary_target(helper.read(8))
if actual_target != expected_target:
raise ValueError(
f"runtime spawn helper binary mismatch: expected {expected_target}, "
@@ -166,12 +158,14 @@ def stage_runtime(destination: Path, version: str, executable: Path, executable_
raise FileNotFoundError(f"runtime executable does not exist: {executable}")
if executable.stat().st_mode & stat.S_IXUSR == 0:
raise PermissionError(f"runtime executable is not executable: {executable}")
expected_target = EXECUTABLE_TARGETS[executable_name]
spawn_helper = Path(f"{executable}{SPAWN_HELPER_SUFFIX}")
if not spawn_helper.is_file():
raise FileNotFoundError(f"runtime spawn helper does not exist: {spawn_helper}")
if spawn_helper.stat().st_mode & stat.S_IXUSR == 0:
raise PermissionError(f"runtime spawn helper is not executable: {spawn_helper}")
validate_spawn_helper(spawn_helper, EXECUTABLE_TARGETS[executable_name])
if expected_target.startswith("macos-"):
if not spawn_helper.is_file():
raise FileNotFoundError(f"runtime spawn helper does not exist: {spawn_helper}")
if spawn_helper.stat().st_mode & stat.S_IXUSR == 0:
raise PermissionError(f"runtime spawn helper is not executable: {spawn_helper}")
validate_spawn_helper(spawn_helper, expected_target)
copy_package(ROOT / "python" / "sdk-runtime", destination)
rewrite_version(destination / "pyproject.toml", version)
runtime_dir = destination / "src" / "deepseek_harness_runtime" / "runtime"
@@ -179,9 +173,10 @@ def stage_runtime(destination: Path, version: str, executable: Path, executable_
destination_executable = runtime_dir / executable_name
shutil.copyfile(executable, destination_executable)
destination_executable.chmod(executable.stat().st_mode & 0o777)
destination_helper = runtime_dir / f"{executable_name}{SPAWN_HELPER_SUFFIX}"
shutil.copyfile(spawn_helper, destination_helper)
destination_helper.chmod(spawn_helper.stat().st_mode & 0o777)
if expected_target.startswith("macos-"):
destination_helper = runtime_dir / f"{executable_name}{SPAWN_HELPER_SUFFIX}"
shutil.copyfile(spawn_helper, destination_helper)
destination_helper.chmod(spawn_helper.stat().st_mode & 0o777)
def verify_wheel(
@@ -209,20 +204,27 @@ def verify_wheel(
assert platform is not None
if len(executables) != 1 or not executables[0].endswith(f"/runtime/{platform[1]}"):
raise RuntimeError(f"{wheel} must contain exactly {platform[1]}, found {executables}")
expected_target = EXECUTABLE_TARGETS[platform[1]]
expected_helper = f"{platform[1]}{SPAWN_HELPER_SUFFIX}"
if len(helpers) != 1 or not helpers[0].endswith(f"/runtime/{expected_helper}"):
raise RuntimeError(f"{wheel} must contain exactly {expected_helper}, found {helpers}")
for executable in [executables[0], helpers[0]]:
expected_helpers = [expected_helper] if expected_target.startswith("macos-") else []
found_helpers = [Path(helper).name for helper in helpers]
if found_helpers != expected_helpers:
expected = ", ".join(expected_helpers) or "none"
found = ", ".join(found_helpers) or "none"
raise RuntimeError(
f"{wheel} runtime helper payload mismatch: expected {expected}; found {found}"
)
for executable in [executables[0], *helpers]:
mode = archive.getinfo(executable).external_attr >> 16
if mode & stat.S_IXUSR == 0:
raise RuntimeError(f"{wheel} runtime executable lost its executable bit: {executable}")
actual_target = spawn_helper_binary_target(archive.read(helpers[0])[:20])
expected_target = EXECUTABLE_TARGETS[platform[1]]
if actual_target != expected_target:
raise RuntimeError(
f"{wheel} spawn helper binary mismatch: expected {expected_target}, "
f"found {actual_target or 'unsupported format or architecture'}"
)
if helpers:
actual_target = spawn_helper_binary_target(archive.read(helpers[0])[:8])
if actual_target != expected_target:
raise RuntimeError(
f"{wheel} spawn helper binary mismatch: expected {expected_target}, "
f"found {actual_target or 'unsupported format or architecture'}"
)
elif runtime_files:
raise RuntimeError(f"SDK wheel unexpectedly contains runtime executables: {runtime_files}")
if package == "sdk":