cleanup(build): minimize native payload handling

This commit is contained in:
Tianyi Cui
2026-07-30 01:29:18 +08:00
parent 7118b4cfe1
commit 8a9d882a11
9 changed files with 47 additions and 119 deletions

View File

@@ -2,5 +2,5 @@
# side as of the last confirmed-consistent state. Both languages carry equal authority; # side as of the last confirmed-consistent state. Both languages carry equal authority;
# after editing either side, bring the other along and re-record with: # after editing either side, bring the other along and re-record with:
# pnpm run verify-translation-pairing --write python/README.md # pnpm run verify-translation-pairing --write python/README.md
README.md: aee682e25fc33287c49131d0f5b92b136ed16bae README.md: dfd9d909122f9245a19fe91d8a394156795b13ae
README.zh.md: 4404114fcdab78468991769a4657370f85997a88 README.zh.md: 151b2cdab4f28384a20d16c86d398f669fbdc818

View File

@@ -22,7 +22,7 @@ pnpm exec tsx scripts/build-exe-for-python-sdk.ts --skip-build # lib/ artifac
pnpm exec tsx scripts/build-exe-for-python-sdk.ts --targets=node24-linux-x64,node24-linux-arm64,node24-macos-arm64 pnpm exec tsx scripts/build-exe-for-python-sdk.ts --targets=node24-linux-x64,node24-linux-arm64,node24-macos-arm64
``` ```
Products land in `dist-exe/` and are synced into this package as `sdk-runtime/src/deepseek_harness_runtime/runtime/dsh-jsonrpc-agent-pkg-<platform>-<arch>` plus the matching `-spawn-helper` required by `node-pty` (platform: `linux`/`macos`; arch: `x64`/`arm64`) — after a local build the SDK finds the runtime with no further setup. The `build-exe-for-python-sdk` CI workflow (manual dispatch, or the `build-exe` PR label) exercises the same products. A full three-target run retains four release wheels; a subset dispatch retains the SDK wheel and selected platform wheels. Which plugins the exe bundles and how the carriers are organized: [sdk-runtime README](sdk-runtime/README.md); the build also refreshes the dev-only node carrier (see "against the Node source" below). Products land in `dist-exe/` and are synced into this package as `sdk-runtime/src/deepseek_harness_runtime/runtime/dsh-jsonrpc-agent-pkg-<platform>-<arch>` (platform: `linux`/`macos`; arch: `x64`/`arm64`); macOS builds also sync the matching `-spawn-helper` required by `node-pty`. After a local build the SDK finds the runtime with no further setup. The `build-exe-for-python-sdk` CI workflow (manual dispatch, or the `build-exe` PR label) exercises the same products. A full three-target run retains four release wheels; a subset dispatch retains the SDK wheel and selected platform wheels. Which plugins the exe bundles and how the carriers are organized: [sdk-runtime README](sdk-runtime/README.md); the build also refreshes the dev-only node carrier (see "against the Node source" below).
## Validating the SDK against the executable ## Validating the SDK against the executable

View File

@@ -22,7 +22,7 @@ pnpm exec tsx scripts/build-exe-for-python-sdk.ts --skip-build # lib/ artifac
pnpm exec tsx scripts/build-exe-for-python-sdk.ts --targets=node24-linux-x64,node24-linux-arm64,node24-macos-arm64 pnpm exec tsx scripts/build-exe-for-python-sdk.ts --targets=node24-linux-x64,node24-linux-arm64,node24-macos-arm64
``` ```
产物落入 `dist-exe/`,并同步进本包的 `sdk-runtime/src/deepseek_harness_runtime/runtime/dsh-jsonrpc-agent-pkg-<platform>-<arch>` 及 `node-pty` 所需的同名 `-spawn-helper` 伴随文件(platform:`linux`/`macos`;arch:`x64`/`arm64`),本地构建完成后 SDK 不需要额外设置就能找到运行时。`build-exe-for-python-sdk` CI 工作流(手动触发,或给 PR 打 `build-exe` 标签)会测试同样的产物。完整构建三个目标时保留 4 个发布用 wheel 包;手动选择部分目标时保留 SDK wheel 与所选平台的 wheel。exe 内置哪些插件、载体如何组织,见 [sdk-runtime README](sdk-runtime/README.md);构建还会顺带刷新仅供开发使用的 `node` 载体(见下文「对着 Node 源码运行」)。 产物落入 `dist-exe/`,并同步进本包的 `sdk-runtime/src/deepseek_harness_runtime/runtime/dsh-jsonrpc-agent-pkg-<platform>-<arch>`(platform:`linux`/`macos`;arch:`x64`/`arm64`);macOS 构建还会同步 `node-pty` 所需的同名 `-spawn-helper` 伴随文件。本地构建完成后 SDK 不需要额外设置就能找到运行时。`build-exe-for-python-sdk` CI 工作流(手动触发,或给 PR 打 `build-exe` 标签)会测试同样的产物。完整构建三个目标时保留 4 个发布用 wheel 包;手动选择部分目标时保留 SDK wheel 与所选平台的 wheel。exe 内置哪些插件、载体如何组织,见 [sdk-runtime README](sdk-runtime/README.md);构建还会顺带刷新仅供开发使用的 `node` 载体(见下文「对着 Node 源码运行」)。
## 用可执行文件验证 SDK ## 用可执行文件验证 SDK

View File

@@ -13,7 +13,6 @@ _PLATFORMS = {
"linux-arm64": ("manylinux_2_28_aarch64", "dsh-jsonrpc-agent-pkg-linux-arm64"), "linux-arm64": ("manylinux_2_28_aarch64", "dsh-jsonrpc-agent-pkg-linux-arm64"),
"macos-arm64": ("macosx_11_0_arm64", "dsh-jsonrpc-agent-pkg-macos-arm64"), "macos-arm64": ("macosx_11_0_arm64", "dsh-jsonrpc-agent-pkg-macos-arm64"),
} }
_SPAWN_HELPER_SUFFIX = "-spawn-helper"
def _host_platform_tag() -> str: def _host_platform_tag() -> str:
@@ -50,7 +49,7 @@ class RuntimeBuildHook(BuildHookInterface):
runtime_files = sorted(runtime_dir.glob("dsh-jsonrpc-agent-pkg-*") if runtime_dir.is_dir() else []) runtime_files = sorted(runtime_dir.glob("dsh-jsonrpc-agent-pkg-*") if runtime_dir.is_dir() else [])
expected_files = [expected_executable] expected_files = [expected_executable]
if "-macos-" in expected_executable: if "-macos-" in expected_executable:
expected_files.append(f"{expected_executable}{_SPAWN_HELPER_SUFFIX}") expected_files.append(f"{expected_executable}-spawn-helper")
found_files = [path.name for path in runtime_files] found_files = [path.name for path in runtime_files]
if found_files != expected_files: if found_files != expected_files:
raise RuntimeError( raise RuntimeError(

View File

@@ -28,7 +28,6 @@ import sys
from pathlib import Path from pathlib import Path
PACKAGE_METADATA_FILENAME = "deepseek-harness-runtime.json" PACKAGE_METADATA_FILENAME = "deepseek-harness-runtime.json"
SPAWN_HELPER_SUFFIX = "-spawn-helper"
RUNTIME_MODE_ENV_VAR = "DSH_RUNTIME_MODE" RUNTIME_MODE_ENV_VAR = "DSH_RUNTIME_MODE"
@@ -85,7 +84,7 @@ def bundled_runtime_path() -> Path:
+ _EXE_ACQUISITION_HINT + _EXE_ACQUISITION_HINT
) )
if tag.startswith("macos-"): if tag.startswith("macos-"):
helper = Path(f"{path}{SPAWN_HELPER_SUFFIX}") helper = Path(f"{path}-spawn-helper")
if not helper.is_file(): if not helper.is_file():
raise FileNotFoundError( raise FileNotFoundError(
f"deepseek-harness-runtime-bin is missing the node-pty spawn helper at {helper}. " f"deepseek-harness-runtime-bin is missing the node-pty spawn helper at {helper}. "
@@ -153,7 +152,6 @@ def _node_launch_args() -> tuple[str, str]:
__all__ = [ __all__ = [
"PACKAGE_METADATA_FILENAME", "PACKAGE_METADATA_FILENAME",
"RUNTIME_MODE_ENV_VAR", "RUNTIME_MODE_ENV_VAR",
"SPAWN_HELPER_SUFFIX",
"bundled_default_config_path", "bundled_default_config_path",
"bundled_package_dir", "bundled_package_dir",
"bundled_runtime_path", "bundled_runtime_path",

View File

@@ -4,7 +4,6 @@ from __future__ import annotations
import json import json
import runpy import runpy
import stat
from pathlib import Path from pathlib import Path
from types import SimpleNamespace from types import SimpleNamespace
@@ -40,51 +39,22 @@ def test_repository_version_rejects_non_stable_versions(tmp_path: Path) -> None:
build_python_release.repository_version(tmp_path) build_python_release.repository_version(tmp_path)
def test_stage_runtime_copies_executable_and_spawn_helper(tmp_path: Path) -> None: @pytest.mark.parametrize(("target", "with_helper"), [("linux-x64", False), ("macos-arm64", True)])
executable = tmp_path / "dsh-jsonrpc-agent-pkg-macos-arm64" def test_stage_runtime_copies_platform_payload(
executable.write_bytes(b"runtime") tmp_path: Path, target: str, with_helper: bool
executable.chmod(0o755) ) -> None:
spawn_helper = Path(f"{executable}-spawn-helper") executable = tmp_path / f"dsh-jsonrpc-agent-pkg-{target}"
spawn_helper.write_bytes(b"helper")
spawn_helper.chmod(0o751)
destination = tmp_path / "staging"
build_python_release.stage_runtime(
destination,
"1.2.3",
executable,
executable.name,
)
runtime_dir = destination / "src" / "deepseek_harness_runtime" / "runtime"
assert (runtime_dir / executable.name).read_bytes() == b"runtime"
copied_helper = runtime_dir / spawn_helper.name
assert copied_helper.read_bytes() == b"helper"
assert copied_helper.stat().st_mode & stat.S_IXUSR
def test_stage_runtime_rejects_missing_spawn_helper(tmp_path: Path) -> None:
executable = tmp_path / "dsh-jsonrpc-agent-pkg-macos-arm64"
executable.write_bytes(b"runtime")
executable.chmod(0o755)
with pytest.raises(FileNotFoundError, match="spawn-helper"):
build_python_release.stage_runtime(
tmp_path / "staging",
"1.2.3",
executable,
executable.name,
)
def test_stage_runtime_copies_linux_executable_without_spawn_helper(tmp_path: Path) -> None:
executable = tmp_path / "dsh-jsonrpc-agent-pkg-linux-x64"
executable.write_bytes(b"runtime") executable.write_bytes(b"runtime")
executable.chmod(0o755) executable.chmod(0o755)
expected = {executable.name: b"runtime"}
if with_helper:
spawn_helper = Path(f"{executable}-spawn-helper")
spawn_helper.write_bytes(b"helper")
spawn_helper.chmod(0o755)
expected[spawn_helper.name] = b"helper"
destination = tmp_path / "staging" destination = tmp_path / "staging"
build_python_release.stage_runtime(destination, "1.2.3", executable, executable.name) build_python_release.stage_runtime(destination, "1.2.3", executable, executable.name)
runtime_dir = destination / "src" / "deepseek_harness_runtime" / "runtime" runtime_dir = destination / "src" / "deepseek_harness_runtime" / "runtime"
runtime_files = [path.name for path in runtime_dir.glob("dsh-jsonrpc-agent-pkg-*")] assert {path.name: path.read_bytes() for path in runtime_dir.glob("dsh-jsonrpc-agent-pkg-*")} == expected
assert runtime_files == [executable.name]

View File

@@ -44,25 +44,18 @@ def test_explicit_mode_wins_over_env_mode(monkeypatch: pytest.MonkeyPatch) -> No
assert args[0].endswith(("-x64", "-arm64")) assert args[0].endswith(("-x64", "-arm64"))
@pytest.mark.parametrize(
("platform_tag", "requires_helper"),
[("linux-x64", False), ("macos-arm64", True)],
)
def test_runtime_requires_spawn_helper_only_on_macos( def test_runtime_requires_spawn_helper_only_on_macos(
tmp_path: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
monkeypatch: pytest.MonkeyPatch,
platform_tag: str,
requires_helper: bool,
) -> None: ) -> None:
runtime_dir = tmp_path / "runtime" runtime_dir = tmp_path / "runtime"
runtime_dir.mkdir() runtime_dir.mkdir()
executable = runtime_dir / f"dsh-jsonrpc-agent-pkg-{platform_tag}" linux = runtime_dir / "dsh-jsonrpc-agent-pkg-linux-x64"
executable.touch() linux.touch()
(runtime_dir / "dsh-jsonrpc-agent-pkg-macos-arm64").touch()
monkeypatch.setattr(runtime, "bundled_package_dir", lambda: tmp_path) monkeypatch.setattr(runtime, "bundled_package_dir", lambda: tmp_path)
monkeypatch.setattr(runtime, "_current_platform_tag", lambda: platform_tag)
if requires_helper: monkeypatch.setattr(runtime, "_current_platform_tag", lambda: "macos-arm64")
with pytest.raises(FileNotFoundError, match="node-pty spawn helper"): with pytest.raises(FileNotFoundError, match="node-pty spawn helper"):
runtime.bundled_runtime_path() runtime.bundled_runtime_path()
else: monkeypatch.setattr(runtime, "_current_platform_tag", lambda: "linux-x64")
assert runtime.bundled_runtime_path() == executable assert runtime.bundled_runtime_path() == linux

View File

@@ -7,7 +7,7 @@
*/ */
import { spawn } from 'node:child_process' import { spawn } from 'node:child_process'
import { existsSync, mkdirSync, statSync } from 'node:fs' import { existsSync, statSync } from 'node:fs'
import { chmod, copyFile, mkdir, readFile, rm, writeFile } from 'node:fs/promises' import { chmod, copyFile, mkdir, readFile, rm, writeFile } from 'node:fs/promises'
import { basename, dirname, join, resolve, sep } from 'node:path' import { basename, dirname, join, resolve, sep } from 'node:path'
import { parseArgs } from 'node:util' import { parseArgs } from 'node:util'
@@ -19,7 +19,6 @@ const DEPLOY_ROOT_PACKAGE = 'dsh-jsonrpc-agent-pkg'
/** The app entry inside the deployed closure. */ /** The app entry inside the deployed closure. */
const ENTRY_BIN = 'node_modules/@deepseek-ai/dsh-jsonrpc-demo/lib/bin.js' const ENTRY_BIN = 'node_modules/@deepseek-ai/dsh-jsonrpc-demo/lib/bin.js'
const OUTPUT_BASENAME = 'dsh-jsonrpc-agent-pkg' const OUTPUT_BASENAME = 'dsh-jsonrpc-agent-pkg'
const SPAWN_HELPER_SUFFIX = '-spawn-helper'
/** Default Node major; SEA mode requires at least Node 22. */ /** Default Node major; SEA mode requires at least Node 22. */
const DEFAULT_NODE_RANGE = 'node24' const DEFAULT_NODE_RANGE = 'node24'
/** Pinned for reproducible builds. */ /** Pinned for reproducible builds. */
@@ -295,7 +294,7 @@ class SingleExeBuild {
async pack(target: Target): Promise<string[]> { async pack(target: Target): Promise<string[]> {
const product = join(this.outDir, `${OUTPUT_BASENAME}-${target.platform}-${target.arch}`) const product = join(this.outDir, `${OUTPUT_BASENAME}-${target.platform}-${target.arch}`)
await this.prepareNativePty(target) await this.prepareNativePty(target)
if (!this.cli.dryRun) mkdirSync(this.outDir, { recursive: true }) if (!this.cli.dryRun) await mkdir(this.outDir, { recursive: true })
await this.run(`pkg ${target.spec}`, pnpmBin(), [ await this.run(`pkg ${target.spec}`, pnpmBin(), [
'dlx', 'dlx',
PKG_SPEC, PKG_SPEC,
@@ -310,13 +309,13 @@ class SingleExeBuild {
throw new Error(`build-exe-for-python-sdk: product ${product} is missing after the pkg run; inspect ${this.outDir}.`) throw new Error(`build-exe-for-python-sdk: product ${product} is missing after the pkg run; inspect ${this.outDir}.`)
} }
if (target.platform !== 'macos') return [product] if (target.platform !== 'macos') return [product]
const spawnHelper = `${product}${SPAWN_HELPER_SUFFIX}` const spawnHelper = `${product}-spawn-helper`
const source = join(this.staging, 'node_modules', 'node-pty', 'prebuilds', `darwin-${target.arch}`, 'spawn-helper')
if (this.cli.dryRun) { if (this.cli.dryRun) {
console.log(`build-exe-for-python-sdk: [dry-run] copy target node-pty spawn-helper to ${spawnHelper}`) console.log(`build-exe-for-python-sdk: [dry-run] cp ${source} ${spawnHelper}`)
} else { } else {
const source = this.resolveSpawnHelper(target)
await copyFile(source, spawnHelper) await copyFile(source, spawnHelper)
await chmod(spawnHelper, statSync(source).mode & 0o777) await chmod(spawnHelper, 0o755)
} }
return [product, spawnHelper] return [product, spawnHelper]
} }
@@ -327,47 +326,27 @@ class SingleExeBuild {
* @param target - the pkg target whose native addon is being staged. * @param target - the pkg target whose native addon is being staged.
*/ */
private async prepareNativePty(target: Target): Promise<void> { private async prepareNativePty(target: Target): Promise<void> {
const stagedRoot = join(this.staging, 'node_modules', 'node-pty') const stagedBuild = join(this.staging, 'node_modules', 'node-pty', 'build')
const stagedBuild = join(stagedRoot, 'build')
if (this.cli.dryRun) console.log(`build-exe-for-python-sdk: [dry-run] rm -rf ${stagedBuild}`) if (this.cli.dryRun) console.log(`build-exe-for-python-sdk: [dry-run] rm -rf ${stagedBuild}`)
else await rm(stagedBuild, { recursive: true, force: true }) else await rm(stagedBuild, { recursive: true, force: true })
if (target.platform !== 'linux') return
const source = join(root, 'packages', 'pty', 'pty-local', 'node_modules', 'node-pty', 'build', 'Release', 'pty.node') const source = join(root, 'packages', 'pty', 'pty-local', 'node_modules', 'node-pty', 'build', 'Release', 'pty.node')
const destination = join(stagedBuild, 'Release', 'pty.node') const destination = join(stagedBuild, 'Release', 'pty.node')
if (this.cli.dryRun) { if (this.cli.dryRun) {
if (target.platform === 'linux') console.log(`build-exe-for-python-sdk: [dry-run] cp ${source} ${destination}`) console.log(`build-exe-for-python-sdk: [dry-run] cp ${source} ${destination}`)
return return
} }
if (target.platform === 'macos') return
const host = Target.host() const host = Target.host()
if (target.platform !== host.platform || target.arch !== host.arch || !existsSync(source)) { if (target.platform !== host.platform || target.arch !== host.arch) {
throw new Error( throw new Error(
`build-exe-for-python-sdk: node-pty native addon for ${target.platform}-${target.arch} is missing; ` 'build-exe-for-python-sdk: build the Linux runtime on its target architecture; '
+ `checked ${source}. Build the Linux runtime on its target architecture.`, + `target ${target.platform}-${target.arch} does not match host ${host.platform}-${host.arch}.`,
) )
} }
await mkdir(dirname(destination), { recursive: true }) await mkdir(dirname(destination), { recursive: true })
await copyFile(source, destination) await copyFile(source, destination)
} }
/**
* Resolve the node-pty helper that matches a pkg target.
* @param target - the pkg target whose helper must be shipped.
* @returns a physical executable outside pkg's virtual snapshot.
*/
private resolveSpawnHelper(target: Target): string {
const helper = join(this.staging, 'node_modules', 'node-pty', 'prebuilds', `darwin-${target.arch}`, 'spawn-helper')
if (!existsSync(helper)) {
throw new Error(
`build-exe-for-python-sdk: node-pty spawn-helper for ${target.platform}-${target.arch} is missing; `
+ `checked ${helper}. Build each runtime on its target platform and architecture.`,
)
}
if (statSync(helper).mode & 0o111) return helper
throw new Error(`build-exe-for-python-sdk: node-pty spawn-helper is not executable: ${helper}`)
}
/** /**
* Print each product path and, outside dry-run mode, its size. * Print each product path and, outside dry-run mode, its size.
* @param products - the product paths returned by {@link pack}. * @param products - the product paths returned by {@link pack}.
@@ -397,7 +376,7 @@ class SingleExeBuild {
} }
return return
} }
mkdirSync(destDir, { recursive: true }) await mkdir(destDir, { recursive: true })
for (const path of products) { for (const path of products) {
const destination = join(destDir, basename(path)) const destination = join(destDir, basename(path))
await copyFile(path, destination) await copyFile(path, destination)

View File

@@ -22,7 +22,10 @@ PLATFORMS = {
"linux-arm64": ("manylinux_2_28_aarch64", "dsh-jsonrpc-agent-pkg-linux-arm64"), "linux-arm64": ("manylinux_2_28_aarch64", "dsh-jsonrpc-agent-pkg-linux-arm64"),
"macos-arm64": ("macosx_11_0_arm64", "dsh-jsonrpc-agent-pkg-macos-arm64"), "macos-arm64": ("macosx_11_0_arm64", "dsh-jsonrpc-agent-pkg-macos-arm64"),
} }
SPAWN_HELPER_SUFFIX = "-spawn-helper"
def runtime_suffixes(executable_name: str) -> tuple[str, ...]:
return ("", "-spawn-helper") if "-macos-" in executable_name else ("",)
def main() -> None: def main() -> None:
@@ -133,24 +136,12 @@ def stage_sdk(destination: Path, version: str) -> None:
def stage_runtime(destination: Path, version: str, executable: Path, executable_name: str) -> None: def stage_runtime(destination: Path, version: str, executable: Path, executable_name: str) -> None:
payload = [(executable, executable_name)]
if "-macos-" in executable_name:
payload.append(
(Path(f"{executable}{SPAWN_HELPER_SUFFIX}"), f"{executable_name}{SPAWN_HELPER_SUFFIX}")
)
for source, _ in payload:
if not source.is_file():
raise FileNotFoundError(f"runtime file does not exist: {source}")
if source.stat().st_mode & stat.S_IXUSR == 0:
raise PermissionError(f"runtime file is not executable: {source}")
copy_package(ROOT / "python" / "sdk-runtime", destination) copy_package(ROOT / "python" / "sdk-runtime", destination)
rewrite_version(destination / "pyproject.toml", version) rewrite_version(destination / "pyproject.toml", version)
runtime_dir = destination / "src" / "deepseek_harness_runtime" / "runtime" runtime_dir = destination / "src" / "deepseek_harness_runtime" / "runtime"
runtime_dir.mkdir(parents=True, exist_ok=True) runtime_dir.mkdir(parents=True, exist_ok=True)
for source, name in payload: for suffix in runtime_suffixes(executable_name):
target = runtime_dir / name shutil.copy2(Path(f"{executable}{suffix}"), runtime_dir / f"{executable_name}{suffix}")
shutil.copyfile(source, target)
target.chmod(source.stat().st_mode & 0o777)
def verify_wheel( def verify_wheel(
@@ -174,9 +165,7 @@ def verify_wheel(
] ]
if package == "runtime": if package == "runtime":
assert platform is not None assert platform is not None
expected_files = [platform[1]] expected_files = [f"{platform[1]}{suffix}" for suffix in runtime_suffixes(platform[1])]
if "-macos-" in platform[1]:
expected_files.append(f"{platform[1]}{SPAWN_HELPER_SUFFIX}")
found_files = sorted(Path(name).name for name in runtime_files) found_files = sorted(Path(name).name for name in runtime_files)
if found_files != expected_files: if found_files != expected_files:
raise RuntimeError(f"{wheel} runtime payload must be {expected_files}, found {found_files}") raise RuntimeError(f"{wheel} runtime payload must be {expected_files}, found {found_files}")