diff --git a/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.i18n.yaml b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.i18n.yaml index ea3a57e072..1b14f5b689 100644 --- a/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.i18n.yaml +++ b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write -2026-07-22-evidence-based-larger-hosted-runners.md: 497c6f297d79245fb40cd30457e4b1d1e36db651 -2026-07-22-evidence-based-larger-hosted-runners.zh.md: bcb0c6e9f11081b2cff696a9b6b425a40ee4aeb4 +2026-07-22-evidence-based-larger-hosted-runners.md: 6654f5eb3e21b48c6d33fd9d74ebd23cf3065d54 +2026-07-22-evidence-based-larger-hosted-runners.zh.md: 3fe5715b20d3b881f8fb439b61900bdb84e5a588 diff --git a/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.md b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.md index 497c6f297d..6654f5eb3e 100644 --- a/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.md +++ b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.md @@ -12,13 +12,13 @@ Larger runners make it possible to pay setup once and parallelize inside the rep ## Decision -The enterprise keeps repo-restricted x64 larger-runner pools for Ubuntu and Windows. Ordinary pull requests name two 32-core hosted pools directly: Ubuntu latest for the remaining primary Node 24 inventory and Windows 2025 for blocking Windows contracts. Exhaustive coverage moved off the metered Ubuntu 24.04 32-core pool onto the in-house self-hosted pool (`vm-backup` label: a 64-core VM running four always-on systemd-managed runner instances plus four registered spares) for trusted same-repo PRs; untrusted PRs — forks and Dependabot — keep coverage on the hosted Ubuntu 24.04 32-core pool so dependency-supplied code never reaches the persistent VM. **Precondition: repository forking stays disabled.** The workflow's pool selector is defense-in-depth only — `pull_request` executes the PR's own workflow definition, so YAML cannot enforce runner trust against a fork that edits it. Disabled forking (org-side, not PR-editable) is the enforcement boundary; the planned public release is an isolated read-only mirror under a separate org, preserving this. Before forking is ever enabled, the runners must first move into an org-level runner group restricted to this repository with base-branch workflow pinning — that migration is the gate, not a follow-up. Public IPs are disabled, and workflow concurrency remains bounded because an autoscaling ceiling neither allocates idle machines nor makes repository work scale without limit. +The enterprise keeps repo-restricted x64 larger-runner pools for Ubuntu and Windows. Ordinary pull requests name three 32-core pools directly: Ubuntu 24.04 for exhaustive coverage, Ubuntu latest for the remaining primary Node 24 inventory, and Windows 2025 for blocking Windows contracts. Public IPs are disabled, and workflow concurrency remains bounded because an autoscaling ceiling neither allocates idle machines nor makes repository work scale without limit. The required primary path depends on those enterprise pools. Standard GitHub-hosted jobs retain the Node 22.19, Node 26, and Python SDK compatibility contracts, while the [portable recovery boundary](2026-07-23-portable-required-pull-request-ci.md) and [serial reference](2026-07-21-serial-cross-platform-ci-reference.md) keep complete standard-runner evidence available on `master`. `suite=larger-runner-benchmark` compares isolated critical lanes across provisioned sizes, and `suite=consolidated-runner-benchmark` compares whole aggregates. Each benchmark reports its observed processor and memory capacity before running repository work. The former gate-level and coarse primary shard jobs are absent from the workflow. Their static, lint, coverage, snapshot, and scenario shard selectors are also absent from the repository, so an unused diagnostic path cannot preserve a second CI architecture. -Linux primary work uses three independent jobs: static gates and the consumer tail on hosted 32-core pools, and coverage on the in-house self-hosted 64-core pool for trusted PRs (hosted 32-core for untrusted ones). Coverage runs alone with its own worker bound, and the static scheduler runs alone so its result has no post-build consumer tail. After static gates finish, that job publishes its emitted `apps/*/lib`, `packages/*/*/lib`, and `vendor/*/lib` tree as a run-scoped artifact. The third job restores that exact tree, then starts lint, Node 24 runtime compatibility, build-backed snapshots, and all artifact consumers without repeating the build. Generated NodeNext consumer directories are excluded from ESLint discovery because the artifact check removes them while these processes overlap. The pnpm store and ESLint cache are restored without putting cache uploads on the pull-request critical path. Performance reports use each job's `startedAt` to `completedAt` interval; runner queue delay is capacity evidence, not repository execution time. +Linux primary work uses three independent 32-core jobs. Coverage runs alone with its own worker bound, and the static scheduler runs alone so its result has no post-build consumer tail. After static gates finish, that job publishes its emitted `apps/*/lib`, `packages/*/*/lib`, and `vendor/*/lib` tree as a run-scoped artifact. The third job restores that exact tree, then starts lint, Node 24 runtime compatibility, build-backed snapshots, and all artifact consumers without repeating the build. Generated NodeNext consumer directories are excluded from ESLint discovery because the artifact check removes them while these processes overlap. The pnpm store and ESLint cache are restored without putting cache uploads on the pull-request critical path. Performance reports use each job's `startedAt` to `completedAt` interval; runner queue delay is capacity evidence, not repository execution time. Windows shares one 32-core setup across the blocking build and production site plus observational built-artifact contracts. Linux owns the duplicate lint, coverage, and snapshot inventories because running those observational copies on Windows extends the paid critical path without adding a blocking platform claim. @@ -48,6 +48,8 @@ The process-bound coverage project contains exactly five suite files. Thirty-two Complete serial Linux, macOS, and Windows references run only when `master` moves. Pull requests use the enterprise required path plus standard-hosted compatibility jobs, while other larger-runner sizes run only by manual dispatch. +An additional serial Linux reference runs on the in-house self-hosted pool (`vm-backup` label: a 64-core VM with four always-on systemd-managed runner instances plus four registered spares) on every `master` push. It is a hot-standby drill, not a required check: each run re-proves that the persistent VM can execute the complete unsharded aggregate, so if the enterprise pools degrade, a required lane can be retargeted with a one-line `runs-on` change onto an environment with continuously verified evidence. Because the lane is push-triggered, it always executes the base branch's workflow definition — no pull-request-editable path can route code to these runners, and the repository additionally keeps forking disabled. + ## Alternatives considered **Keep the three coarse primary Linux lanes.** The core, CPU, and production-site jobs met the latency targets, but they paid three setup waves and left primary Node work sharded after larger runners were available. The all-size trace showed that one unnecessary dependency, not a lack of host capacity, kept the single-box aggregate above one minute. diff --git a/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.zh.md b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.zh.md index bcb0c6e9f1..3fe5715b20 100644 --- a/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.zh.md +++ b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.zh.md @@ -12,13 +12,13 @@ Status: implemented ## 决策 -企业保留仅限本仓库使用的 Ubuntu 和 Windows x64 大型运行器池。普通拉取请求直接指定 2 个 32 核托管运行器池：Ubuntu latest 用于其余主 Node 24 清单，Windows 2025 用于阻塞性 Windows 契约。完整覆盖率已从计费的 Ubuntu 24.04 32 核池迁移至公司自有的自托管池（`vm-backup` 标签：一台 64 核虚拟机，运行 4 个常驻的 systemd 管理运行器实例，另有 4 个已注册备用位），仅面向可信的同仓库拉取请求；不可信的拉取请求——fork 与 Dependabot——的覆盖率仍在托管的 Ubuntu 24.04 32 核池上运行，确保依赖方提供的代码永远不会进入持久化虚拟机。**前置条件：仓库必须保持禁用 fork。**工作流中的运行器池选择表达式仅是纵深防御——`pull_request` 执行的是拉取请求自带的工作流定义，因此 YAML 无法对能修改它的 fork 实施运行器信任约束。真正的强制边界是组织侧（拉取请求无法修改）的 fork 禁用设置；规划中的开源发布采用独立组织下的只读镜像仓库，正是为了保持这一边界。将来若要启用 fork，必须先把运行器迁入组织级 runner group（限定本仓库并绑定基线分支工作流）——该迁移是启用 fork 的先决门槛，而非事后跟进项。公网 IP 已禁用；工作流并发仍设有边界，因为自动扩缩容上限既不会分配闲置机器，也不意味着仓库工作可以无限扩展。 +企业保留仅限本仓库使用的 Ubuntu 和 Windows x64 大型运行器池。普通拉取请求直接指定 3 个 32 核运行器池：Ubuntu 24.04 用于完整覆盖率，Ubuntu latest 用于其余主 Node 24 清单，Windows 2025 用于阻塞性 Windows 契约。公网 IP 已禁用；工作流并发仍设有边界，因为自动扩缩容上限既不会分配闲置机器，也不意味着仓库工作可以无限扩展。 必需主路径依赖这些企业级运行器池。GitHub 标准托管作业保留 Node 22.19、Node 26 和 Python SDK 兼容性契约，而[可移植恢复边界](2026-07-23-portable-required-pull-request-ci.md)与[串行参考流程](2026-07-21-serial-cross-platform-ci-reference.md)则在 `master` 上持续提供完整的标准运行器证据。`suite=larger-runner-benchmark` 比较已预配规格上相互独立的关键通道，`suite=consolidated-runner-benchmark` 则比较完整聚合流程。每项基准测试都会先报告实测的处理器和内存容量，再运行仓库工作。 原有的门禁级和粗粒度主流程分片作业已从工作流中移除。相应的静态、lint、覆盖率、快照和场景分片选择器也已从仓库中移除，因此未使用的诊断路径无法继续维系第二套 CI 架构。 -Linux 主流程使用 3 个相互独立的作业：静态门禁与消费方尾部作业运行在托管 32 核池上，覆盖率对可信拉取请求运行在公司自有的自托管 64 核池上（不可信请求仍用托管 32 核池）。覆盖率单独运行，并设有自己的工作线程上限；静态调度器也单独运行，因此构建后的消费方不会拖延其结果。静态门禁完成后，该作业将其生成的 `apps/*/lib`、`packages/*/*/lib` 和 `vendor/*/lib` 目录树作为仅供本次运行使用的产物发布。第三个作业恢复完全相同的目录树，再让 lint、Node 24 运行时兼容性、依赖构建产物的快照和所有产物消费方基于构建完成后的工作树启动，而不重复构建。生成的 NodeNext 消费方目录不会纳入 ESLint 的文件发现范围，因为这些进程重叠执行时，产物检查会删除这些目录。pnpm store 和 ESLint 缓存会得到恢复，但缓存上传不会进入拉取请求关键路径。性能报告采用每个作业从 `startedAt` 到 `completedAt` 的区间；运行器排队延迟是容量证据，而非仓库执行时间。 +Linux 主流程使用 3 个相互独立的 32 核作业。覆盖率单独运行，并设有自己的工作线程上限；静态调度器也单独运行，因此构建后的消费方不会拖延其结果。静态门禁完成后，该作业将其生成的 `apps/*/lib`、`packages/*/*/lib` 和 `vendor/*/lib` 目录树作为仅供本次运行使用的产物发布。第三个作业恢复完全相同的目录树，再让 lint、Node 24 运行时兼容性、依赖构建产物的快照和所有产物消费方基于构建完成后的工作树启动，而不重复构建。生成的 NodeNext 消费方目录不会纳入 ESLint 的文件发现范围，因为这些进程重叠执行时，产物检查会删除这些目录。pnpm store 和 ESLint 缓存会得到恢复，但缓存上传不会进入拉取请求关键路径。性能报告采用每个作业从 `startedAt` 到 `completedAt` 的区间；运行器排队延迟是容量证据，而非仓库执行时间。 Windows 以一次 32 核环境设置同时承载阻塞性构建、生产网站和观测性的构建产物契约。重复的 lint、覆盖率和快照清单由 Linux 承担，因为在 Windows 上运行这些观测性副本会延长付费关键路径，却不会新增任何阻塞性平台契约。 @@ -48,6 +48,8 @@ Windows 仓库工作在超过 16 核后收益很小，但 32 核池可以让完 只有在 `master` 移动时，才运行完整的 Linux、macOS 和 Windows 串行参考。拉取请求使用企业级运行器必需路径和标准托管兼容性作业，其他大型运行器规格仅通过手动触发运行。 +另有一条串行 Linux 参考在每次 `master` 推送时运行于公司自有的自托管池（`vm-backup` 标签：一台 64 核虚拟机，运行 4 个常驻的 systemd 管理运行器实例，另有 4 个已注册备用位）。它是热备演练而非必需检查：每次运行都重新证明这台持久化虚拟机能够执行完整的未分片聚合流程，因此当企业池发生故障时，只需一行 `runs-on` 修改即可把必需通道切换到一个具有持续验证证据的环境上。该通道由 push 触发，执行的始终是基线分支自身的工作流定义——不存在任何可由拉取请求编辑的路径能把代码路由到这些运行器上；此外仓库继续保持禁用 fork。 + ## 曾考虑的替代方案 **保留 3 个粗粒度 Linux 主流程通道。** 核心、CPU 和生产网站作业均达到延迟目标，但它们需要 3 轮设置，而且在大型运行器已经可用后仍对主 Node 工作进行分片。全规格运行轨迹表明，让单机聚合流程超过 1 分钟的是一项不必要的依赖，而非主机容量不足。 diff --git a/.agents/notes/implemented/process/2026-07-23-portable-required-pull-request-ci.i18n.yaml b/.agents/notes/implemented/process/2026-07-23-portable-required-pull-request-ci.i18n.yaml index ed97fe08a7..f8b54b0ec5 100644 --- a/.agents/notes/implemented/process/2026-07-23-portable-required-pull-request-ci.i18n.yaml +++ b/.agents/notes/implemented/process/2026-07-23-portable-required-pull-request-ci.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write -2026-07-23-portable-required-pull-request-ci.md: 29b2cfa3f431a4a8be4aaa685b16cffdb4bf2593 -2026-07-23-portable-required-pull-request-ci.zh.md: 8b6d067637ce83c09529977f463f16dfa4af5a8b +2026-07-23-portable-required-pull-request-ci.md: 9cf8d97016300c5258c075879176aa6abd64e59e +2026-07-23-portable-required-pull-request-ci.zh.md: c6839a133d0c3fe7a699362f6168e17d827a5b61 diff --git a/.agents/notes/implemented/process/2026-07-23-portable-required-pull-request-ci.md b/.agents/notes/implemented/process/2026-07-23-portable-required-pull-request-ci.md index 29b2cfa3f4..9cf8d97016 100644 --- a/.agents/notes/implemented/process/2026-07-23-portable-required-pull-request-ci.md +++ b/.agents/notes/implemented/process/2026-07-23-portable-required-pull-request-ci.md @@ -12,7 +12,7 @@ Billing health, a runner definition's `Ready` state, and a large autoscaling cei ## Decision -[CI](../../../../.github/workflows/ci.yml) runs the required primary Node 24 and Windows jobs on repo-restricted enterprise 32-core pools, except exhaustive coverage, which runs on the in-house self-hosted 64-core pool for trusted same-repo pull requests (hosted 32-core for forks and Dependabot). Standard `ubuntu-latest` jobs retain Node 22.19, Node 26, and Python SDK compatibility, and `master` runs complete serial Linux, macOS, and Windows references. Those standard-hosted jobs keep the portable execution boundary observable without duplicating the primary inventory on every pull request. +[CI](../../../../.github/workflows/ci.yml) runs the required primary Node 24 and Windows jobs on repo-restricted enterprise 32-core pools. Standard `ubuntu-latest` jobs retain Node 22.19, Node 26, and Python SDK compatibility, and `master` runs complete serial Linux, macOS, and Windows references. Those standard-hosted jobs keep the portable execution boundary observable without duplicating the primary inventory on every pull request. The two Linux primary jobs, Node compatibility, Python SDK, and `windows node 24 / complete` remain dependencies of `all checks passed`; branch protection continues to require `e2e` and `all checks passed`. There is no automatic fallback when an enterprise label cannot allocate: the standard jobs continue to report their own contracts, but they cannot manufacture the missing required result. diff --git a/.agents/notes/implemented/process/2026-07-23-portable-required-pull-request-ci.zh.md b/.agents/notes/implemented/process/2026-07-23-portable-required-pull-request-ci.zh.md index 8b6d067637..c6839a133d 100644 --- a/.agents/notes/implemented/process/2026-07-23-portable-required-pull-request-ci.zh.md +++ b/.agents/notes/implemented/process/2026-07-23-portable-required-pull-request-ci.zh.md @@ -12,7 +12,7 @@ Status: implemented ## 决策 -[CI](../../../../.github/workflows/ci.yml) 在仅限本仓库使用的企业级 32 核运行器池上运行必需的主 Node 24 作业和 Windows 作业；唯一例外是完整覆盖率——可信的同仓库拉取请求在公司自有的自托管 64 核池上运行（fork 与 Dependabot 仍用托管 32 核池）。标准 `ubuntu-latest` 作业保留 Node 22.19、Node 26 和 Python SDK 兼容性，`master` 则运行完整的 Linux、macOS 和 Windows 串行参考流程。这些标准托管作业让可移植执行边界保持可观测，而不必在每个拉取请求中重复主清单。 +[CI](../../../../.github/workflows/ci.yml) 在仅限本仓库使用的企业级 32 核运行器池上运行必需的主 Node 24 作业和 Windows 作业。标准 `ubuntu-latest` 作业保留 Node 22.19、Node 26 和 Python SDK 兼容性，`master` 则运行完整的 Linux、macOS 和 Windows 串行参考流程。这些标准托管作业让可移植执行边界保持可观测，而不必在每个拉取请求中重复主清单。 两项 Linux 主作业、Node 兼容性、Python SDK 和 `windows node 24 / complete` 继续作为 `all checks passed` 的依赖项；分支保护继续要求 `e2e` 和 `all checks passed`。企业级运行器标签无法分配运行器时没有自动后备机制：标准作业会继续报告各自的契约，但无法产出缺失的必需结果。 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dc5ad98ec4..2666c93b8c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,10 +27,8 @@ env: jobs: - # Three independent Linux jobs isolate coverage, static analysis, and the - # build-backed consumer tail: static and consumers on hosted enterprise - # 32-core pools; coverage on the in-house self-hosted pool for trusted PRs - # (hosted for forks/Dependabot). The static job publishes its exact build so + # Three enterprise jobs isolate coverage, static analysis, and the + # build-backed consumer tail. The static job publishes its exact build so # consumers do not repeat the longest part of their critical path. node-24: if: github.event_name == 'pull_request' @@ -79,46 +77,17 @@ jobs: node-24-coverage: if: github.event_name == 'pull_request' - # Trusted same-repo PRs run on the in-house pool (self-hosted, 64-core; - # 4 always-on systemd-managed instances plus 4 registered spares) instead - # of the metered enterprise pool. Untrusted PRs — forks and Dependabot - # (same-repo but dependency-supplied code; same author test as e2e.yml) — - # stay on the hosted enterprise pool so no untrusted code reaches the - # persistent self-hosted VM. Selecting the pool via runs-on keeps this a - # single job, so the all-checks-passed aggregate never sees a skip. - runs-on: >- - ${{ (github.event.pull_request.head.repo.full_name != github.repository - || github.event.pull_request.user.login == 'dependabot[bot]') - && 'dsh-enterprise-ubuntu-24-04-32core-test' - || fromJSON('["self-hosted", "linux", "x64", "vm-backup"]') }} + runs-on: dsh-enterprise-ubuntu-24-04-32core-test name: node 24 / coverage env: - # Worker bound is per-leg: the hosted 32-core runner is exclusive to - # one job, but the self-hosted pool shares one 64-core VM across four - # runner instances, so concurrent PRs would otherwise stack up to - # 4×24 = 96 workers and re-trigger the aggregate-contention failures - # documented for the timing-sensitive process suites. 12 per job caps - # the shared host at 48 workers even fully loaded. - DSH_COVERAGE_MAX_WORKERS: >- - ${{ (github.event.pull_request.head.repo.full_name != github.repository - || github.event.pull_request.user.login == 'dependabot[bot]') - && '24' || '12' }} + DSH_COVERAGE_MAX_WORKERS: '24' DSH_GATE_CONCURRENCY: '8' steps: - uses: actions/checkout@v6 with: persist-credentials: false - # Restore the pnpm-store cache only on the hosted (untrusted-PR) leg, - # where the VM is ephemeral and the same-region download is fast. On - # the self-hosted leg pnpm's persistent store lives outside - # /home/runner, so this restore would spend ~52 s pulling ~180 MB into - # a path pnpm never reads (measured; install then took 2.8 s straight - # from the persistent store). Condition mirrors the runs-on selector. - uses: actions/cache/restore@v4 - if: >- - github.event.pull_request.head.repo.full_name != github.repository - || github.event.pull_request.user.login == 'dependabot[bot]' with: path: /home/runner/.local/share/pnpm/store/v11 key: ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-pnpm-${{ hashFiles('pnpm-lock.yaml') }} @@ -396,6 +365,45 @@ jobs: DSH_SNAPSHOT_MAX_CONCURRENCY: '1' run: pnpm run check:ci + # Hot-standby drill for the in-house self-hosted pool: every master move + # re-runs the complete unsharded aggregate on the persistent 64-core VM, + # continuously proving that environment can take over a required lane if + # the hosted pools degrade (the switch is then a one-line runs-on change). + # Push-triggered, so it always executes the base branch's own workflow + # definition — no PR-editable path selects these runners. Non-blocking for + # pull requests; no cache steps because the VM's persistent pnpm store and + # tool caches make them redundant (and saving here would poison the hosted + # cache namespace with self-hosted paths). + serial-linux-selfhosted: + if: github.event_name == 'push' && github.ref == 'refs/heads/master' + name: serial / linux (self-hosted standby) + runs-on: [self-hosted, linux, x64, vm-backup] + steps: + - uses: actions/checkout@v6 + + - uses: actions/setup-node@v6 + with: + node-version: ${{ env.PRIMARY_NODE_VERSION }} + + - name: Enable corepack (pnpm) + run: corepack enable + + - name: Install (immutable) + run: pnpm install --frozen-lockfile + + - name: Prepare bubblewrap (unrestrict userns) + run: bash scripts/prepare-ci-bubblewrap.sh + + - name: Run complete unsharded primary Node CI serially + env: + DSH_COVERAGE_MAX_WORKERS: '1' + DSH_E2E_MAX_WORKERS: '1' + DSH_ESLINT_CACHE: '1' + DSH_GATE_CONCURRENCY: '1' + DSH_PUBLINT_CONCURRENCY: '1' + DSH_SNAPSHOT_MAX_CONCURRENCY: '1' + run: pnpm run check:ci + serial-macos: if: github.event_name == 'push' && github.ref == 'refs/heads/master' name: serial / macos