From 3d965082443d7216e8755b122652255226095aa3 Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Tue, 21 Jul 2026 20:51:28 +0800 Subject: [PATCH] ci: restore hosted-run headroom --- .../2026-07-06-parallel-github-ci-gates.md | 4 +- .github/workflows/ci.yml | 24 +++++++++-- scripts/lint-shards.spec.ts | 34 ++++++++++++++++ scripts/lint-shards.ts | 40 +++++++++++++++++++ scripts/run-gates.ts | 13 +++--- scripts/static-shards.ts | 2 +- 6 files changed, 105 insertions(+), 12 deletions(-) create mode 100644 scripts/lint-shards.spec.ts create mode 100644 scripts/lint-shards.ts diff --git a/.agents/notes/implemented/process/2026-07-06-parallel-github-ci-gates.md b/.agents/notes/implemented/process/2026-07-06-parallel-github-ci-gates.md index 0058a30660..a76c425da7 100644 --- a/.agents/notes/implemented/process/2026-07-06-parallel-github-ci-gates.md +++ b/.agents/notes/implemented/process/2026-07-06-parallel-github-ci-gates.md @@ -14,9 +14,9 @@ The artifact boundary remains load-bearing. `publint`, `verify-node-next-types`, [CI](../../../../.github/workflows/ci.yml) bounds every non-Windows job to one minute and every Windows job to three minutes. The timeout is an executable regression ceiling; the lane design leaves headroom below it rather than treating a timeout as normal control flow. -[scripts/run-gates.ts](../../../../scripts/run-gates.ts) remains the common bounded scheduler, but GitHub supplies explicit shard names for the expensive gate families. [scripts/static-shards.ts](../../../../scripts/static-shards.ts) partitions static gates into foundation, API-contract, catalog, prose, and documentation-site lanes and rejects a missing or duplicate gate assignment. [scripts/coverage-shards.ts](../../../../scripts/coverage-shards.ts) assigns every workspace package to exactly one source-coverage lane; its test expands the live package tree, so a new package makes CI red until it has an owner. Each coverage lane includes only its owned source files, repeats the exhaustive companion topology test, and runs without a preceding build because the complete coverage suite passes from a tree with every generated `lib/` removed. +[scripts/run-gates.ts](../../../../scripts/run-gates.ts) remains the common bounded scheduler, but GitHub supplies explicit shard names for the expensive gate families. [scripts/static-shards.ts](../../../../scripts/static-shards.ts) partitions static gates into foundation, API-contract, catalog, prose, and documentation-site lanes and rejects a missing or duplicate gate assignment. Lint uses disjoint package-source, package-test, and repository-complement lanes; the complement still starts from `.` so a new top-level lint target cannot disappear between shards, and it owns the single cross-file duplication run. [scripts/coverage-shards.ts](../../../../scripts/coverage-shards.ts) assigns every workspace package to exactly one source-coverage lane; its test expands the live package tree, so a new package makes CI red until it has an owner. Each coverage lane includes only its owned source files, repeats the exhaustive companion topology test, and runs without a preceding build because the complete coverage suite passes from a tree with every generated `lib/` removed. -Snapshot replay is four Vitest file shards. Each snapshot job builds the shipped runtime while its Linux runner installs bubblewrap, then runs only its assigned replay files. Static, coverage, and snapshot sharding changes only GitHub scheduling: the ordinary local package scripts still run their complete suites. +Snapshot replay is four Vitest file shards. Each snapshot job builds the shipped runtime while its Linux runner installs bubblewrap from the hosted image's existing package index, then runs only its assigned replay files. CI explicitly retains the suite's bounded concurrency of five subprocesses instead of clamping it to the runner's two logical CPUs, because replay spends most of its time waiting on child protocol I/O. Static, lint, coverage, and snapshot sharding changes only GitHub scheduling: the ordinary local package scripts still run their complete suites. Artifacts use three lanes: one metadata lane for `publint`, NodeNext declarations, and compiled invariant loading, plus two Vitest shards for built-bin smoke. Each lane produces its own build before its consumers. Repeating the short build costs runner minutes but avoids an upload/download dependency and keeps each job's critical path bounded. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a8dd6f9ec0..1ae8eb35de 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,7 +26,9 @@ jobs: DSH_ARTIFACT_SHARD: ${{ matrix.artifact_shard }} DSH_COVERAGE_MAX_WORKERS: ${{ matrix.coverage_max_workers }} DSH_COVERAGE_SHARD: ${{ matrix.coverage_shard }} + DSH_LINT_SHARD: ${{ matrix.lint_shard }} DSH_STATIC_SHARD: ${{ matrix.static_shard }} + DSH_SNAPSHOT_MAX_CONCURRENCY: ${{ matrix.snapshot_max_concurrency }} DSH_SNAPSHOT_PREBUILT: ${{ matrix.snapshot_prebuilt }} DSH_SNAPSHOT_SHARD: ${{ matrix.snapshot_shard }} DSH_ESLINT_CACHE: ${{ matrix.eslint_cache }} @@ -54,10 +56,21 @@ jobs: command: pnpm run check:ci:static gate_concurrency: '1' static_shard: site - - lane: lint + - lane: lint-package-sources command: pnpm run check:ci:lint gate_concurrency: '1' eslint_cache: '1' + lint_shard: package-sources + - lane: lint-package-tests + command: pnpm run check:ci:lint + gate_concurrency: '1' + eslint_cache: '1' + lint_shard: package-tests + - lane: lint-repository + command: pnpm run check:ci:lint + gate_concurrency: '1' + eslint_cache: '1' + lint_shard: repository - lane: coverage-core command: pnpm run check:ci:coverage gate_concurrency: '1' @@ -141,21 +154,25 @@ jobs: - lane: snapshot-1 command: pnpm run check:ci:snapshot gate_concurrency: '1' + snapshot_max_concurrency: '5' snapshot_prebuilt: '1' snapshot_shard: '1/4' - lane: snapshot-2 command: pnpm run check:ci:snapshot gate_concurrency: '1' + snapshot_max_concurrency: '5' snapshot_prebuilt: '1' snapshot_shard: '2/4' - lane: snapshot-3 command: pnpm run check:ci:snapshot gate_concurrency: '1' + snapshot_max_concurrency: '5' snapshot_prebuilt: '1' snapshot_shard: '3/4' - lane: snapshot-4 command: pnpm run check:ci:snapshot gate_concurrency: '1' + snapshot_max_concurrency: '5' snapshot_prebuilt: '1' snapshot_shard: '4/4' - lane: artifacts-metadata @@ -207,8 +224,7 @@ jobs: pnpm run build & build_pid=$! ( - sudo apt-get update -q - sudo apt-get install -yq bubblewrap + sudo apt-get install -yq --no-install-recommends bubblewrap sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 \ || echo "apparmor userns knob absent — the functional probe decides" ) & @@ -221,7 +237,7 @@ jobs: exit "$sandbox_status" - uses: actions/cache@v4 - if: matrix.lane == 'lint' + if: startsWith(matrix.lane, 'lint-') with: path: .cache/eslint key: ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-eslint-${{ hashFiles('pnpm-lock.yaml', 'eslint.config.mjs', 'tsconfig.json', 'packages/*/*/tsconfig.json', 'examples/*/tsconfig.json') }} diff --git a/scripts/lint-shards.spec.ts b/scripts/lint-shards.spec.ts new file mode 100644 index 0000000000..df50790260 --- /dev/null +++ b/scripts/lint-shards.spec.ts @@ -0,0 +1,34 @@ +import { describe, expect, it } from 'vitest' +import { selectLintShard } from './lint-shards.ts' + +describe('lint gate shards', () => { + it('keeps the unsharded local command complete', () => { + expect(selectLintShard()).toEqual({ eslintTargets: ['.'], includeDuplication: true }) + expect(selectLintShard('')).toEqual({ eslintTargets: ['.'], includeDuplication: true }) + }) + + it('partitions package sources, package tests, and their repository complement', () => { + expect(selectLintShard('package-sources')).toEqual({ + eslintTargets: ['packages/*/*/src/**/*.ts'], + includeDuplication: false, + }) + expect(selectLintShard('package-tests')).toEqual({ + eslintTargets: ['packages/*/*/tests/**/*.ts'], + includeDuplication: false, + }) + expect(selectLintShard('repository')).toEqual({ + eslintTargets: [ + '.', + '--ignore-pattern', + 'packages/*/*/src/**', + '--ignore-pattern', + 'packages/*/*/tests/**', + ], + includeDuplication: true, + }) + }) + + it('rejects an unknown lane', () => { + expect(() => selectLintShard('missing')).toThrow('unknown DSH_LINT_SHARD') + }) +}) diff --git a/scripts/lint-shards.ts b/scripts/lint-shards.ts new file mode 100644 index 0000000000..ec09a7aca2 --- /dev/null +++ b/scripts/lint-shards.ts @@ -0,0 +1,40 @@ +/** Lint-lane selection for GitHub Actions. */ + +/** One ESLint target set and whether it owns the cross-file duplication gate. */ +export interface LintSelection { + /** Shell-free arguments passed to ESLint before its cache options. */ + eslintTargets: readonly string[] + /** Whether this lane also runs the repository-wide duplication check. */ + includeDuplication: boolean +} + +/** + * Select an exhaustive lint partition without changing the ordinary local lint command. + * + * @param name Optional stable shard name from `DSH_LINT_SHARD`. + * @returns ESLint targets and ownership of the duplication gate. + */ +export function selectLintShard(name?: string): LintSelection { + switch (name) { + case undefined: + case '': + return { eslintTargets: ['.'], includeDuplication: true } + case 'package-sources': + return { eslintTargets: ['packages/*/*/src/**/*.ts'], includeDuplication: false } + case 'package-tests': + return { eslintTargets: ['packages/*/*/tests/**/*.ts'], includeDuplication: false } + case 'repository': + return { + eslintTargets: [ + '.', + '--ignore-pattern', + 'packages/*/*/src/**', + '--ignore-pattern', + 'packages/*/*/tests/**', + ], + includeDuplication: true, + } + default: + throw new Error(`run-gates: unknown DSH_LINT_SHARD ${JSON.stringify(name)}.`) + } +} diff --git a/scripts/run-gates.ts b/scripts/run-gates.ts index 8eade29028..262fac886f 100644 --- a/scripts/run-gates.ts +++ b/scripts/run-gates.ts @@ -9,6 +9,7 @@ import { availableParallelism } from 'node:os' import { resolve } from 'node:path' import { performance } from 'node:perf_hooks' import { coverageArgs } from './coverage-shards.ts' +import { selectLintShard } from './lint-shards.ts' import { selectStaticGates } from './static-shards.ts' type Mode = @@ -162,11 +163,13 @@ function gatesForMode(selected: Mode): Gate[] { return ciPrimaryGates() case 'ci-static': return ciStaticGates() - case 'ci-lint': + case 'ci-lint': { + const selection = selectLintShard(process.env.DSH_LINT_SHARD) return [ - lintGate(), - pnpmScript('duplication', 'duplication'), + lintGate(selection.eslintTargets), + ...selection.includeDuplication ? [pnpmScript('duplication', 'duplication')] : [], ] + } case 'ci-coverage': return [coverageGate()] case 'ci-snapshot': @@ -267,11 +270,11 @@ function ciArtifactGates(): Gate[] { return [...metadataGates, builtBinSmokeGate()] } -function lintGate(): Gate { +function lintGate(eslintTargets: readonly string[] = ['.']): Gate { if (process.env.DSH_ESLINT_CACHE === '1') { return pnpmExec('lint', [ 'eslint', - '.', + ...eslintTargets, '--cache', '--cache-location', '.cache/eslint/', diff --git a/scripts/static-shards.ts b/scripts/static-shards.ts index 4d7a63b8c9..7376aa44d7 100644 --- a/scripts/static-shards.ts +++ b/scripts/static-shards.ts @@ -23,7 +23,7 @@ export const staticShards = [ }, { name: 'api-contracts', - gateIds: ['doc-typecheck', 'export-jsdoc', 'scoped-events', 'type-equivalence'], + gateIds: ['doc-typecheck', 'cordis-api', 'export-jsdoc', 'scoped-events', 'type-equivalence'], }, { name: 'catalogs',