ci: split primary cold-path budget
This commit is contained in:
61
.github/workflows/ci.yml
vendored
61
.github/workflows/ci.yml
vendored
@@ -319,19 +319,16 @@ jobs:
|
|||||||
run: ${{ matrix.command }}
|
run: ${{ matrix.command }}
|
||||||
|
|
||||||
# One large runner pays hosted setup once, then the repository scheduler
|
# One large runner pays hosted setup once, then the repository scheduler
|
||||||
# overlaps the unsharded primary inventory except the production site build.
|
# overlaps the unsharded core inventory. CPU and site chains use coarse lanes.
|
||||||
node-24:
|
node-24:
|
||||||
if: github.event_name != 'workflow_dispatch' || inputs.suite == 'optimized-larger-runners'
|
if: github.event_name != 'workflow_dispatch' || inputs.suite == 'optimized-larger-runners'
|
||||||
runs-on: dsh-ubuntu-24-04-96core
|
runs-on: dsh-ubuntu-24-04-96core
|
||||||
name: node 24 / core
|
name: node 24 / core
|
||||||
env:
|
env:
|
||||||
# Thirty-two Vitest forks intermittently crash Node's CJS lexer on this image.
|
|
||||||
DSH_COVERAGE_MAX_WORKERS: '16'
|
|
||||||
DSH_ESLINT_CACHE: '1'
|
DSH_ESLINT_CACHE: '1'
|
||||||
DSH_ESLINT_CONCURRENCY: '32'
|
DSH_ESLINT_CONCURRENCY: '32'
|
||||||
DSH_GATE_CONCURRENCY: '32'
|
DSH_GATE_CONCURRENCY: '32'
|
||||||
DSH_PUBLINT_CONCURRENCY: '32'
|
DSH_PUBLINT_CONCURRENCY: '32'
|
||||||
DSH_SNAPSHOT_MAX_CONCURRENCY: '32'
|
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v6
|
- uses: actions/checkout@v6
|
||||||
|
|
||||||
@@ -383,8 +380,8 @@ jobs:
|
|||||||
- name: Run unsharded primary Node core CI concurrently
|
- name: Run unsharded primary Node core CI concurrently
|
||||||
run: pnpm run check:ci:large-runner
|
run: pnpm run check:ci:large-runner
|
||||||
|
|
||||||
# Keep only the longest independent Linux gate on a second coarse-grained
|
# Keep the longest independent Linux gate on its own coarse-grained runner so
|
||||||
# runner so cold install variance does not push the primary box over a minute.
|
# cold install variance does not push either primary aggregate over a minute.
|
||||||
node-24-site:
|
node-24-site:
|
||||||
if: github.event_name != 'workflow_dispatch' || inputs.suite == 'optimized-larger-runners'
|
if: github.event_name != 'workflow_dispatch' || inputs.suite == 'optimized-larger-runners'
|
||||||
runs-on: dsh-ubuntu-24-04-16core
|
runs-on: dsh-ubuntu-24-04-16core
|
||||||
@@ -418,24 +415,39 @@ jobs:
|
|||||||
node-compat:
|
node-compat:
|
||||||
if: github.event_name != 'workflow_dispatch' || inputs.suite == 'optimized-larger-runners'
|
if: github.event_name != 'workflow_dispatch' || inputs.suite == 'optimized-larger-runners'
|
||||||
# Distinct larger-runner pools avoid both standard-runner setup outliers and
|
# Distinct larger-runner pools avoid both standard-runner setup outliers and
|
||||||
# the delayed second allocation observed when two jobs shared one pool.
|
# the delayed second allocation observed when two jobs shared one pool. The
|
||||||
|
# primary Node row also owns the CPU/dependency-critical primary gate chain.
|
||||||
runs-on: ${{ matrix.runner }}
|
runs-on: ${{ matrix.runner }}
|
||||||
name: node ${{ matrix.node }}
|
name: ${{ matrix.name }}
|
||||||
env:
|
env:
|
||||||
DSH_GATE_CONCURRENCY: '2'
|
DSH_COVERAGE_MAX_WORKERS: ${{ matrix.coverage_workers }}
|
||||||
|
DSH_GATE_CONCURRENCY: ${{ matrix.gate_concurrency }}
|
||||||
DSH_NODE_COMPAT_SKIP_TYPECHECK: ${{ matrix.skip_typecheck }}
|
DSH_NODE_COMPAT_SKIP_TYPECHECK: ${{ matrix.skip_typecheck }}
|
||||||
|
DSH_SNAPSHOT_MAX_CONCURRENCY: ${{ matrix.snapshot_workers }}
|
||||||
strategy:
|
strategy:
|
||||||
fail-fast: false
|
fail-fast: false
|
||||||
matrix:
|
matrix:
|
||||||
include:
|
include:
|
||||||
- node: '22.19'
|
- node: '22.19'
|
||||||
|
name: node 22.19
|
||||||
runner: dsh-ubuntu-24-04-4core
|
runner: dsh-ubuntu-24-04-4core
|
||||||
|
command: pnpm run check:node-compat
|
||||||
|
gate_concurrency: '2'
|
||||||
skip_typecheck: '1'
|
skip_typecheck: '1'
|
||||||
- node: 24
|
- node: 24
|
||||||
runner: dsh-ubuntu-24-04-8core
|
name: node 24 / cpu
|
||||||
|
runner: dsh-ubuntu-24-04-64core
|
||||||
|
command: pnpm run check:ci:primary-cpu
|
||||||
|
coverage_workers: '16'
|
||||||
|
gate_concurrency: '6'
|
||||||
|
primary_cpu: true
|
||||||
|
snapshot_workers: '16'
|
||||||
skip_typecheck: '1'
|
skip_typecheck: '1'
|
||||||
- node: 26
|
- node: 26
|
||||||
|
name: node 26
|
||||||
runner: dsh-ubuntu-24-04-32core
|
runner: dsh-ubuntu-24-04-32core
|
||||||
|
command: pnpm run check:node-compat
|
||||||
|
gate_concurrency: '2'
|
||||||
skip_typecheck: '1'
|
skip_typecheck: '1'
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v6
|
- uses: actions/checkout@v6
|
||||||
@@ -458,14 +470,37 @@ jobs:
|
|||||||
${{ runner.os }}-node-${{ matrix.node }}-pnpm-
|
${{ runner.os }}-node-${{ matrix.node }}-pnpm-
|
||||||
|
|
||||||
- name: Install (immutable)
|
- name: Install (immutable)
|
||||||
|
if: matrix.primary_cpu != true
|
||||||
run: pnpm install --frozen-lockfile
|
run: pnpm install --frozen-lockfile
|
||||||
|
|
||||||
- name: Run compatibility gates
|
- name: Install and prepare bubblewrap
|
||||||
run: pnpm run check:node-compat
|
if: matrix.primary_cpu == true
|
||||||
|
run: |
|
||||||
|
pnpm install --frozen-lockfile &
|
||||||
|
install_pid=$!
|
||||||
|
(
|
||||||
|
if ! sudo apt-get install -yq --no-install-recommends bubblewrap; then
|
||||||
|
echo "initial bubblewrap install failed; refreshing APT indexes and retrying"
|
||||||
|
sudo apt-get update -q
|
||||||
|
sudo apt-get install -yq --no-install-recommends bubblewrap
|
||||||
|
fi
|
||||||
|
sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 \
|
||||||
|
|| echo "apparmor userns knob absent — the functional probe decides"
|
||||||
|
) &
|
||||||
|
sandbox_pid=$!
|
||||||
|
install_status=0
|
||||||
|
wait "$install_pid" || install_status=$?
|
||||||
|
sandbox_status=0
|
||||||
|
wait "$sandbox_pid" || sandbox_status=$?
|
||||||
|
if (( install_status != 0 )); then exit "$install_status"; fi
|
||||||
|
exit "$sandbox_status"
|
||||||
|
|
||||||
|
- name: Run assigned gates
|
||||||
|
run: ${{ matrix.command }}
|
||||||
|
|
||||||
python-sdk:
|
python-sdk:
|
||||||
if: github.event_name != 'workflow_dispatch' || inputs.suite == 'optimized-larger-runners'
|
if: github.event_name != 'workflow_dispatch' || inputs.suite == 'optimized-larger-runners'
|
||||||
runs-on: dsh-ubuntu-24-04-64core
|
runs-on: dsh-ubuntu-24-04-8core
|
||||||
name: python 3.10 / keyless SDK
|
name: python 3.10 / keyless SDK
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v6
|
- uses: actions/checkout@v6
|
||||||
|
|||||||
@@ -26,6 +26,7 @@
|
|||||||
"test:snapshot:record": "DSH_SNAPSHOT=record vitest run --config vitest.snapshot.config.ts --update",
|
"test:snapshot:record": "DSH_SNAPSHOT=record vitest run --config vitest.snapshot.config.ts --update",
|
||||||
"test:snapshot:refresh": "DSH_SNAPSHOT=refresh vitest run --config vitest.snapshot.config.ts",
|
"test:snapshot:refresh": "DSH_SNAPSHOT=refresh vitest run --config vitest.snapshot.config.ts",
|
||||||
"check:ci": "tsx scripts/run-gates.ts ci-primary",
|
"check:ci": "tsx scripts/run-gates.ts ci-primary",
|
||||||
|
"check:ci:primary-cpu": "tsx scripts/run-gates.ts ci-primary-cpu",
|
||||||
"check:ci:large-runner": "tsx scripts/run-gates.ts ci-primary-large-runner",
|
"check:ci:large-runner": "tsx scripts/run-gates.ts ci-primary-large-runner",
|
||||||
"check:ci:static": "tsx scripts/run-gates.ts ci-static",
|
"check:ci:static": "tsx scripts/run-gates.ts ci-static",
|
||||||
"check:ci:lint": "tsx scripts/run-gates.ts ci-lint",
|
"check:ci:lint": "tsx scripts/run-gates.ts ci-lint",
|
||||||
|
|||||||
@@ -15,6 +15,7 @@ import { selectStaticGates } from './static-shards.ts'
|
|||||||
|
|
||||||
type Mode =
|
type Mode =
|
||||||
| 'ci-primary'
|
| 'ci-primary'
|
||||||
|
| 'ci-primary-cpu'
|
||||||
| 'ci-primary-large-runner'
|
| 'ci-primary-large-runner'
|
||||||
| 'ci-static'
|
| 'ci-static'
|
||||||
| 'ci-lint'
|
| 'ci-lint'
|
||||||
@@ -92,6 +93,7 @@ if (results.some(result => result.gate.allowFailure !== true && (result.status =
|
|||||||
function parseMode(raw: string | undefined): Mode {
|
function parseMode(raw: string | undefined): Mode {
|
||||||
switch (raw) {
|
switch (raw) {
|
||||||
case 'ci-primary':
|
case 'ci-primary':
|
||||||
|
case 'ci-primary-cpu':
|
||||||
case 'ci-primary-large-runner':
|
case 'ci-primary-large-runner':
|
||||||
case 'ci-static':
|
case 'ci-static':
|
||||||
case 'ci-lint':
|
case 'ci-lint':
|
||||||
@@ -107,7 +109,7 @@ function parseMode(raw: string | undefined): Mode {
|
|||||||
return raw
|
return raw
|
||||||
default:
|
default:
|
||||||
throw new Error(
|
throw new Error(
|
||||||
`run-gates: expected mode ci-primary | ci-primary-large-runner | ci-static | ci-lint | ci-coverage | ci-snapshot | ci-artifacts | ci-windows-blocking | ci-windows-complete | ci-windows-observational | node-compat | pre-push | doc-sync, got ${JSON.stringify(raw)}.`,
|
`run-gates: expected mode ci-primary | ci-primary-cpu | ci-primary-large-runner | ci-static | ci-lint | ci-coverage | ci-snapshot | ci-artifacts | ci-windows-blocking | ci-windows-complete | ci-windows-observational | node-compat | pre-push | doc-sync, got ${JSON.stringify(raw)}.`,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -173,6 +175,8 @@ function gatesForMode(selected: Mode): Gate[] {
|
|||||||
switch (selected) {
|
switch (selected) {
|
||||||
case 'ci-primary':
|
case 'ci-primary':
|
||||||
return ciPrimaryGates()
|
return ciPrimaryGates()
|
||||||
|
case 'ci-primary-cpu':
|
||||||
|
return ciPrimaryCpuGates()
|
||||||
case 'ci-primary-large-runner':
|
case 'ci-primary-large-runner':
|
||||||
return ciPrimaryLargeRunnerGates()
|
return ciPrimaryLargeRunnerGates()
|
||||||
case 'ci-static':
|
case 'ci-static':
|
||||||
@@ -199,19 +203,7 @@ function gatesForMode(selected: Mode): Gate[] {
|
|||||||
case 'ci-windows-observational':
|
case 'ci-windows-observational':
|
||||||
return ciWindowsObservationalGates()
|
return ciWindowsObservationalGates()
|
||||||
case 'node-compat':
|
case 'node-compat':
|
||||||
return [
|
return nodeCompatGates()
|
||||||
...flagEnabled('DSH_NODE_COMPAT_SKIP_TYPECHECK') ? [] : [pnpmScript('typecheck', 'typecheck')],
|
|
||||||
pnpmExec('source-worker-smoke', [
|
|
||||||
'vitest',
|
|
||||||
'run',
|
|
||||||
'packages/workflow/workflow-workerthread/tests/source-worker.compat.spec.ts',
|
|
||||||
], { label: 'source worker smoke' }),
|
|
||||||
pnpmExec('jsonl-zstd-smoke', [
|
|
||||||
'vitest',
|
|
||||||
'run',
|
|
||||||
'packages/session-persistence/session-persistence-jsonl/tests/zstd.compat.spec.ts',
|
|
||||||
], { label: 'JSONL Zstandard smoke' }),
|
|
||||||
]
|
|
||||||
case 'pre-push':
|
case 'pre-push':
|
||||||
return [
|
return [
|
||||||
pnpmScript('runtime-closure', 'verify-runtime-closure', { label: 'runtime closure' }),
|
pnpmScript('runtime-closure', 'verify-runtime-closure', { label: 'runtime closure' }),
|
||||||
@@ -258,10 +250,10 @@ function ciPrimaryGates(): Gate[] {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function ciPrimaryLargeRunnerGates(): Gate[] {
|
function ciPrimaryLargeRunnerGates(): Gate[] {
|
||||||
// Typecheck does not consume build output, so a large runner can start both
|
// The CPU lane owns typecheck, coverage, and the build-to-snapshot chain.
|
||||||
// together while snapshot and artifact consumers still wait for the build.
|
// This core lane starts its own build eagerly for the remaining artifact consumers.
|
||||||
return ciPrimaryGates()
|
return ciPrimaryGates()
|
||||||
.filter(gate => gate.id !== 'docs-site-build')
|
.filter(gate => !['coverage', 'docs-site-build', 'snapshot', 'typecheck'].includes(gate.id))
|
||||||
.map((gate) => {
|
.map((gate) => {
|
||||||
if (gate.id !== 'build') return gate
|
if (gate.id !== 'build') return gate
|
||||||
const eagerBuild = { ...gate }
|
const eagerBuild = { ...gate }
|
||||||
@@ -270,6 +262,39 @@ function ciPrimaryLargeRunnerGates(): Gate[] {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function ciPrimaryCpuGates(): Gate[] {
|
||||||
|
// Build and snapshot stay together so the dependent replay consumes this lane's output.
|
||||||
|
return [
|
||||||
|
pnpmScript('typecheck', 'typecheck'),
|
||||||
|
coverageGate(),
|
||||||
|
pnpmScript('build', 'build'),
|
||||||
|
snapshotGate(),
|
||||||
|
...nodeCompatSmokeGates(),
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
function nodeCompatGates(): Gate[] {
|
||||||
|
return [
|
||||||
|
...flagEnabled('DSH_NODE_COMPAT_SKIP_TYPECHECK') ? [] : [pnpmScript('typecheck', 'typecheck')],
|
||||||
|
...nodeCompatSmokeGates(),
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
function nodeCompatSmokeGates(): Gate[] {
|
||||||
|
return [
|
||||||
|
pnpmExec('source-worker-smoke', [
|
||||||
|
'vitest',
|
||||||
|
'run',
|
||||||
|
'packages/workflow/workflow-workerthread/tests/source-worker.compat.spec.ts',
|
||||||
|
], { label: 'source worker smoke' }),
|
||||||
|
pnpmExec('jsonl-zstd-smoke', [
|
||||||
|
'vitest',
|
||||||
|
'run',
|
||||||
|
'packages/session-persistence/session-persistence-jsonl/tests/zstd.compat.spec.ts',
|
||||||
|
], { label: 'JSONL Zstandard smoke' }),
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
function ciStaticGates(): Gate[] {
|
function ciStaticGates(): Gate[] {
|
||||||
const gates = [
|
const gates = [
|
||||||
pnpmScript('runtime-closure', 'verify-runtime-closure', { label: 'runtime closure' }),
|
pnpmScript('runtime-closure', 'verify-runtime-closure', { label: 'runtime closure' }),
|
||||||
|
|||||||
Reference in New Issue
Block a user