diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 719e2f8e9b..0e1135f447 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,6 +15,7 @@ on: - serial-reference - larger-runner-benchmark - consolidated-runner-benchmark + - sharded-reference - optimized-larger-runners concurrency: @@ -28,8 +29,8 @@ env: PRIMARY_NODE_VERSION: '24' jobs: - node-24: - if: github.event_name != 'workflow_dispatch' || inputs.suite == 'optimized-larger-runners' + node-24-sharded: + if: github.event_name == 'workflow_dispatch' && inputs.suite == 'sharded-reference' runs-on: ${{ startsWith(matrix.lane, 'snapshot-') && 'dsh-ubuntu-24-04-8core' || 'dsh-ubuntu-24-04-4core' }} name: node 24 / ${{ matrix.lane }} env: @@ -317,6 +318,75 @@ jobs: - name: Run gates run: ${{ matrix.command }} + # One large runner pays hosted setup once, then the repository scheduler + # overlaps the complete unsharded primary gate inventory inside that box. + node-24: + if: github.event_name != 'workflow_dispatch' || inputs.suite == 'optimized-larger-runners' + runs-on: dsh-ubuntu-24-04-96core + name: node 24 / complete + env: + DSH_COVERAGE_MAX_WORKERS: '32' + DSH_ESLINT_CACHE: '1' + DSH_ESLINT_CONCURRENCY: '32' + DSH_GATE_CONCURRENCY: '32' + DSH_PUBLINT_CONCURRENCY: '32' + DSH_SNAPSHOT_MAX_CONCURRENCY: '32' + steps: + - uses: actions/checkout@v6 + + - uses: actions/setup-node@v6 + with: + node-version: ${{ env.PRIMARY_NODE_VERSION }} + + - name: Enable corepack and resolve pnpm store path + id: pnpm-store + run: | + corepack enable + echo "path=$(pnpm store path --silent)" >> "$GITHUB_OUTPUT" + + - uses: actions/cache@v4 + with: + path: ${{ steps.pnpm-store.outputs.path }} + key: ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-pnpm-${{ hashFiles('pnpm-lock.yaml') }} + restore-keys: | + ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-pnpm- + + - uses: actions/cache@v4 + with: + path: .cache/eslint + key: ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-eslint-full-${{ hashFiles('pnpm-lock.yaml', 'eslint.config.mjs', 'tsconfig.json', 'packages/*/*/tsconfig.json', 'examples/*/tsconfig.json') }} + restore-keys: | + ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-eslint-full- + + - name: Install, build, and prepare bubblewrap + run: | + pnpm install --frozen-lockfile & + install_pid=$! + ( + if ! sudo apt-get install -yq --no-install-recommends bubblewrap; then + echo "initial bubblewrap install failed; refreshing APT indexes and retrying" + sudo apt-get update -q + sudo apt-get install -yq --no-install-recommends bubblewrap + fi + sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 \ + || echo "apparmor userns knob absent — the functional probe decides" + ) & + sandbox_pid=$! + install_status=0 + wait "$install_pid" || install_status=$? + build_status=0 + if (( install_status == 0 )); then + pnpm run build || build_status=$? + fi + sandbox_status=0 + wait "$sandbox_pid" || sandbox_status=$? + if (( install_status != 0 )); then exit "$install_status"; fi + if (( build_status != 0 )); then exit "$build_status"; fi + exit "$sandbox_status" + + - name: Run complete unsharded primary Node CI concurrently + run: pnpm run check:ci:prebuilt + node-compat: if: github.event_name != 'workflow_dispatch' || inputs.suite == 'optimized-larger-runners' runs-on: ubuntu-latest @@ -378,12 +448,13 @@ jobs: - name: Run complete keyless Python suite run: uv run --python 3.10 --group test --project python/sdk pytest - # Blocking Windows build and production-site lanes keep the already-green + # Manual sharded Windows reference matching the former production topology. + # Blocking build and production-site lanes keep the already-green # native outputs protected without putting both critical paths in one job. # The broader observational gate matrix below exposes the remaining # portability work without blocking mainline merges. - windows-build: - if: github.event_name != 'workflow_dispatch' || inputs.suite == 'optimized-larger-runners' + windows-build-sharded: + if: github.event_name == 'workflow_dispatch' && inputs.suite == 'sharded-reference' runs-on: dsh-windows-2025-4core name: windows / build steps: @@ -401,8 +472,8 @@ jobs: - name: Build (tsc -b + tsdown) run: pnpm run build - windows-site: - if: github.event_name != 'workflow_dispatch' || inputs.suite == 'optimized-larger-runners' + windows-site-sharded: + if: github.event_name == 'workflow_dispatch' && inputs.suite == 'sharded-reference' runs-on: dsh-windows-2025-4core name: windows / production site steps: @@ -425,8 +496,8 @@ jobs: # have dedicated support. Run the gates from native PowerShell: an MSYS parent # would change the environment being measured. This job intentionally stays # out of all-checks-passed.needs. - windows-gates: - if: github.event_name != 'workflow_dispatch' || inputs.suite == 'optimized-larger-runners' + windows-gates-sharded: + if: github.event_name == 'workflow_dispatch' && inputs.suite == 'sharded-reference' continue-on-error: true runs-on: dsh-windows-2025-4core name: windows node 24 / ${{ matrix.lane }} @@ -516,7 +587,97 @@ jobs: shell: pwsh run: ${{ matrix.command }} - # Manual reference runs deliberately avoid the optimized matrices above. + # The two blocking Windows build surfaces share one setup and run concurrently. + windows-build: + if: github.event_name != 'workflow_dispatch' || inputs.suite == 'optimized-larger-runners' + runs-on: dsh-windows-2025-16core + name: windows / blocking builds + steps: + - uses: actions/checkout@v6 + + - uses: actions/setup-node@v6 + with: + node-version: ${{ env.PRIMARY_NODE_VERSION }} + + - name: Enable corepack and resolve pnpm store path + id: pnpm-store + shell: pwsh + run: | + corepack enable + "path=$(pnpm store path --silent)" >> $env:GITHUB_OUTPUT + + - uses: actions/cache@v4 + with: + path: ${{ steps.pnpm-store.outputs.path }} + key: ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-pnpm-${{ hashFiles('pnpm-lock.yaml') }} + restore-keys: | + ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-pnpm- + + - name: Install (immutable) + shell: pwsh + run: pnpm install --frozen-lockfile + + - name: Run blocking Windows builds concurrently + shell: pwsh + env: + DSH_GATE_CONCURRENCY: '2' + run: pnpm run check:ci:windows-blocking + + # Wider Windows portability gates remain observational, but one larger box + # now runs their complete unsharded inventory with in-runner concurrency. + windows-gates: + if: github.event_name != 'workflow_dispatch' || inputs.suite == 'optimized-larger-runners' + continue-on-error: true + runs-on: dsh-windows-2025-16core + name: windows node 24 / observational + env: + DSH_ESLINT_CACHE: '1' + DSH_ESLINT_CONCURRENCY: '16' + DSH_GATE_CONCURRENCY: '16' + DSH_PUBLINT_CONCURRENCY: '16' + steps: + - uses: actions/checkout@v6 + + - name: Enable Developer Mode (symlink support) + shell: pwsh + run: >- + reg add "HKLM\SOFTWARE\Microsoft\Windows\CurrentVersion\AppModelUnlock" + /t REG_DWORD /f /v "AllowDevelopmentWithoutDevLicense" /d "1" + + - uses: actions/setup-node@v6 + with: + node-version: ${{ env.PRIMARY_NODE_VERSION }} + + - name: Enable corepack and resolve pnpm store path + id: pnpm-store + shell: pwsh + run: | + corepack enable + "path=$(pnpm store path --silent)" >> $env:GITHUB_OUTPUT + + - uses: actions/cache@v4 + with: + path: ${{ steps.pnpm-store.outputs.path }} + key: ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-pnpm-${{ hashFiles('pnpm-lock.yaml') }} + restore-keys: | + ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-pnpm- + + - uses: actions/cache@v4 + with: + path: .cache/eslint + key: ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-eslint-full-${{ hashFiles('pnpm-lock.yaml', 'eslint.config.mjs', 'tsconfig.json', 'packages/*/*/tsconfig.json', 'examples/*/tsconfig.json') }} + restore-keys: | + ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-eslint-full- + + - name: Install (immutable) + shell: pwsh + run: pnpm install --frozen-lockfile + + - name: Run observational Windows gates concurrently + shell: pwsh + run: pnpm run check:ci:windows-observational + + # Manual reference runs deliberately avoid the optimized jobs above. # Each host executes the complete, unsharded primary Node aggregate with one # gate worker, giving reviewers a simple cross-platform oracle for completeness # and timing. @@ -834,9 +995,14 @@ jobs: sandbox_pid=$! install_status=0 wait "$install_pid" || install_status=$? + build_status=0 + if (( install_status == 0 )); then + pnpm run build || build_status=$? + fi sandbox_status=0 wait "$sandbox_pid" || sandbox_status=$? if (( install_status != 0 )); then exit "$install_status"; fi + if (( build_status != 0 )); then exit "$build_status"; fi exit "$sandbox_status" - name: Install (immutable) @@ -853,7 +1019,7 @@ jobs: DSH_GATE_CONCURRENCY: ${{ matrix.workers }} DSH_PUBLINT_CONCURRENCY: ${{ matrix.workers }} DSH_SNAPSHOT_MAX_CONCURRENCY: ${{ matrix.workers }} - run: pnpm run check:ci + run: pnpm run check:ci:prebuilt - name: Run blocking Windows builds concurrently if: matrix.platform == 'windows' @@ -874,7 +1040,7 @@ jobs: all-checks-passed: name: all checks passed runs-on: ubuntu-latest - needs: [node-24, node-compat, python-sdk, windows-build, windows-site] + needs: [node-24, node-compat, python-sdk, windows-build] if: always() && (github.event_name != 'workflow_dispatch' || inputs.suite == 'optimized-larger-runners') steps: - name: Fail if any needed job did not succeed diff --git a/package.json b/package.json index b758974ef7..a8ee301da3 100644 --- a/package.json +++ b/package.json @@ -26,6 +26,7 @@ "test:snapshot:record": "DSH_SNAPSHOT=record vitest run --config vitest.snapshot.config.ts --update", "test:snapshot:refresh": "DSH_SNAPSHOT=refresh vitest run --config vitest.snapshot.config.ts", "check:ci": "tsx scripts/run-gates.ts ci-primary", + "check:ci:prebuilt": "tsx scripts/run-gates.ts ci-primary-prebuilt", "check:ci:static": "tsx scripts/run-gates.ts ci-static", "check:ci:lint": "tsx scripts/run-gates.ts ci-lint", "check:ci:coverage": "tsx scripts/run-gates.ts ci-coverage", diff --git a/scripts/run-gates.ts b/scripts/run-gates.ts index 98326d6db9..c9d058518b 100644 --- a/scripts/run-gates.ts +++ b/scripts/run-gates.ts @@ -15,6 +15,7 @@ import { selectStaticGates } from './static-shards.ts' type Mode = | 'ci-primary' + | 'ci-primary-prebuilt' | 'ci-static' | 'ci-lint' | 'ci-coverage' @@ -87,6 +88,7 @@ if (results.some(result => result.status === 'failed' || result.status === 'skip function parseMode(raw: string | undefined): Mode { switch (raw) { case 'ci-primary': + case 'ci-primary-prebuilt': case 'ci-static': case 'ci-lint': case 'ci-coverage': @@ -100,7 +102,7 @@ function parseMode(raw: string | undefined): Mode { return raw default: throw new Error( - `run-gates: expected mode ci-primary | ci-static | ci-lint | ci-coverage | ci-snapshot | ci-artifacts | ci-windows-blocking | ci-windows-observational | node-compat | pre-push | doc-sync, got ${JSON.stringify(raw)}.`, + `run-gates: expected mode ci-primary | ci-primary-prebuilt | ci-static | ci-lint | ci-coverage | ci-snapshot | ci-artifacts | ci-windows-blocking | ci-windows-observational | node-compat | pre-push | doc-sync, got ${JSON.stringify(raw)}.`, ) } } @@ -166,6 +168,8 @@ function gatesForMode(selected: Mode): Gate[] { switch (selected) { case 'ci-primary': return ciPrimaryGates() + case 'ci-primary-prebuilt': + return ciPrimaryPrebuiltGates() case 'ci-static': return ciStaticGates() case 'ci-lint': { @@ -246,6 +250,14 @@ function ciPrimaryGates(): Gate[] { ] } +function ciPrimaryPrebuiltGates(): Gate[] { + return ciPrimaryGates() + .filter(gate => gate.id !== 'build') + .map(gate => gate.needs?.includes('build') === true + ? { ...gate, needs: gate.needs.filter(id => id !== 'build') } + : gate) +} + function ciStaticGates(): Gate[] { const gates = [ pnpmScript('runtime-closure', 'verify-runtime-closure', { label: 'runtime closure' }),