ci: consolidate primary checks on one larger runner
This commit is contained in:
@@ -7,7 +7,7 @@ Four layers, importable separately:
|
||||
- **`launchAcpTestAgent` (launcher)** — boots a source agent under tsx or a built `lib` agent under plain Node from a temp cwd, connects the SDK client over a raw-byte stdout tee, collects session updates and stderr, surfaces asynchronous spawn failures through startup, fails closed on unhandled permission requests, and owns graceful or signalled shutdown. Shutdown waits for process exit, inherited stdio closure, and ACP parser exhaustion before resolving or propagating a child error, so captures are complete and callers can remove owned paths after either outcome. When Windows accepts forced termination but publishes its exit marker asynchronously, shutdown gives that marker a bounded grace before treating fallback refusal as a second failure. Snapshot and ordinary e2e suites share this process boundary; a test supplies only agent paths, cwd, environment overrides, and any permission policy.
|
||||
- **`runScenario` (harness)** — drives ACP JSON-RPC stdio from a deterministic `input.json` script through the launcher, tees raw stdout for the expected-output and purity checks, and harvests every persisted raw JSONL session log (parent and subagent children, primary-first) after graceful stdin EOF. `AgentUnderTest` supplies absolute `binScript`, optional `libBinScript`, `configPath`, and `tsconfigPath` paths because the subprocess cwd is outside the repo. Startup failures preserve captured agent stderr in the rejected diagnostic.
|
||||
- **Normalizers** — pure functions turning the two captured surfaces into stable text: `normalizeStdout` (JSON-RPC ids → first-seen sequence; UUIDs/cwd → tokens; cwd-rooted separators selected as canonical `/` or host-native; `session_info_update.updatedAt` → `{{updatedAt}}`; doubles as the stdout-purity check), `normalizeSessionLog` (times zeroed, `seq` kept, the same cwd-path policy), `scrubSystemPrompts` (prompt text → `{{system}}`), `scrubToolSchemas` (schema bulk → `{{tools}}`), and `scrubRequestHeaders` (all header bulk → `{{system}}`/`{{tools}}`/`{{messagePrefix}}` outside each pin, structure kept — [pinned-header Agent Note](../../../.agents/notes/implemented/testing/2026-07-06-pin-request-header-content-in-one-scenario.md)).
|
||||
- **`defineAcpSnapshotSuite` (factory)** — registers the whole describe/it tree for a scenario table: per-scenario expected-output and re-persisted-log comparisons, record/refresh fixture write-back, rejection of structured `UNKNOWN_TOOL` results, the per-header-class pin (`system-prompt.expected.md` plus `tool-schemas.expected.json`) with its live uniformity guard, and the fixture guard block (no orphan scenario dirs, required files present, exactly one pin per class, every JSONL prompt/schema-scrubbed, non-pinning fixtures fully header-scrubbed). Replay may partition subprocess-backed scenarios with `scenarioShard`; every lane still runs fixture guards against the complete table, while record and refresh reject sharding because they write fixtures. Refresh preserves existing volatile fields by event position and gives a newly inserted `session/title` its preceding event's time, so feature-driven insertions do not churn the remainder of a fixture. Each scenario directory's `session.jsonl` plus contiguous `session.<n>.jsonl` siblings are the ordered primary/child inventory; the scenario table does not duplicate their count. Must be called at vitest collection time.
|
||||
- **`defineAcpSnapshotSuite` (factory)** — registers the whole describe/it tree for a scenario table: per-scenario expected-output and re-persisted-log comparisons, record/refresh fixture write-back, rejection of structured `UNKNOWN_TOOL` results, the per-header-class pin (`system-prompt.expected.md` plus `tool-schemas.expected.json`) with its live uniformity guard, and the fixture guard block (no orphan scenario dirs, required files present, exactly one pin per class, every JSONL prompt/schema-scrubbed, non-pinning fixtures fully header-scrubbed). Refresh preserves existing volatile fields by event position and gives a newly inserted `session/title` its preceding event's time, so feature-driven insertions do not churn the remainder of a fixture. Each scenario directory's `session.jsonl` plus contiguous `session.<n>.jsonl` siblings are the ordered primary/child inventory; the scenario table does not duplicate their count. Must be called at vitest collection time.
|
||||
|
||||
A consuming `*.snapshot.ts` is the scenario table plus one factory call:
|
||||
|
||||
|
||||
@@ -48,4 +48,3 @@ export {
|
||||
type Scenario,
|
||||
type SnapshotSuiteOptions,
|
||||
} from './suite.ts'
|
||||
export type { SnapshotScenarioShard } from './scenario-shard.ts'
|
||||
|
||||
@@ -1,36 +0,0 @@
|
||||
/** Scenario-level sharding for one ACP snapshot suite. */
|
||||
|
||||
/** A one-based, exhaustive partition of a scenario table. */
|
||||
export interface SnapshotScenarioShard {
|
||||
/** One-based lane index. */
|
||||
index: number
|
||||
/** Total number of lanes. */
|
||||
total: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Select one stable modulo partition while rejecting empty or malformed lanes.
|
||||
*
|
||||
* @param scenarios Complete ordered scenario table.
|
||||
* @param shard Optional one-based shard description.
|
||||
* @returns The complete table or the selected non-empty partition.
|
||||
*/
|
||||
export function selectSnapshotScenarios<T>(
|
||||
scenarios: readonly T[],
|
||||
shard?: SnapshotScenarioShard,
|
||||
): T[] {
|
||||
if (shard === undefined) return [...scenarios]
|
||||
if (!Number.isSafeInteger(shard.index) || shard.index < 1) {
|
||||
throw new Error(`acp-snapshot: shard index must be a positive integer, got ${shard.index}`)
|
||||
}
|
||||
if (!Number.isSafeInteger(shard.total) || shard.total < 1) {
|
||||
throw new Error(`acp-snapshot: shard total must be a positive integer, got ${shard.total}`)
|
||||
}
|
||||
if (shard.index > shard.total) {
|
||||
throw new Error(`acp-snapshot: shard index ${shard.index} exceeds total ${shard.total}`)
|
||||
}
|
||||
if (shard.total > scenarios.length) {
|
||||
throw new Error(`acp-snapshot: ${shard.total} shards exceed ${scenarios.length} scenarios`)
|
||||
}
|
||||
return scenarios.filter((_, offset) => offset % shard.total === shard.index - 1)
|
||||
}
|
||||
@@ -29,7 +29,6 @@ import {
|
||||
scrubSystemPrompts,
|
||||
scrubToolSchemas,
|
||||
} from './normalize.ts'
|
||||
import { selectSnapshotScenarios, type SnapshotScenarioShard } from './scenario-shard.ts'
|
||||
|
||||
/** The readable system-prompt snapshot beside each header-pinning fixture. */
|
||||
const SYSTEM_PROMPT_SNAPSHOT = 'system-prompt.expected.md'
|
||||
@@ -170,11 +169,6 @@ export interface SnapshotSuiteOptions {
|
||||
snapshotsDir: string
|
||||
/** The scenario table; exactly one entry per header class must set `pinsHeader`. */
|
||||
scenarios: Scenario[]
|
||||
/**
|
||||
* Optional replay-only scenario partition. Fixture guards still validate the
|
||||
* complete table in every lane; only subprocess-backed scenario tests split.
|
||||
*/
|
||||
scenarioShard?: SnapshotScenarioShard
|
||||
/**
|
||||
* `replay` (keyless, the default tier), `record` (live API; re-records the
|
||||
* `recorded` scenarios' fixtures and refreshes the Vitest expected outputs under
|
||||
@@ -517,11 +511,7 @@ export function stabilizeRefreshLog(fresh: string, existing: string, replacement
|
||||
* @param options The agent, snapshots directory, scenario table, and mode.
|
||||
*/
|
||||
export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
|
||||
const { agent, snapshotsDir, scenarios, mode, scenarioShard } = options
|
||||
if (scenarioShard !== undefined && mode !== 'replay') {
|
||||
throw new Error('acp-snapshot: scenario sharding is supported only in replay mode')
|
||||
}
|
||||
const selectedScenarios = selectSnapshotScenarios(scenarios, scenarioShard)
|
||||
const { agent, snapshotsDir, scenarios, mode } = options
|
||||
const RECORDING = mode === 'record'
|
||||
const REFRESHING = mode === 'refresh'
|
||||
const childMode: 'replay' | 'record' = RECORDING ? 'record' : 'replay'
|
||||
@@ -546,7 +536,7 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
|
||||
}
|
||||
|
||||
scenarioSuite('snapshot scenarios', () => {
|
||||
for (const scenario of selectedScenarios) {
|
||||
for (const scenario of scenarios) {
|
||||
// In RECORD mode, only re-run the `recorded` (live-API) scenarios; the `authored` ones
|
||||
// (sidecar-driven errors/cancel) are never re-recorded. `posixOnly` scenarios skip on
|
||||
// Windows, where their process semantics cannot be driven.
|
||||
|
||||
@@ -1,25 +0,0 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { selectSnapshotScenarios } from '../src/scenario-shard.ts'
|
||||
|
||||
describe('ACP snapshot scenario shards', () => {
|
||||
it('keeps the ordinary suite complete', () => {
|
||||
expect(selectSnapshotScenarios(['a', 'b', 'c'])).toEqual(['a', 'b', 'c'])
|
||||
})
|
||||
|
||||
it('partitions the ordered table without gaps or overlap', () => {
|
||||
const scenarios = ['a', 'b', 'c', 'd', 'e']
|
||||
expect(selectSnapshotScenarios(scenarios, { index: 1, total: 2 })).toEqual(['a', 'c', 'e'])
|
||||
expect(selectSnapshotScenarios(scenarios, { index: 2, total: 2 })).toEqual(['b', 'd'])
|
||||
})
|
||||
|
||||
it.each([
|
||||
[{ index: 0, total: 1 }, 'index must be a positive integer'],
|
||||
[{ index: 1.5, total: 2 }, 'index must be a positive integer'],
|
||||
[{ index: 1, total: 0 }, 'total must be a positive integer'],
|
||||
[{ index: 1, total: Number.NaN }, 'total must be a positive integer'],
|
||||
[{ index: 3, total: 2 }, 'exceeds total'],
|
||||
[{ index: 1, total: 4 }, 'exceed 3 scenarios'],
|
||||
] as const)('rejects malformed shard %#', (shard, message) => {
|
||||
expect(() => selectSnapshotScenarios(['a', 'b', 'c'], shard)).toThrow(message)
|
||||
})
|
||||
})
|
||||
@@ -107,16 +107,6 @@ describe('defineAcpSnapshotSuite: replay mode', () => {
|
||||
defineAcpSnapshotSuite({ agent: AGENT, snapshotsDir: REPLAY_DIR, scenarios: REPLAY_SCENARIOS, mode: 'replay' })
|
||||
})
|
||||
|
||||
describe('defineAcpSnapshotSuite: sharded replay mode', () => {
|
||||
defineAcpSnapshotSuite({
|
||||
agent: AGENT,
|
||||
snapshotsDir: REPLAY_DIR,
|
||||
scenarios: REPLAY_SCENARIOS,
|
||||
mode: 'replay',
|
||||
scenarioShard: { index: 2, total: 2 },
|
||||
})
|
||||
})
|
||||
|
||||
// The record suite's tests run in registration order: rec-pin re-records the
|
||||
// pinned fixture FIRST, so rec-child's uniformity guard reads the fresh pin.
|
||||
describe('defineAcpSnapshotSuite: record mode', () => {
|
||||
@@ -166,18 +156,6 @@ describe('defineAcpSnapshotSuite: record inventory write-back', () => {
|
||||
})
|
||||
|
||||
describe('defineAcpSnapshotSuite: registration contract', () => {
|
||||
it('rejects scenario sharding in a fixture-writing mode', () => {
|
||||
expect(() => {
|
||||
defineAcpSnapshotSuite({
|
||||
agent: AGENT,
|
||||
snapshotsDir: REPLAY_DIR,
|
||||
scenarios: REPLAY_SCENARIOS,
|
||||
mode: 'refresh',
|
||||
scenarioShard: { index: 1, total: 2 },
|
||||
})
|
||||
}).toThrow('supported only in replay mode')
|
||||
})
|
||||
|
||||
it("throws when a scenario's header class has no pinning scenario", () => {
|
||||
expect(() => {
|
||||
defineAcpSnapshotSuite({
|
||||
|
||||
Reference in New Issue
Block a user