Merge branch 'codex/simp-snapshot-fixture-inventory' into codex/simp-shared-acp-test-launcher

# Conflicts:
#	examples/acp-agent/tests/escalation.e2e.ts
#	packages/support/README.md
#	packages/support/acp-snapshot/README.md
#	packages/support/acp-snapshot/src/harness.ts
#	packages/support/acp-snapshot/tests/harness.spec.ts
This commit is contained in:
Tianyi Cui
2026-07-18 11:58:15 +08:00
657 changed files with 36667 additions and 7474 deletions

View File

@@ -157,9 +157,11 @@ export interface RunOptions {
export async function runScenario(input: InputScript, opts: RunOptions): Promise<RunResult> {
const cwd = await mkdtemp(join(tmpdir(), 'acp-snap-cwd-'))
const sessionsRoot = await mkdtemp(join(tmpdir(), 'acp-snap-sessions-'))
// Everything past the temp-dir creation runs under a try/finally that always
// removes both dirs — so a failure in workspace seeding, spawn, or any step
// never leaks them (the "e2e tests own their resources" rule).
// Fixed path length: spill-policy budgets the preview against the REAL path
// before stdout normalization, so tmpdir() length differences churn goldens.
const spillRoot = '/tmp/dsh-acp-snapshot-spill'
// Everything past the temp-dir creation is followed by failure-safe cleanup,
// so a failure in workspace seeding, spawn, or any step never leaks resources.
let launched: LaunchedAcpTestAgent | undefined
let sessionId: string | undefined
let sessionLogs: HarvestedLog[] = []
@@ -174,6 +176,9 @@ export async function runScenario(input: InputScript, opts: RunOptions): Promise
DSH_SNAPSHOT: opts.mode,
DSH_SNAPSHOT_FILE: opts.fixtureFile,
DSH_SNAPSHOT_SESSIONS_ROOT: sessionsRoot,
DSH_SNAPSHOT_SPILL_ROOT: spillRoot,
DSH_HOME: join(cwd, '.dsh'),
DSH_AGENTS_HOME: join(cwd, '.agents'),
...opts.overrideFile !== undefined ? { DSH_SNAPSHOT_OVERRIDE: opts.overrideFile } : {},
...opts.childFiles !== undefined && opts.childFiles.length > 0
? { DSH_SNAPSHOT_CHILD_FILES: opts.childFiles.join(delimiter) }
@@ -240,11 +245,19 @@ export async function runScenario(input: InputScript, opts: RunOptions): Promise
}
})().then(
value => ({ status: 'fulfilled', value } as const),
(error: unknown) => ({ status: 'rejected', error } as const),
(error: unknown) => {
const stderr = launched?.stderr() ?? ''
return {
status: 'rejected',
error: stderr === ''
? error
: new Error(`snapshot-harness: scenario failed: ${String(error)}\nagent stderr:\n${stderr}`, { cause: error }),
} as const
},
)
// Failure-safe teardown: wait for a still-running child, then attempt BOTH
// directory removals even when an earlier cleanup rejects. Report every
// Failure-safe teardown: wait for a still-running child, then attempt every
// owned-path removal even when an earlier cleanup rejects. Report every
// teardown failure alongside a scenario failure so neither orthogonal
// outcome hides the other.
const cleanupResults: PromiseSettledResult<unknown>[] = []
@@ -255,6 +268,7 @@ export async function runScenario(input: InputScript, opts: RunOptions): Promise
await cleanup(() => launched?.close('SIGKILL') ?? Promise.resolve())
await cleanup(() => rm(cwd, { recursive: true, force: true }))
await cleanup(() => rm(sessionsRoot, { recursive: true, force: true }))
await cleanup(() => rm(spillRoot, { recursive: true, force: true }))
const cleanupFailures = cleanupResults
.filter((result): result is PromiseRejectedResult => result.status === 'rejected')

View File

@@ -14,6 +14,16 @@ const MESSAGE_PREFIX = '{{messagePrefix}}'
/** A UUID v4 string, the shape `randomUUID()` produces for session ids. */
const UUID_RE = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/gi
const LOCAL_SPILL_PATH_RE = new RegExp(
String.raw`\{\{cwd\}\}/\.spill/session-[0-9a-f]{12}/[0-9a-f]{12}-([A-Za-z0-9._~-]+?)`
+ String.raw`(?=\. Use read with offset/limit|[\s)]|$)`,
'g',
)
const SNAPSHOT_SPILL_PATH_RE = new RegExp(
String.raw`/tmp/dsh-acp-snapshot-spill/session-[0-9a-f]{12}/[0-9a-f]{12}-([A-Za-z0-9._~-]+?)`
+ String.raw`(?=\. Use read with offset/limit|[\s)]|$)`,
'g',
)
/** Inputs the normalizers need to recognize a run's volatile values. */
export interface NormalizeContext {
@@ -29,6 +39,9 @@ function scrubString(value: string, ctx: NormalizeContext): string {
// cwd first (longest, most specific), then explicit session ids, then any
// residual UUID (covers ids that appear in places we didn't enumerate).
out = out.split(ctx.cwd).join(CWD)
out = out.split(`/private${CWD}`).join(CWD)
out = out.replace(LOCAL_SPILL_PATH_RE, (_match, name: string) => `{{spillLocator:${name}}}`)
out = out.replace(SNAPSHOT_SPILL_PATH_RE, (_match, name: string) => `{{spillLocator:${name}}}`)
for (const id of ctx.sessionIds) out = out.split(id).join(SESSION_ID)
out = out.replace(UUID_RE, SESSION_ID)
return out

View File

@@ -4,6 +4,9 @@
* both replay input and expected output. Record mode refreshes reproducible
* model scenarios from the live API, while refresh mode replays committed
* scripts and rewrites derived artifacts without a key.
* Replay scenarios run concurrently because each subprocess owns unique temp
* cwd and persistence roots and reads only committed fixtures. Record and
* refresh stay serial while writing.
*
* Exactly one scenario per header-composition class pins the full prompt and
* tool-schema sequences in dedicated sidecars. Every live header is checked
@@ -424,7 +427,7 @@ export function stabilizeRefreshLog(fresh: string, existing: string, replacement
}
/**
* Register the suite: one `describe` per scenario (the golden/log compares and
* Register the suite: one test per scenario (the golden/log compares and
* the header-uniformity guard) plus the fixture guard block (no orphan
* scenario dirs, required files present, exactly one pin per header class,
* pinning fixtures well-formed, every JSONL prompt-scrubbed, non-pinning
@@ -440,6 +443,7 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
const RECORDING = mode === 'record'
const REFRESHING = mode === 'refresh'
const childMode: 'replay' | 'record' = RECORDING ? 'record' : 'replay'
const scenarioSuite = mode === 'replay' ? describe.concurrent : describe
/** The class a scenario's header composition belongs to (see {@link Scenario.headerClass}). */
const classOf = (scenario: Scenario): string => scenario.headerClass ?? 'default'
@@ -459,11 +463,11 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
}
}
for (const scenario of scenarios) {
describe(`snapshot: ${scenario.name}`, () => {
scenarioSuite('snapshot scenarios', () => {
for (const scenario of scenarios) {
// In RECORD mode, only re-run the `recorded` (live-API) scenarios; the `authored` ones
// (sidecar-driven errors/cancel) are never re-recorded.
it.skipIf(RECORDING && !scenario.recorded)('matches the goldens', async () => {
it.skipIf(RECORDING && !scenario.recorded)(`snapshot: ${scenario.name} matches the goldens`, async ({ expect }) => {
const dir = join(snapshotsDir, scenario.name)
const input = JSON.parse(await readFile(join(dir, 'input.json'), 'utf8')) as InputScript
const overrideFile = join(dir, 'replay.override.json')
@@ -642,8 +646,8 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
}
}
})
})
}
}
})
describe('snapshot fixtures', () => {
it('every scenario directory is registered (no orphans)', async () => {