Merge master into codex/simp-session-log-representation
This commit is contained in:
@@ -168,6 +168,9 @@ export interface RunOptions {
|
||||
export async function runScenario(input: InputScript, opts: RunOptions): Promise<RunResult> {
|
||||
const cwd = await mkdtemp(join(tmpdir(), 'acp-snap-cwd-'))
|
||||
const sessionsRoot = await mkdtemp(join(tmpdir(), 'acp-snap-sessions-'))
|
||||
// Fixed path length: spill-policy budgets the preview against the REAL path
|
||||
// before stdout normalization, so tmpdir() length differences churn goldens.
|
||||
const spillRoot = '/tmp/dsh-acp-snapshot-spill'
|
||||
// Everything past the temp-dir creation runs under a try/finally that always
|
||||
// removes both dirs — so a failure in workspace seeding, spawn, or any step
|
||||
// never leaks them (the "e2e tests own their resources" rule).
|
||||
@@ -187,6 +190,7 @@ export async function runScenario(input: InputScript, opts: RunOptions): Promise
|
||||
DSH_SNAPSHOT: opts.mode,
|
||||
DSH_SNAPSHOT_FILE: opts.fixtureFile,
|
||||
DSH_SNAPSHOT_SESSIONS_ROOT: sessionsRoot,
|
||||
DSH_SNAPSHOT_SPILL_ROOT: spillRoot,
|
||||
DSH_HOME: join(cwd, '.dsh'),
|
||||
DSH_AGENTS_HOME: join(cwd, '.agents'),
|
||||
...opts.overrideFile !== undefined ? { DSH_SNAPSHOT_OVERRIDE: opts.overrideFile } : {},
|
||||
@@ -281,6 +285,10 @@ export async function runScenario(input: InputScript, opts: RunOptions): Promise
|
||||
// Harvest EVERY persisted log (parent + any subagent children) while the
|
||||
// temp dirs still exist, ordered primary-first.
|
||||
sessionLogs = await harvestSessionLogs(sessionsRoot)
|
||||
} catch (error: unknown) {
|
||||
const stderr = stderrChunks.join('')
|
||||
if (stderr === '') throw error
|
||||
throw new Error(`snapshot-harness: scenario failed: ${String(error)}\nagent stderr:\n${stderr}`, { cause: error })
|
||||
} finally {
|
||||
// Failure-safe teardown: kill a still-running child and drop the temp dirs
|
||||
// even if seeding/spawn/a step/harvest threw, so a flaky run never leaks a
|
||||
@@ -291,6 +299,7 @@ export async function runScenario(input: InputScript, opts: RunOptions): Promise
|
||||
}
|
||||
await rm(cwd, { recursive: true, force: true })
|
||||
await rm(sessionsRoot, { recursive: true, force: true })
|
||||
await rm(spillRoot, { recursive: true, force: true })
|
||||
}
|
||||
|
||||
return {
|
||||
|
||||
@@ -14,6 +14,16 @@ const MESSAGE_PREFIX = '{{messagePrefix}}'
|
||||
|
||||
/** A UUID v4 string, the shape `randomUUID()` produces for session ids. */
|
||||
const UUID_RE = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/gi
|
||||
const LOCAL_SPILL_PATH_RE = new RegExp(
|
||||
String.raw`\{\{cwd\}\}/\.spill/session-[0-9a-f]{12}/[0-9a-f]{12}-([A-Za-z0-9._~-]+?)`
|
||||
+ String.raw`(?=\. Use read with offset/limit|[\s)]|$)`,
|
||||
'g',
|
||||
)
|
||||
const SNAPSHOT_SPILL_PATH_RE = new RegExp(
|
||||
String.raw`/tmp/dsh-acp-snapshot-spill/session-[0-9a-f]{12}/[0-9a-f]{12}-([A-Za-z0-9._~-]+?)`
|
||||
+ String.raw`(?=\. Use read with offset/limit|[\s)]|$)`,
|
||||
'g',
|
||||
)
|
||||
|
||||
/** Inputs the normalizers need to recognize a run's volatile values. */
|
||||
export interface NormalizeContext {
|
||||
@@ -29,6 +39,9 @@ function scrubString(value: string, ctx: NormalizeContext): string {
|
||||
// cwd first (longest, most specific), then explicit session ids, then any
|
||||
// residual UUID (covers ids that appear in places we didn't enumerate).
|
||||
out = out.split(ctx.cwd).join(CWD)
|
||||
out = out.split(`/private${CWD}`).join(CWD)
|
||||
out = out.replace(LOCAL_SPILL_PATH_RE, (_match, name: string) => `{{spillLocator:${name}}}`)
|
||||
out = out.replace(SNAPSHOT_SPILL_PATH_RE, (_match, name: string) => `{{spillLocator:${name}}}`)
|
||||
for (const id of ctx.sessionIds) out = out.split(id).join(SESSION_ID)
|
||||
out = out.replace(UUID_RE, SESSION_ID)
|
||||
return out
|
||||
|
||||
@@ -4,6 +4,9 @@
|
||||
* both replay input and expected output. Record mode refreshes reproducible
|
||||
* model scenarios from the live API, while refresh mode replays committed
|
||||
* scripts and rewrites derived artifacts without a key.
|
||||
* Replay scenarios run concurrently because each subprocess owns unique temp
|
||||
* cwd and persistence roots and reads only committed fixtures. Record and
|
||||
* refresh stay serial while writing.
|
||||
*
|
||||
* Exactly one scenario per header-composition class pins the full prompt and
|
||||
* tool-schema sequences in dedicated sidecars. Every live header is checked
|
||||
@@ -406,7 +409,7 @@ export function stabilizeRefreshLog(fresh: string, existing: string, replacement
|
||||
}
|
||||
|
||||
/**
|
||||
* Register the suite: one `describe` per scenario (the golden/log compares and
|
||||
* Register the suite: one test per scenario (the golden/log compares and
|
||||
* the header-uniformity guard) plus the fixture guard block (no orphan
|
||||
* scenario dirs, required files present, exactly one pin per header class,
|
||||
* pinning fixtures well-formed, every JSONL prompt-scrubbed, non-pinning
|
||||
@@ -422,6 +425,7 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
|
||||
const RECORDING = mode === 'record'
|
||||
const REFRESHING = mode === 'refresh'
|
||||
const childMode: 'replay' | 'record' = RECORDING ? 'record' : 'replay'
|
||||
const scenarioSuite = mode === 'replay' ? describe.concurrent : describe
|
||||
|
||||
/** The class a scenario's header composition belongs to (see {@link Scenario.headerClass}). */
|
||||
const classOf = (scenario: Scenario): string => scenario.headerClass ?? 'default'
|
||||
@@ -441,11 +445,11 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
|
||||
}
|
||||
}
|
||||
|
||||
for (const scenario of scenarios) {
|
||||
describe(`snapshot: ${scenario.name}`, () => {
|
||||
scenarioSuite('snapshot scenarios', () => {
|
||||
for (const scenario of scenarios) {
|
||||
// In RECORD mode, only re-run the `recorded` (live-API) scenarios; the `authored` ones
|
||||
// (sidecar-driven errors/cancel) are never re-recorded.
|
||||
it.skipIf(RECORDING && !scenario.recorded)('matches the goldens', async () => {
|
||||
it.skipIf(RECORDING && !scenario.recorded)(`snapshot: ${scenario.name} matches the goldens`, async ({ expect }) => {
|
||||
const dir = join(snapshotsDir, scenario.name)
|
||||
const input = JSON.parse(await readFile(join(dir, 'input.json'), 'utf8')) as InputScript
|
||||
const overrideFile = join(dir, 'replay.override.json')
|
||||
@@ -601,8 +605,8 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
|
||||
}
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
describe('snapshot fixtures', () => {
|
||||
it('every scenario directory is registered (no orphans)', async () => {
|
||||
|
||||
Reference in New Issue
Block a user