Coder/deepseek-harness
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

test: rename golden snapshot files to expected

Browse Source
This commit is contained in:
Tianyi Cui
2026-07-19 17:45:49 +08:00
parent b3c4b732d8
commit 0893aaa7cb
157 changed files with 160 additions and 159 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
packages/support/acp-snapshot/src/harness.ts
View File

@@ -6,7 +6,7 @@
* It boots the REAL agent bin subprocess via the cordis Loader (so the
* export-shape bug class stays guarded — see docs/postmortem/0001), drives it
* over real ACP JSON-RPC stdio with a deterministic input script, tees raw
* stdout (for the golden + a purity check) into an SDK `ClientSideConnection`,
* stdout (for the expected-output and purity checks) into an SDK `ClientSideConnection`,
* and — in record mode — harvests the persisted session JSONL after a graceful
* shutdown flush. The pure normalizers in ./normalize.ts turn the captured
* stdout frames and the session-log events into stable, snapshot-able text.
@@ -162,7 +162,7 @@ export async function runScenario(input: InputScript, opts: RunOptions): Promise
const cwd = await mkdtemp(join(tmpdir(), 'acp-snap-cwd-'))
const sessionsRoot = await mkdtemp(join(tmpdir(), 'acp-snap-sessions-'))
// Fixed path length: spill-policy budgets the preview against the REAL path
// before stdout normalization, so tmpdir() length differences churn goldens.
// before stdout normalization, so tmpdir() length differences churn expected outputs.
const spillRoot = '/tmp/dsh-acp-snapshot-spill'
// Everything past the temp-dir creation is followed by failure-safe cleanup,
// so a failure in workspace seeding, spawn, or any step never leaks resources.
@@ -171,7 +171,7 @@ export async function runScenario(input: InputScript, opts: RunOptions): Promise
let sessionLogs: HarvestedLog[] = []
const outcome = await (async (): Promise<RunResult> => {
// Seed the workspace if the scenario ships one (a file the agent reads/edits).
// Copied into the temp cwd so the agent's bash tools see it; the goldens
// Copied into the temp cwd so the agent's bash tools see it; the expected outputs
// normalize the cwd, so the seeded paths stay stable across runs.
if (opts.workspaceDir !== undefined && existsSync(opts.workspaceDir)) {
await cp(opts.workspaceDir, cwd, { recursive: true })

2
packages/support/acp-snapshot/src/index.ts
View File

@@ -2,7 +2,7 @@
* ACP snapshot suite kit — the shared machinery behind the keyless snapshot
* tier (`pnpm run test:snapshot`). Four layers, composable per example: the
* shared subprocess/client launcher ({@link launchAcpTestAgent}), the scripted
* scenario harness ({@link runScenario}), the pure golden normalizers
* scenario harness ({@link runScenario}), the pure expected-output normalizers
* ({@link normalizeStdout} / {@link normalizeSessionLog} /
* {@link scrubRequestHeaders} / {@link scrubSystemPrompts}), and the suite
* factory ({@link defineAcpSnapshotSuite}) that registers a scenario table as a

8
packages/support/acp-snapshot/src/normalize.ts
View File

@@ -60,7 +60,7 @@ function scrubValue(value: unknown, ctx: NormalizeContext): unknown {
}
/**
* Normalize a raw stdout transcript (newline-delimited JSON-RPC frames) into a stable golden
* Normalize a raw stdout transcript (newline-delimited JSON-RPC frames) into a stable expected output
* in the same shape as the wire: one compact JSON frame per line (NDJSON), with the JSON-RPC
* `id` rewritten to a per-transcript sequence (1, 2, 3, …) and all volatile strings scrubbed.
* Invalid JSON throws, doubling as a protocol-stdout purity check.
@@ -72,7 +72,7 @@ function scrubValue(value: unknown, ctx: NormalizeContext): unknown {
export function normalizeStdout(rawStdout: string, ctx: NormalizeContext): string {
const lines = rawStdout.split('\n').filter(line => line.trim().length > 0)
// Map each distinct JSON-RPC id (request/response correlate by id) to a stable
// sequence number, in first-seen order, so id churn doesn't perturb the golden.
// sequence number, in first-seen order, so id churn doesn't perturb the expected output.
const idSeq = new Map<string, number>()
const stableId = (id: unknown): number => {
const key = JSON.stringify(id)
@@ -91,7 +91,7 @@ export function normalizeStdout(rawStdout: string, ctx: NormalizeContext): strin
}
/**
* Normalize a session JSONL log into a stable golden: the header line's
* Normalize a session JSONL log into a stable expected output: the header line's
* volatile fields (`createdAt`, `id`, `cwd`) and every event's `time` are
* zeroed/scrubbed, all volatile strings scrubbed, and `seq` is LEFT INTACT
* (deterministic by contract). Output is JSONL in the same shape as the input —
@@ -112,7 +112,7 @@ export function normalizeSessionLog(rawLog: string, ctx: NormalizeContext): stri
// Event line: zero the epoch-ms timestamp; keep seq (deterministic).
record.time = 0
// A hook/result carries the hook's wall-clock runtime (`data.durationMs`),
// which is run-to-run noise like `time` — zero it so the golden reflects
// which is run-to-run noise like `time` — zero it so the expected output reflects
// the hook's decision/exit, not how long the shell took.
if (record.type === 'hook/result' && record.data !== null && typeof record.data === 'object') {
const data = record.data as Record<string, unknown>

22
packages/support/acp-snapshot/src/suite.ts
View File

@@ -30,10 +30,10 @@ import {
} from './normalize.ts'
/** The readable system-prompt snapshot beside each header-pinning fixture. */
const SYSTEM_PROMPT_SNAPSHOT = 'system-prompt.golden.md'
const SYSTEM_PROMPT_SNAPSHOT = 'system-prompt.expected.md'
/** The structured tool-schema snapshot beside each header-pinning fixture. */
const TOOL_SCHEMAS_SNAPSHOT = 'tool-schemas.golden.json'
const TOOL_SCHEMAS_SNAPSHOT = 'tool-schemas.expected.json'
/** Stable session-log token standing in for the sidecar's initial schemas. */
const TOOLS_TOKEN = '{{tools}}'
@@ -41,7 +41,7 @@ const TOOLS_TOKEN = '{{tools}}'
/** A snapshot scenario and how its fixtures are produced. */
export interface Scenario {
name: string
/** Whether the scenario drives at least one model turn (so a JSONL golden applies). */
/** Whether the scenario drives at least one model turn (so a JSONL expected output applies). */
hasModelTurn: boolean
/**
* Whether the run persists a comparable session log to diff against the
@@ -112,8 +112,8 @@ export interface SnapshotSuiteOptions {
scenarios: Scenario[]
/**
* `replay` (keyless, the default tier), `record` (live API; re-records the
* `recorded` scenarios' fixtures and refreshes the Vitest goldens under
* `--update`), or `refresh` (keyless replay that rewrites stdout goldens and
* `recorded` scenarios' fixtures and refreshes the Vitest expected outputs under
* `--update`), or `refresh` (keyless replay that rewrites stdout expected outputs and
* comparable session fixtures from the replay run). The caller derives this
* from `$DSH_SNAPSHOT` — env reading stays outside this library.
*/
@@ -427,7 +427,7 @@ export function stabilizeRefreshLog(fresh: string, existing: string, replacement
}
/**
* Register the suite: one test per scenario (the golden/log compares and
* Register the suite: one test per scenario (the expected-output and log comparisons and
* the header-uniformity guard) plus the fixture guard block (no orphan
* scenario dirs, required files present, exactly one pin per header class,
* pinning fixtures well-formed, every JSONL prompt-scrubbed, non-pinning
@@ -467,7 +467,7 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
for (const scenario of scenarios) {
// In RECORD mode, only re-run the `recorded` (live-API) scenarios; the `authored` ones
// (sidecar-driven errors/cancel) are never re-recorded.
it.skipIf(RECORDING && !scenario.recorded)(`snapshot: ${scenario.name} matches the goldens`, async ({ expect }) => {
it.skipIf(RECORDING && !scenario.recorded)(`snapshot: ${scenario.name} matches the expected outputs`, async ({ expect }) => {
const dir = join(snapshotsDir, scenario.name)
const input = JSON.parse(await readFile(join(dir, 'input.json'), 'utf8')) as InputScript
const overrideFile = join(dir, 'replay.override.json')
@@ -573,9 +573,9 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
const stdout = normalizeStdout(result.rawStdout, ctx)
if (REFRESHING) {
await writeFile(join(dir, 'stdout.golden.jsonl'), stdout)
await writeFile(join(dir, 'stdout.expected.jsonl'), stdout)
}
await expect(stdout).toMatchFileSnapshot(join(dir, 'stdout.golden.jsonl'))
await expect(stdout).toMatchFileSnapshot(join(dir, 'stdout.expected.jsonl'))
// A model turn always produces a log worth comparing; a hook scenario can
// produce one without a model turn (a `rejected` turn carrying `hook/*`).
@@ -651,7 +651,7 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
describe('snapshot fixtures', () => {
it('every scenario directory is registered (no orphans)', async () => {
// toMatchFileSnapshot does not prune orphaned golden/fixture files, so a
// toMatchFileSnapshot does not prune orphaned expected-output or fixture files, so a
// renamed/removed scenario could leave a stale dir that nothing exercises.
// Fail loud on any snapshots/<dir> not present in the scenario table.
const entries = await readdir(snapshotsDir, { withFileTypes: true })
@@ -665,7 +665,7 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
for (const { name, overridden, pinsHeader } of scenarios) {
const dir = join(snapshotsDir, name)
expect(existsSync(join(dir, 'input.json')), `${name}/input.json`).toBe(true)
expect(existsSync(join(dir, 'stdout.golden.jsonl')), `${name}/stdout.golden.jsonl`).toBe(true)
expect(existsSync(join(dir, 'stdout.expected.jsonl')), `${name}/stdout.expected.jsonl`).toBe(true)
expect(existsSync(join(dir, 'session.jsonl')), `${name}/session.jsonl`).toBe(true)
expect(existsSync(join(dir, 'replay.override.json')), `${name}/replay.override.json presence must match \`overridden\``)
.toBe(overridden === true)