Merge branch 'master' into codex/skill-system

packages/support/acp-snapshot/src/suite.ts

@@ -23,8 +23,11 @@
  *
  * `pnpm run test:snapshot:record` (DSH_SNAPSHOT=record + -u) re-records the
  * `session.jsonl` fixtures against the real API and refreshes the stdout golden
- * in one pass; the caller resolves that env into {@link SnapshotSuiteOptions}
- * (env reading stays at the suite edge, not in this library).
+ * in one pass. `pnpm run test:snapshot:refresh` (DSH_SNAPSHOT=refresh) instead
+ * replays the committed model scripts keylessly and writes the current stdout
+ * + persisted-log goldens back without calling a live LLM. The caller resolves
+ * that env into {@link SnapshotSuiteOptions} (env reading stays at the suite
+ * edge, not in this library).
  *
  * @module @deepseek-ai/dsh-acp-snapshot/suite
  */
@@ -124,12 +127,13 @@ export interface SnapshotSuiteOptions {
   /** The scenario table; exactly one entry must set `pinsHeader`. */
   scenarios: Scenario[]
   /**
-   * `replay` (keyless, the default tier) or `record` (live API; re-records the
-   * `recorded` scenarios' fixtures and refreshes the vitest goldens under
-   * `--update`). The caller derives this from `$DSH_SNAPSHOT` — env reading
-   * stays outside this library.
+   * `replay` (keyless, the default tier), `record` (live API; re-records the
+   * `recorded` scenarios' fixtures and refreshes the Vitest goldens under
+   * `--update`), or `refresh` (keyless replay that rewrites stdout goldens and
+   * comparable session fixtures from the replay run). The caller derives this
+   * from `$DSH_SNAPSHOT` — env reading stays outside this library.
    */
-  mode: 'replay' | 'record'
+  mode: 'replay' | 'record' | 'refresh'
 }
 
 /**
@@ -201,6 +205,86 @@ export function headerDeltaCount(rawLog: string): number {
     .length
 }
 
+/** A literal string replacement used to carry an existing fixture's volatile value into a refreshed log. */
+export interface FixtureReplacement {
+  /** The fresh replay-run value to replace. */
+  from: string
+  /** The existing fixture value to keep. */
+  to: string
+}
+
+function parseJsonlRecords(text: string): Record<string, unknown>[] {
+  return text.split('\n')
+    .filter(line => line.trim().length > 0)
+    .map(line => JSON.parse(line) as Record<string, unknown>)
+}
+
+/**
+ * Build the cross-log id/cwd replacements used by refresh write-back.
+ *
+ * @param logs The freshly harvested logs, in fixture order.
+ * @param fixtures The existing fixture contents, in matching order.
+ * @returns Literal replacements from fresh volatile values to the fixture's old values.
+ */
+export function refreshFixtureReplacements(logs: HarvestedLog[], fixtures: string[]): FixtureReplacement[] {
+  const replacements: FixtureReplacement[] = []
+  for (let i = 0; i < logs.length; i++) {
+    const fresh = parseJsonlRecords((logs[i] as HarvestedLog).content)[0]
+    const existing = parseJsonlRecords(fixtures[i] ?? '')[0]
+    for (const field of ['id', 'cwd'] as const) {
+      const from = fresh?.[field]
+      const to = existing?.[field]
+      if (typeof from === 'string' && typeof to === 'string' && from.length > 0 && from !== to) {
+        replacements.push({ from, to })
+      }
+    }
+  }
+  return replacements
+}
+
+function preserveFixtureVolatiles(record: Record<string, unknown>, existing: Record<string, unknown> | undefined): void {
+  if (existing === undefined || existing.type !== record.type) return
+  if (record.type === 'session') {
+    for (const field of ['id', 'createdAt', 'cwd', 'parentSession'] as const) {
+      if (field in record && field in existing) record[field] = existing[field]
+    }
+    return
+  }
+  if ('time' in record && 'time' in existing) record.time = existing.time
+  if (record.type !== 'hook/result') return
+  const data = record.data
+  const existingData = existing.data
+  if (
+    data !== null && typeof data === 'object'
+    && existingData !== null && typeof existingData === 'object'
+    && 'durationMs' in data && 'durationMs' in existingData
+  ) {
+    (data as Record<string, unknown>).durationMs = (existingData as Record<string, unknown>).durationMs
+  }
+}
+
+/**
+ * Rewrite a fresh replay-produced log so repeated refreshes do not churn
+ * volatile fixture fields. Meaningful event payloads come from `fresh`; the
+ * existing fixture lends session ids, cwd, creation times, event times, and
+ * hook durations where the record shape still matches.
+ *
+ * @param fresh The newly harvested session JSONL.
+ * @param existing The committed fixture JSONL being refreshed.
+ * @param replacements Cross-log literal replacements from {@link refreshFixtureReplacements}.
+ * @returns The stabilized JSONL content to write back.
+ */
+export function stabilizeRefreshLog(fresh: string, existing: string, replacements: FixtureReplacement[]): string {
+  let stable = fresh
+  for (const { from, to } of replacements) stable = stable.split(from).join(to)
+  const existingRecords = parseJsonlRecords(existing)
+  const records = parseJsonlRecords(stable)
+  for (let i = 0; i < records.length; i++) {
+    preserveFixtureVolatiles(records[i] as Record<string, unknown>, existingRecords[i])
+  }
+  return records.map(record => JSON.stringify(record)).join('\n') + '\n'
+}
+
 /**
  * Register the suite: one `describe` per scenario (the golden/log compares and
  * the header-uniformity guard) plus the fixture guard block (no orphan
@@ -215,6 +299,8 @@ export function headerDeltaCount(rawLog: string): number {
 export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
   const { agent, snapshotsDir, scenarios, mode } = options
   const RECORDING = mode === 'record'
+  const REFRESHING = mode === 'refresh'
+  const childMode: 'replay' | 'record' = RECORDING ? 'record' : 'replay'
 
   /** The class a scenario's header composition belongs to (see {@link Scenario.headerClass}). */
   const classOf = (scenario: Scenario): string => scenario.headerClass ?? 'default'
@@ -238,15 +324,18 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
     describe(`snapshot: ${scenario.name}`, () => {
       // In RECORD mode, only re-run the `recorded` (live-API) scenarios; the
       // `authored` ones (sidecar-driven errors/cancel) are never re-recorded.
+      // REFRESH mode is replay-backed and deterministic, so it runs every
+      // scenario and rewrites the comparable fixtures from that replay run.
       it.skipIf(RECORDING && !scenario.recorded)('matches the goldens', async () => {
         const dir = join(snapshotsDir, scenario.name)
         const input = JSON.parse(await readFile(join(dir, 'input.json'), 'utf8')) as InputScript
         const overrideFile = join(dir, 'replay.override.json')
         const workspaceDir = join(dir, 'workspace')
         const childSessions = scenario.childSessions ?? 0
+        const comparesLog = scenario.comparesLog ?? scenario.hasModelTurn
         const result = await runScenario(input, {
           agent,
-          mode,
+          mode: childMode,
           fixtureFile: join(dir, 'session.jsonl'),
           ...existsSync(overrideFile) ? { overrideFile } : {},
           // In REPLAY, forward the recorded child fixtures so each subagent session
@@ -271,30 +360,47 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
         }
 
         // RECORD mode (recorded model scenarios only): persist the freshly-harvested
-        // logs back to their fixtures — the primary to session.jsonl, each child to
-        // session.<n>.jsonl in harvest order. `--update` refreshes the Vitest
-        // goldens but NOT these fixtures, so write them here. A non-pinning
-        // scenario's fixtures are written header-scrubbed, so a re-record can
-        // never smuggle the full prompt/schema content back into every fixture.
+        // live logs back to their fixtures. REFRESH mode does the same from a
+        // keyless replay run for every comparable log, including authored
+        // scenarios that live record deliberately skips. The primary goes to
+        // session.jsonl, each child to session.<n>.jsonl in harvest order. A
+        // non-pinning scenario's fixtures are written header-scrubbed, so a
+        // re-record/refresh can never smuggle the full prompt/schema content
+        // back into every fixture.
         const scrub = scenario.pinsHeader === true
           ? (log: string): string => log
           : scrubRequestHeaders
-        if (RECORDING && scenario.recorded && scenario.hasModelTurn) {
-          expect(result.sessionLogs.length, 'record produced no session log to harvest').toBeGreaterThan(0)
+        const fixtureFiles = ['session.jsonl', ...Array.from({ length: childSessions }, (_, i) => `session.${i + 1}.jsonl`)]
+        const existingFixtures = REFRESHING
+          ? await Promise.all(fixtureFiles.map(file => readFile(join(dir, file), 'utf8')))
+          : []
+        const replacements = REFRESHING ? refreshFixtureReplacements(result.sessionLogs, existingFixtures) : []
+        const writesSessionFixtures = (RECORDING && scenario.recorded && scenario.hasModelTurn)
+          || (REFRESHING && comparesLog)
+        if (writesSessionFixtures) {
+          expect(result.sessionLogs.length, `${mode} produced no session log to harvest`).toBeGreaterThan(0)
           expect(result.sessionLogs.length, `expected ${childSessions + 1} session logs (parent + children)`)
             .toBe(childSessions + 1)
-          await writeFile(join(dir, 'session.jsonl'), scrub((result.sessionLogs[0] as HarvestedLog).content))
+          const primary = (result.sessionLogs[0] as HarvestedLog).content
+          await writeFile(join(dir, 'session.jsonl'), scrub(
+            REFRESHING ? stabilizeRefreshLog(primary, existingFixtures[0] as string, replacements) : primary,
+          ))
           for (let i = 1; i < result.sessionLogs.length; i++) {
-            await writeFile(join(dir, `session.${i}.jsonl`), scrub((result.sessionLogs[i] as HarvestedLog).content))
+            const child = (result.sessionLogs[i] as HarvestedLog).content
+            await writeFile(join(dir, `session.${i}.jsonl`), scrub(
+              REFRESHING ? stabilizeRefreshLog(child, existingFixtures[i] as string, replacements) : child,
+            ))
           }
         }
 
-        await expect(normalizeStdout(result.rawStdout, ctx))
-          .toMatchFileSnapshot(join(dir, 'stdout.golden.jsonl'))
+        const stdout = normalizeStdout(result.rawStdout, ctx)
+        if (REFRESHING) {
+          await writeFile(join(dir, 'stdout.golden.jsonl'), stdout)
+        }
+        await expect(stdout).toMatchFileSnapshot(join(dir, 'stdout.golden.jsonl'))
 
         // A model turn always produces a log worth comparing; a hook scenario can
         // produce one without a model turn (a `rejected` turn carrying `hook/*`).
-        const comparesLog = scenario.comparesLog ?? scenario.hasModelTurn
         if (comparesLog) {
           // The harvested logs (primary-first) must match their committed fixtures
           // 1:1. Each side passes through normalizeSessionLog, scrubbed against ITS
@@ -307,7 +413,6 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
           // reason, and config, but not its bulk content (pinned once, in the
           // `pinsHeader` scenario).
           expect(result.sessionLogs.length, 'this scenario must persist a session log').toBe(childSessions + 1)
-          const fixtureFiles = ['session.jsonl', ...Array.from({ length: childSessions }, (_, i) => `session.${i + 1}.jsonl`)]
           for (let i = 0; i < fixtureFiles.length; i++) {
             const harvested = scrub((result.sessionLogs[i] as HarvestedLog).content)
             const fixture = scrub(await readFile(join(dir, fixtureFiles[i] as string), 'utf8'))