Merge origin/master into worktree/ci-under-minute
# Conflicts: # packages/support/acp-snapshot/README.md
This commit is contained in:
@@ -6,8 +6,8 @@ Four layers, importable separately:
|
||||
|
||||
- **`launchAcpTestAgent` (launcher)** — boots an unbuilt ACP agent from a temp cwd, pins tsx to the repo tsconfig, connects the SDK client over a raw-byte stdout tee, collects session updates and stderr, surfaces asynchronous spawn failures through its startup lifecycle, fails closed on unhandled permission requests, and owns graceful or signalled shutdown. Shutdown waits for process exit, inherited stdio closure, and ACP parser exhaustion before resolving or propagating a child error, so captures are complete and callers can remove owned paths after either outcome. Snapshot and ordinary e2e suites share this process boundary; a test supplies only agent paths, cwd, environment overrides, and any permission policy.
|
||||
- **`runScenario` (harness)** — boots the real agent bin as a subprocess via tsx (unbuilt, Loader path), drives it over ACP JSON-RPC stdio from a deterministic `input.json` script, tees raw stdout for the expected-output and purity checks, and harvests every persisted session JSONL (parent + subagent children, primary-first) after a graceful stdin-EOF shutdown. Parameterized by `AgentUnderTest` (`binScript`, `configPath`, `tsconfigPath` — absolute paths; the subprocess cwd is a temp dir outside the repo). Startup failures preserve captured agent stderr in the rejected diagnostic.
|
||||
- **Normalizers** — pure functions turning the two captured surfaces into stable text: `normalizeStdout` (JSON-RPC ids → first-seen sequence; UUIDs/cwd → tokens; doubles as the stdout-purity check), `normalizeSessionLog` (times zeroed, `seq` kept), `scrubSystemPrompts` (prompt text → `{{system}}`), `scrubToolSchemas` (schema bulk → `{{tools}}`), and `scrubRequestHeaders` (all header bulk → `{{system}}`/`{{tools}}`/`{{messagePrefix}}` outside each pin, structure kept — [pinned-header Agent Note](../../../.agents/notes/implemented/testing/2026-07-06-pin-request-header-content-in-one-scenario.md)).
|
||||
- **`defineAcpSnapshotSuite` (factory)** — registers the whole describe/it tree for a scenario table: per-scenario expected-output and re-persisted-log comparisons, record/refresh fixture write-back, rejection of structured `UNKNOWN_TOOL` results, the per-header-class pin (`system-prompt.expected.md` plus `tool-schemas.expected.json`) with its live uniformity guard, and the fixture guard block (no orphan scenario dirs, required files present, exactly one pin per class, every JSONL prompt/schema-scrubbed, non-pinning fixtures fully header-scrubbed). Replay may partition subprocess-backed scenarios with `scenarioShard`; every lane still runs fixture guards against the complete table, while record and refresh reject sharding because they write fixtures. Each scenario directory's `session.jsonl` plus contiguous `session.<n>.jsonl` siblings are the ordered primary/child inventory; the scenario table does not duplicate their count. Must be called at vitest collection time.
|
||||
- **Normalizers** — pure functions turning the two captured surfaces into stable text: `normalizeStdout` (JSON-RPC ids → first-seen sequence; UUIDs/cwd → tokens; `session_info_update.updatedAt` → `{{updatedAt}}`; doubles as the stdout-purity check), `normalizeSessionLog` (times zeroed, `seq` kept), `scrubSystemPrompts` (prompt text → `{{system}}`), `scrubToolSchemas` (schema bulk → `{{tools}}`), and `scrubRequestHeaders` (all header bulk → `{{system}}`/`{{tools}}`/`{{messagePrefix}}` outside each pin, structure kept — [pinned-header Agent Note](../../../.agents/notes/implemented/testing/2026-07-06-pin-request-header-content-in-one-scenario.md)).
|
||||
- **`defineAcpSnapshotSuite` (factory)** — registers the whole describe/it tree for a scenario table: per-scenario expected-output and re-persisted-log comparisons, record/refresh fixture write-back, rejection of structured `UNKNOWN_TOOL` results, the per-header-class pin (`system-prompt.expected.md` plus `tool-schemas.expected.json`) with its live uniformity guard, and the fixture guard block (no orphan scenario dirs, required files present, exactly one pin per class, every JSONL prompt/schema-scrubbed, non-pinning fixtures fully header-scrubbed). Replay may partition subprocess-backed scenarios with `scenarioShard`; every lane still runs fixture guards against the complete table, while record and refresh reject sharding because they write fixtures. Refresh preserves existing volatile fields by event position and gives a newly inserted `session/title` its preceding event's time, so feature-driven insertions do not churn the remainder of a fixture. Each scenario directory's `session.jsonl` plus contiguous `session.<n>.jsonl` siblings are the ordered primary/child inventory; the scenario table does not duplicate their count. Must be called at vitest collection time.
|
||||
|
||||
A consuming `*.snapshot.ts` is the scenario table plus one factory call:
|
||||
|
||||
|
||||
@@ -41,6 +41,8 @@ export {
|
||||
} from './normalize.ts'
|
||||
export {
|
||||
defineAcpSnapshotSuite,
|
||||
refreshFixtureReplacements,
|
||||
stabilizeRefreshLog,
|
||||
type Scenario,
|
||||
type SnapshotSuiteOptions,
|
||||
} from './suite.ts'
|
||||
|
||||
@@ -11,6 +11,7 @@ const CWD = '{{cwd}}'
|
||||
const SYSTEM = '{{system}}'
|
||||
const TOOLS = '{{tools}}'
|
||||
const MESSAGE_PREFIX = '{{messagePrefix}}'
|
||||
const UPDATED_AT = '{{updatedAt}}'
|
||||
|
||||
/** A UUID v4 string, the shape `randomUUID()` produces for session ids. */
|
||||
const UUID_RE = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/gi
|
||||
@@ -85,6 +86,8 @@ export function normalizeStdout(rawStdout: string, ctx: NormalizeContext): strin
|
||||
if ('id' in frame && frame.id !== undefined && frame.id !== null) {
|
||||
frame.id = stableId(frame.id)
|
||||
}
|
||||
const update = (frame.params as { update?: Record<string, unknown> } | undefined)?.update
|
||||
if (update?.sessionUpdate === 'session_info_update') update.updatedAt = UPDATED_AT
|
||||
return scrubValue(frame, ctx) as Record<string, unknown>
|
||||
})
|
||||
return frames.map(f => JSON.stringify(f)).join('\n') + '\n'
|
||||
|
||||
@@ -426,8 +426,21 @@ export function stabilizeRefreshLog(fresh: string, existing: string, replacement
|
||||
for (const { from, to } of replacements) stable = stable.split(from).join(to)
|
||||
const existingRecords = parseJsonlRecords(existing)
|
||||
const records = parseJsonlRecords(stable)
|
||||
let existingIndex = 0
|
||||
let previousEventTime: unknown
|
||||
for (let i = 0; i < records.length; i++) {
|
||||
preserveFixtureVolatiles(records[i] as Record<string, unknown>, existingRecords[i])
|
||||
const record = records[i] as Record<string, unknown>
|
||||
const existingRecord = existingRecords[existingIndex]
|
||||
const insertedTitle = record.type === 'session/title' && existingRecord?.type !== 'session/title'
|
||||
if (insertedTitle) {
|
||||
/* v8 ignore next -- a title is turn-enclosed, so a preceding event time exists in every valid fixture. */
|
||||
if (typeof previousEventTime !== 'number') throw new Error('acp-snapshot: inserted title has no preceding event time')
|
||||
record.time = previousEventTime
|
||||
} else {
|
||||
preserveFixtureVolatiles(record, existingRecord)
|
||||
existingIndex += 1
|
||||
}
|
||||
if (typeof record.time === 'number') previousEventTime = record.time
|
||||
}
|
||||
return records.map(record => JSON.stringify(record)).join('\n') + '\n'
|
||||
}
|
||||
|
||||
@@ -55,6 +55,24 @@ describe('normalizeStdout', () => {
|
||||
expect(out).not.toContain('"id"')
|
||||
})
|
||||
|
||||
it('stabilizes the timestamp carried by session title updates', () => {
|
||||
const raw = JSON.stringify({
|
||||
jsonrpc: '2.0',
|
||||
method: 'session/update',
|
||||
params: {
|
||||
sessionId: ctx.sessionIds[0],
|
||||
update: {
|
||||
sessionUpdate: 'session_info_update',
|
||||
title: 'Stable title',
|
||||
updatedAt: '2026-07-20T17:03:13.689Z',
|
||||
},
|
||||
},
|
||||
})
|
||||
const out = normalizeStdout(raw, ctx)
|
||||
expect(out).toContain('"updatedAt":"{{updatedAt}}"')
|
||||
expect(out).not.toContain('2026-07-20T17:03:13.689Z')
|
||||
})
|
||||
|
||||
it('throws on a non-JSON stdout line (the purity check)', () => {
|
||||
const raw = `${JSON.stringify({ jsonrpc: '2.0', id: 1 })}\noops a log leaked\n`
|
||||
expect(() => normalizeStdout(raw, ctx)).toThrow()
|
||||
|
||||
@@ -429,6 +429,36 @@ describe('refreshFixtureReplacements', () => {
|
||||
})
|
||||
|
||||
describe('stabilizeRefreshLog', () => {
|
||||
it('aligns volatile times across a newly inserted log event', () => {
|
||||
const fresh = [
|
||||
'{"type":"session","id":"same","createdAt":200}',
|
||||
'{"type":"turn/start","seq":0,"time":21}',
|
||||
'{"type":"user/message","seq":1,"time":22}',
|
||||
'{"type":"session/title","seq":2,"time":999}',
|
||||
'{"type":"step/start","seq":3,"time":1000}',
|
||||
'{"type":"request/header","seq":4,"time":1001}',
|
||||
'',
|
||||
].join('\n')
|
||||
const existing = [
|
||||
'{"type":"session","id":"same","createdAt":100}',
|
||||
'{"type":"turn/start","seq":0,"time":11}',
|
||||
'{"type":"user/message","seq":1,"time":12}',
|
||||
'{"type":"step/start","seq":2,"time":13}',
|
||||
'{"type":"request/header","seq":3,"time":14}',
|
||||
'',
|
||||
].join('\n')
|
||||
|
||||
expect(stabilizeRefreshLog(fresh, existing, [])).toBe([
|
||||
'{"type":"session","id":"same","createdAt":100}',
|
||||
'{"type":"turn/start","seq":0,"time":11}',
|
||||
'{"type":"user/message","seq":1,"time":12}',
|
||||
'{"type":"session/title","seq":2,"time":12}',
|
||||
'{"type":"step/start","seq":3,"time":13}',
|
||||
'{"type":"request/header","seq":4,"time":14}',
|
||||
'',
|
||||
].join('\n'))
|
||||
})
|
||||
|
||||
it('keeps volatile fixture fields while preserving fresh meaningful payloads', () => {
|
||||
const fresh = [
|
||||
'{"type":"session","id":"new-child","createdAt":200,"cwd":"/new","parentSession":"new-parent","seedLength":1}',
|
||||
|
||||
Reference in New Issue
Block a user