refactor: simplify session log representation
This commit is contained in:
@@ -35,7 +35,7 @@ defineAcpSnapshotSuite({
|
||||
})
|
||||
```
|
||||
|
||||
A scenario booting a differently-composed tree sets its own `configPath` (an overlay whose basename still ends in `cordis.yml`, so the bin's replay swap finds the sibling `*cordis.snapshot.yml`) and, when that composition changes the request header, its own `headerClass` with its own pinning scenario — the acp-agent example's Code Mode scenarios are the template. Each pinning directory's generated `system-prompt.golden.md` is the reviewable snapshot of the normalized composed prompt; `session.jsonl` stores `"system":"{{system}}"` while retaining the complete tool list.
|
||||
A scenario booting a differently-composed tree sets its own `configPath` (an overlay whose basename still ends in `cordis.yml`, so the bin's replay swap finds the sibling `*cordis.snapshot.yml`) and, when that composition changes the request header, its own `headerClass` with its own pinning scenario — the acp-agent example's Code Mode scenarios are the template. Each pinning directory's generated `system-prompt.golden.md` is the reviewable snapshot of the normalized composed prompt; `session.jsonl` stores `"system":"{{system}}"` while retaining the complete tool list. A pin whose scenario legitimately changes its header mid-run declares `expectedHeaderChanges`; the Markdown snapshot then records each later full prompt under a `request/header change` marker.
|
||||
|
||||
The example also ships a `cordis.snapshot.yml` replay overlay next to its `cordis.yml` (the bin swaps them under `DSH_SNAPSHOT=replay` — [single-source replay config RFC](../../../docs/rfc/implemented/testing/2026-07-04-single-source-acp-replay-config.md)); replay fixtures are served by [`dsh-llm-replay`](../llm-replay/README.md), which this package points at via the `DSH_SNAPSHOT_*` env vars it sets on the child. `pnpm run test:snapshot:record` calls the live LLM and rewrites the recorded scenarios' model fixtures; `pnpm run test:snapshot:refresh` stays keyless, runs the replay overlay, and rewrites stdout, comparable session-log goldens, and each pin's Markdown prompt snapshot from the committed model scripts. Fixture roles, record/replay/refresh semantics, and scenario-table fields are documented on `Scenario` and in the [snapshot RFC](../../../docs/rfc/implemented/testing/2026-06-19-acp-snapshot-tests.md).
|
||||
|
||||
|
||||
@@ -135,8 +135,8 @@ export function normalizeSessionLog(rawLog: string, ctx: NormalizeContext): stri
|
||||
}
|
||||
|
||||
/**
|
||||
* Replace system-prompt content in request headers and header deltas with
|
||||
* `{{system}}` tokens while retaining field presence and delta structure.
|
||||
* Replace system-prompt content in request headers with `{{system}}` tokens
|
||||
* while retaining field presence.
|
||||
* Other header content stays verbatim, so a header-pinning fixture can keep
|
||||
* its complete tool schemas while every JSONL fixture omits the prompt text.
|
||||
* Lines without a system payload pass through byte-for-byte; the transform is
|
||||
@@ -153,9 +153,9 @@ export function scrubSystemPrompts(rawLog: string): string {
|
||||
* Replace all bulky request-header content in a session JSONL with stable
|
||||
* tokens. This includes the system-prompt fields handled by
|
||||
* {@link scrubSystemPrompts}, tool schemas, and session-prefix messages. It
|
||||
* keeps system-delta line positions and arity, tool-delta names, prefix
|
||||
* message counts, field presence, config, and reason. Lines without content
|
||||
* to scrub pass through byte-for-byte, and the transform is idempotent.
|
||||
* keeps prefix message counts, field presence, config, and reason. Lines
|
||||
* without content to scrub pass through byte-for-byte, and the transform is
|
||||
* idempotent.
|
||||
*
|
||||
* @param rawLog The raw session `.jsonl` content.
|
||||
* @returns The JSONL with all header bulk tokenized, other lines byte-identical.
|
||||
@@ -184,33 +184,7 @@ function scrubHeaderContent(rawLog: string, scrubToolsAndPrefix: boolean): strin
|
||||
}
|
||||
return touched ? JSON.stringify(record) : line
|
||||
}
|
||||
if (record.type === 'request/header-delta') {
|
||||
let touched = false
|
||||
const system = data.system as Record<string, unknown> | null | undefined
|
||||
if (system !== null && typeof system === 'object' && Array.isArray(system.insert)) {
|
||||
system.insert = system.insert.map(() => SYSTEM)
|
||||
touched = true
|
||||
}
|
||||
const tools = data.tools as Record<string, unknown> | null | undefined
|
||||
if (scrubToolsAndPrefix && tools !== null && typeof tools === 'object') {
|
||||
if (Array.isArray(tools.added)) { tools.added = tools.added.map(scrubToolSchema); touched = true }
|
||||
if (Array.isArray(tools.changed)) { tools.changed = tools.changed.map(scrubToolSchema); touched = true }
|
||||
}
|
||||
if (scrubToolsAndPrefix && Array.isArray(data.messagePrefix)) {
|
||||
data.messagePrefix = data.messagePrefix.map(() => MESSAGE_PREFIX)
|
||||
touched = true
|
||||
}
|
||||
return touched ? JSON.stringify(record) : line
|
||||
}
|
||||
return line
|
||||
})
|
||||
return out.join('\n')
|
||||
}
|
||||
|
||||
/** Tokenize one tool schema's bulk (description, parameters, anything else), keeping its identifying `name`. */
|
||||
function scrubToolSchema(tool: unknown): unknown {
|
||||
if (tool === null || typeof tool !== 'object' || Array.isArray(tool)) return tool
|
||||
const out: Record<string, unknown> = {}
|
||||
for (const [k, v] of Object.entries(tool)) out[k] = k === 'name' ? v : TOOLS
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -16,7 +16,7 @@
|
||||
* scenario stores the readable prompt in `system-prompt.golden.md` and keeps its full
|
||||
* tool schemas in `session.jsonl`, while every other fixture also scrubs tools
|
||||
* to `{{tools}}`. A per-run uniformity guard compares both artifacts against
|
||||
* every live header and forbids unrepresented header deltas (see the
|
||||
* every live header and forbids unrepresented changed headers (see the
|
||||
* pinned-header RFC,
|
||||
* docs/rfc/implemented/testing/2026-07-06-pin-request-header-content-in-one-scenario.md).
|
||||
*
|
||||
@@ -106,14 +106,11 @@ export interface Scenario {
|
||||
*/
|
||||
pinsHeader?: boolean
|
||||
/**
|
||||
* How many `request/header-delta` events this PINNING scenario's fixture
|
||||
* legitimately carries (default 0). A recorded mid-run header change — a
|
||||
* config-option switch rewriting a prompt section — is part of the pinned
|
||||
* surface, with readable prompt text in Markdown; any OTHER count
|
||||
* still fails, so fixture rot stays caught. Meaningless off the pin (the
|
||||
* live uniformity guard keeps non-pinning scenarios delta-free).
|
||||
* How many changed `request/header` snapshots this PINNING scenario's primary
|
||||
* fixture legitimately carries (default 0). Their full prompt text is kept in
|
||||
* the readable Markdown pin; any other count fails. Meaningless off the pin.
|
||||
*/
|
||||
expectedHeaderDeltas?: number
|
||||
expectedHeaderChanges?: number
|
||||
/**
|
||||
* Which header-composition class this scenario belongs to. Scenarios that
|
||||
* boot the same config compose the same header; each class has exactly one
|
||||
@@ -225,79 +222,46 @@ export function normalizedSystemPrompts(rawLog: string, ctx: NormalizeContext):
|
||||
})
|
||||
}
|
||||
|
||||
/** One normalized system-prompt edit carried by a `request/header-delta`. */
|
||||
export interface SystemPromptDeltaSnapshot {
|
||||
/** How many leading lines remain from the prior prompt. */
|
||||
keepStart: number
|
||||
/** How many trailing lines remain from the prior prompt. */
|
||||
keepEnd: number
|
||||
/** The normalized replacement lines inserted between the retained ranges. */
|
||||
insert: string[]
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract normalized system-prompt edits from request-header deltas in log
|
||||
* order. Deltas without a well-formed system edit are omitted; their non-prompt
|
||||
* structure remains pinned in JSONL.
|
||||
*
|
||||
* @param rawLog The session `.jsonl` content to inspect.
|
||||
* @param ctx The volatile values of the run that produced it.
|
||||
* @returns The normalized system-prompt edits, in event order.
|
||||
*/
|
||||
export function normalizedSystemPromptDeltas(rawLog: string, ctx: NormalizeContext): SystemPromptDeltaSnapshot[] {
|
||||
return normalizeSessionLog(rawLog, ctx)
|
||||
.split('\n')
|
||||
.filter(line => line.trim().length > 0)
|
||||
.map(line => JSON.parse(line) as { type?: unknown; data?: { system?: unknown } })
|
||||
.filter(record => record.type === 'request/header-delta')
|
||||
.flatMap((record) => {
|
||||
const system = record.data?.system
|
||||
if (system === null || typeof system !== 'object') return []
|
||||
const { keepStart, keepEnd, insert } = system as { keepStart?: unknown; keepEnd?: unknown; insert?: unknown }
|
||||
if (typeof keepStart !== 'number' || typeof keepEnd !== 'number' || !Array.isArray(insert)) return []
|
||||
if (!insert.every(line => typeof line === 'string')) return []
|
||||
return [{ keepStart, keepEnd, insert: insert }]
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Render a normalized prompt as a repository-friendly Markdown snapshot.
|
||||
* Prompt text is unchanged except that a missing terminal newline is added so
|
||||
* the committed file follows the repository newline contract.
|
||||
*
|
||||
* @param prompt The normalized system prompt.
|
||||
* @param deltas Normalized prompt edits to append as readable sections.
|
||||
* @param changes Full normalized prompts from later changed-header snapshots.
|
||||
* @returns Markdown snapshot text ending in a newline.
|
||||
*/
|
||||
export function formatSystemPromptSnapshot(
|
||||
prompt: string,
|
||||
deltas: readonly SystemPromptDeltaSnapshot[] = [],
|
||||
changes: readonly string[] = [],
|
||||
): string {
|
||||
let snapshot = prompt.endsWith('\n') ? prompt : `${prompt}\n`
|
||||
for (const [index, delta] of deltas.entries()) {
|
||||
snapshot += `\n<!-- request/header-delta ${index + 1}: keepStart=${delta.keepStart}, keepEnd=${delta.keepEnd} -->\n\n`
|
||||
const insert = delta.insert.join('\n')
|
||||
snapshot += insert.endsWith('\n') ? insert : `${insert}\n`
|
||||
for (const [index, change] of changes.entries()) {
|
||||
snapshot += `\n<!-- request/header change ${index + 1} -->\n\n`
|
||||
snapshot += change.endsWith('\n') ? change : `${change}\n`
|
||||
}
|
||||
return snapshot
|
||||
}
|
||||
|
||||
/** Return the initial-prompt portion of a possibly delta-bearing snapshot. */
|
||||
/** Return the initial-prompt portion of a possibly multi-header snapshot. */
|
||||
function initialSystemPromptSnapshot(snapshot: string): string {
|
||||
const marker = snapshot.indexOf('\n<!-- request/header-delta ')
|
||||
const marker = snapshot.indexOf('\n<!-- request/header change ')
|
||||
return marker < 0 ? snapshot : snapshot.slice(0, marker)
|
||||
}
|
||||
|
||||
/**
|
||||
* Count the `request/header-delta` events in a session JSONL.
|
||||
* Count changed `request/header` snapshots in a session JSONL.
|
||||
*
|
||||
* @param rawLog The session `.jsonl` content.
|
||||
* @returns How many `request/header-delta` events the log carries.
|
||||
* @returns How many headers carry reason `change`.
|
||||
*/
|
||||
export function headerDeltaCount(rawLog: string): number {
|
||||
export function headerChangeCount(rawLog: string): number {
|
||||
return rawLog.split('\n')
|
||||
.filter(line => line.trim().length > 0)
|
||||
.filter(line => (JSON.parse(line) as { type?: unknown }).type === 'request/header-delta')
|
||||
.filter((line) => {
|
||||
const record = JSON.parse(line) as { type?: unknown; data?: { reason?: unknown } }
|
||||
return record.type === 'request/header' && record.data?.reason === 'change'
|
||||
})
|
||||
.length
|
||||
}
|
||||
|
||||
@@ -490,18 +454,10 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
|
||||
))
|
||||
}
|
||||
if (scenario.pinsHeader === true) {
|
||||
const prompts = result.sessionLogs.flatMap(log => normalizedSystemPrompts(log.content, ctx))
|
||||
expect(prompts.length, `${mode} produced no system prompt to snapshot`).toBeGreaterThan(0)
|
||||
const initialSnapshot = formatSystemPromptSnapshot(prompts[0] as string)
|
||||
for (const prompt of prompts) {
|
||||
expect(formatSystemPromptSnapshot(prompt), 'the pinning run produced divergent system prompts')
|
||||
.toEqual(initialSnapshot)
|
||||
}
|
||||
const primary = result.sessionLogs[0] as HarvestedLog
|
||||
const snapshot = formatSystemPromptSnapshot(
|
||||
prompts[0] as string,
|
||||
normalizedSystemPromptDeltas(primary.content, ctx),
|
||||
)
|
||||
const prompts = normalizedSystemPrompts(primary.content, ctx)
|
||||
expect(prompts.length, `${mode} produced no system prompt to snapshot`).toBeGreaterThan(0)
|
||||
const snapshot = formatSystemPromptSnapshot(prompts[0] as string, prompts.slice(1))
|
||||
await writeFile(join(dir, SYSTEM_PROMPT_SNAPSHOT), snapshot)
|
||||
}
|
||||
}
|
||||
@@ -534,9 +490,8 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
|
||||
|
||||
// Header-uniformity guard: every live header in a class must equal the
|
||||
// class pin split across its JSONL header (system token + real tools)
|
||||
// and readable Markdown prompt. A pinning scenario may carry its
|
||||
// declared header deltas; their prompt edits live in the Markdown
|
||||
// golden while JSONL retains the tokenized edit structure.
|
||||
// and readable Markdown prompt. A pinning scenario may carry declared
|
||||
// changed-header snapshots; each full prompt lives in that Markdown.
|
||||
/* v8 ignore next -- construction guarantees the pin exists; a miss would fail the one-header assertion loudly. */
|
||||
const pinningScenario = pinningByClass.get(classOf(scenario)) ?? scenario
|
||||
const pinningDir = join(snapshotsDir, pinningScenario.name)
|
||||
@@ -544,29 +499,32 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
|
||||
const pinned = normalizedHeaders(pinnedFixture, fixtureContext(pinnedFixture))
|
||||
const promptSnapshot = await readFile(join(pinningDir, SYSTEM_PROMPT_SNAPSHOT), 'utf8')
|
||||
const initialPromptSnapshot = initialSystemPromptSnapshot(promptSnapshot)
|
||||
expect(pinned.length, `the pinning fixture (${pinningScenario.name}) must carry exactly one request/header`)
|
||||
.toBe(1)
|
||||
expect(pinned.length, `the pinning fixture (${pinningScenario.name}) has an unexpected request/header count`)
|
||||
.toBe(1 + (pinningScenario.expectedHeaderChanges ?? 0))
|
||||
for (const [logIndex, log] of result.sessionLogs.entries()) {
|
||||
const expectedDeltas = scenario.pinsHeader === true && logIndex === 0
|
||||
? scenario.expectedHeaderDeltas ?? 0
|
||||
const expectedChanges = scenario.pinsHeader === true && logIndex === 0
|
||||
? scenario.expectedHeaderChanges ?? 0
|
||||
: 0
|
||||
expect(headerDeltaCount(log.content), `session ${log.id}: request/header-delta count`)
|
||||
.toBe(expectedDeltas)
|
||||
expect(headerChangeCount(log.content), `session ${log.id}: changed request/header count`)
|
||||
.toBe(expectedChanges)
|
||||
const headers = normalizedHeaders(scrubSystemPrompts(log.content), ctx)
|
||||
const prompts = normalizedSystemPrompts(log.content, ctx)
|
||||
expect(prompts.length, `session ${log.id}: every request/header must carry a string system prompt`)
|
||||
.toBe(headers.length)
|
||||
for (const [k, header] of headers.entries()) {
|
||||
const expected = expectedChanges > 0 ? pinned[k] : pinned[0]
|
||||
expect(header, `session ${log.id}: request/header #${k + 1} diverged from the pinned (${pinningScenario.name}) header`)
|
||||
.toEqual(pinned[0])
|
||||
expect(formatSystemPromptSnapshot(prompts[k] as string), `session ${log.id}: initial system prompt #${k + 1} diverged from ${pinningScenario.name}/${SYSTEM_PROMPT_SNAPSHOT}`)
|
||||
.toEqual(initialPromptSnapshot)
|
||||
.toEqual(expected)
|
||||
if (expectedChanges === 0) {
|
||||
expect(formatSystemPromptSnapshot(prompts[k] as string), `session ${log.id}: initial system prompt #${k + 1} diverged from ${pinningScenario.name}/${SYSTEM_PROMPT_SNAPSHOT}`)
|
||||
.toEqual(initialPromptSnapshot)
|
||||
}
|
||||
}
|
||||
if (scenario.pinsHeader === true && logIndex === 0) {
|
||||
expect(formatSystemPromptSnapshot(
|
||||
prompts[0] as string,
|
||||
normalizedSystemPromptDeltas(log.content, ctx),
|
||||
), `session ${log.id}: system-prompt deltas diverged from ${pinningScenario.name}/${SYSTEM_PROMPT_SNAPSHOT}`)
|
||||
prompts.slice(1),
|
||||
), `session ${log.id}: changed system prompts diverged from ${pinningScenario.name}/${SYSTEM_PROMPT_SNAPSHOT}`)
|
||||
.toEqual(promptSnapshot)
|
||||
}
|
||||
}
|
||||
@@ -632,22 +590,22 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
|
||||
}
|
||||
})
|
||||
|
||||
it('every pinning fixture carries one request/header, one readable prompt, and its declared deltas', async () => {
|
||||
it('every pinning fixture carries one anchor, one readable prompt artifact, and its declared changes', async () => {
|
||||
// The live uniformity guard runs only in NON-pinning scenarios, so a
|
||||
// class made of just its pinning scenario would otherwise accept a
|
||||
// re-recorded pin with several headers or an undeclared mid-run
|
||||
// header-delta — shapes the pin design cannot represent. Assert the
|
||||
// re-recorded pin with undeclared mid-run header changes. Assert the
|
||||
// committed pins directly; a scenario whose arc legitimately rewrites
|
||||
// a prompt section declares the exact count via expectedHeaderDeltas.
|
||||
// a prompt section declares the exact count via expectedHeaderChanges.
|
||||
for (const scenario of pinningByClass.values()) {
|
||||
const fixture = await readFile(join(snapshotsDir, scenario.name, 'session.jsonl'), 'utf8')
|
||||
const headers = normalizedHeaders(fixture, fixtureContext(fixture))
|
||||
const promptSnapshot = await readFile(join(snapshotsDir, scenario.name, SYSTEM_PROMPT_SNAPSHOT), 'utf8')
|
||||
expect(headers.length, `${scenario.name}: a pinning fixture must carry exactly one request/header`).toBe(1)
|
||||
expect(headers.length, `${scenario.name}: unexpected request/header count`)
|
||||
.toBe(1 + (scenario.expectedHeaderChanges ?? 0))
|
||||
expect(promptSnapshot.length, `${scenario.name}/${SYSTEM_PROMPT_SNAPSHOT} must not be empty`).toBeGreaterThan(0)
|
||||
expect(promptSnapshot.endsWith('\n'), `${scenario.name}/${SYSTEM_PROMPT_SNAPSHOT} must end in a newline`).toBe(true)
|
||||
expect(headerDeltaCount(fixture), `${scenario.name}: a pinning fixture must carry exactly its declared request/header-deltas`)
|
||||
.toBe(scenario.expectedHeaderDeltas ?? 0)
|
||||
expect(headerChangeCount(fixture), `${scenario.name}: a pinning fixture must carry exactly its declared changed headers`)
|
||||
.toBe(scenario.expectedHeaderChanges ?? 0)
|
||||
}
|
||||
})
|
||||
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
"lines": [
|
||||
{ "type": "session", "id": "{{SID}}", "createdAt": 100, "cwd": "{{CWD}}" },
|
||||
{ "type": "request/header", "seq": 0, "time": 100, "data": { "header": { "config": { "model": "fake" }, "system": "SYS PROMPT", "tools": [{ "name": "t1", "description": "D1", "parameters": { "type": "object" } }] }, "reason": "initial" } },
|
||||
{ "type": "request/header-delta", "seq": 1, "time": 100, "data": { "system": { "keepStart": 1, "keepEnd": 0, "insert": ["NEW PROMPT LINE"] } } },
|
||||
{ "type": "request/header", "seq": 1, "time": 100, "data": { "header": { "config": { "model": "fake" }, "system": "SYS PROMPT\n\nNEW PROMPT LINE", "tools": [{ "name": "t1", "description": "D1", "parameters": { "type": "object" } }] }, "reason": "change" } },
|
||||
{ "type": "turn/start", "seq": 2, "time": 100, "data": { "turn": 1 } }
|
||||
]
|
||||
}]
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
{"type":"session","id":"12121212-3434-4545-8686-787878787878","createdAt":7,"cwd":"/rec/pin-cwd"}
|
||||
{"type":"request/header","seq":0,"time":7,"data":{"header":{"config":{"model":"fake"},"system":"{{system}}","tools":[{"name":"t1","description":"D1","parameters":{"type":"object"}}]},"reason":"initial"}}
|
||||
{"type":"request/header-delta","seq":1,"time":7,"data":{"system":{"keepStart":1,"keepEnd":0,"insert":["{{system}}"]}}}
|
||||
{"type":"request/header","seq":1,"time":7,"data":{"header":{"config":{"model":"fake"},"system":"{{system}}","tools":[{"name":"t1","description":"D1","parameters":{"type":"object"}}]},"reason":"change"}}
|
||||
{"type":"turn/start","seq":2,"time":7,"data":{"turn":1}}
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
SYS PROMPT
|
||||
|
||||
<!-- request/header-delta 1: keepStart=1, keepEnd=0 -->
|
||||
<!-- request/header change 1 -->
|
||||
|
||||
SYS PROMPT
|
||||
|
||||
NEW PROMPT LINE
|
||||
|
||||
@@ -180,89 +180,19 @@ describe('scrubRequestHeaders', () => {
|
||||
expect(scrubRequestHeaders(`${headerLine}\n${odd}\n`)).toContain('"messagePrefix":"weird"')
|
||||
})
|
||||
|
||||
it('scrubs a header-delta prefix replacement to one token per message', () => {
|
||||
const delta = JSON.stringify({
|
||||
type: 'request/header-delta', seq: 8, time: 9,
|
||||
data: { messagePrefix: [{ role: 'user', content: [{ type: 'text', text: 'leaked opener' }] }] },
|
||||
})
|
||||
const out = scrubRequestHeaders(`${headerLine}\n${delta}\n`)
|
||||
expect(out).toContain('"messagePrefix":["{{messagePrefix}}"]')
|
||||
expect(out).not.toContain('leaked opener')
|
||||
// The empty-array transition-to-absence stays a structural fact.
|
||||
const toNone = JSON.stringify({ type: 'request/header-delta', seq: 9, time: 9, data: { messagePrefix: [] } })
|
||||
expect(scrubRequestHeaders(`${headerLine}\n${toNone}\n`)).toContain('"messagePrefix":[]')
|
||||
})
|
||||
|
||||
it('leaves a delta with no scrubbable payload byte-identical (config-only, or non-array shapes)', () => {
|
||||
const configOnly = JSON.stringify({ type: 'request/header-delta', seq: 8, time: 9, data: { config: { model: 'm2' } } })
|
||||
const oddShapes = JSON.stringify({ type: 'request/header-delta', seq: 9, time: 9, data: { system: { insert: 'not-an-array' }, tools: null } })
|
||||
it('leaves malformed headers with no scrubbable payload byte-identical', () => {
|
||||
const headerless = JSON.stringify({ type: 'request/header', seq: 10, time: 9, data: { reason: 'initial' } })
|
||||
const nullData = JSON.stringify({ type: 'request/header', seq: 11, time: 9, data: null })
|
||||
const raw = `${headerLine}\n${configOnly}\n${oddShapes}\n${headerless}\n${nullData}\n`
|
||||
const raw = `${headerLine}\n${headerless}\n${nullData}\n`
|
||||
expect(scrubRequestHeaders(raw)).toBe(raw)
|
||||
})
|
||||
|
||||
it('scrubs a one-sided tools delta and passes non-object schema entries through', () => {
|
||||
const addedOnly = JSON.stringify({
|
||||
type: 'request/header-delta', seq: 8, time: 9,
|
||||
data: { tools: { added: [null, 'weird', { name: 'x', description: 'D' }] } },
|
||||
})
|
||||
const out = scrubRequestHeaders(`${headerLine}\n${addedOnly}\n`)
|
||||
// Non-object entries survive untouched; the object entry keeps only name.
|
||||
expect(out).toContain('"added":[null,"weird",{"name":"x","description":"{{tools}}"}]')
|
||||
const changedOnly = JSON.stringify({
|
||||
type: 'request/header-delta', seq: 8, time: 9,
|
||||
data: { tools: { changed: [{ name: 'y', parameters: {} }] } },
|
||||
})
|
||||
expect(scrubRequestHeaders(`${headerLine}\n${changedOnly}\n`))
|
||||
.toContain('"changed":[{"name":"y","parameters":"{{tools}}"}]')
|
||||
})
|
||||
|
||||
it('scrubs a header-delta system payload but keeps its line positions and arity', () => {
|
||||
const delta = JSON.stringify({
|
||||
type: 'request/header-delta', seq: 8, time: 9,
|
||||
data: { system: { keepStart: 1, keepEnd: 4, insert: ['leaked prompt line', 'second line'] }, config: { model: 'm2' } },
|
||||
})
|
||||
const out = scrubRequestHeaders(`${headerLine}\n${delta}\n`)
|
||||
// One token PER inserted line: the edit's position AND extent survive.
|
||||
expect(out).toContain('"insert":["{{system}}","{{system}}"]')
|
||||
expect(out).toContain('"keepStart":1')
|
||||
expect(out).toContain('"keepEnd":4')
|
||||
expect(out).toContain('"config":{"model":"m2"}')
|
||||
expect(out).not.toContain('leaked prompt line')
|
||||
expect(out).not.toContain('{{tools}}') // no tools delta → none invented
|
||||
})
|
||||
|
||||
it('scrubs a header-delta tools payload but keeps the added/removed/changed names', () => {
|
||||
const delta = JSON.stringify({
|
||||
type: 'request/header-delta', seq: 8, time: 9,
|
||||
data: {
|
||||
tools: {
|
||||
added: [{ name: 'grep', description: 'Search files.', parameters: { type: 'object' } }],
|
||||
removed: ['bash_kill'],
|
||||
changed: [{ name: 'read', description: 'Read v2.', parameters: { type: 'object' } }],
|
||||
},
|
||||
},
|
||||
})
|
||||
const out = scrubRequestHeaders(`${headerLine}\n${delta}\n`)
|
||||
// WHICH tools changed is behavior and survives; their bulk does not.
|
||||
expect(out).toContain('"added":[{"name":"grep","description":"{{tools}}","parameters":"{{tools}}"}]')
|
||||
expect(out).toContain('"removed":["bash_kill"]')
|
||||
expect(out).toContain('"changed":[{"name":"read","description":"{{tools}}","parameters":"{{tools}}"}]')
|
||||
expect(out).not.toContain('Search files')
|
||||
expect(out).not.toContain('Read v2')
|
||||
})
|
||||
|
||||
it('passes every other line through byte-for-byte and is idempotent', () => {
|
||||
const other = JSON.stringify({ type: 'assistant/chunk', seq: 4, time: 9, data: { turn: 1, step: 1, chunk: { type: 'text-delta', index: 0, text: 'hi' } } })
|
||||
const delta = JSON.stringify({
|
||||
type: 'request/header-delta', seq: 8, time: 9,
|
||||
data: { system: { keepStart: 0, keepEnd: 0, insert: ['x'] }, tools: { added: [{ name: 't', description: 'd', parameters: {} }], removed: [], changed: [] } },
|
||||
})
|
||||
const raw = `${headerLine}\n${headerEvent({ config: { model: 'm' }, system: 's', tools: [] })}\n${delta}\n${other}\n`
|
||||
const raw = `${headerLine}\n${headerEvent({ config: { model: 'm' }, system: 's', tools: [] })}\n${other}\n`
|
||||
const once = scrubRequestHeaders(raw)
|
||||
expect(once.split('\n')[0]).toBe(headerLine)
|
||||
expect(once.split('\n')[3]).toBe(other)
|
||||
expect(once.split('\n')[2]).toBe(other)
|
||||
expect(scrubRequestHeaders(once)).toBe(once)
|
||||
})
|
||||
})
|
||||
@@ -280,12 +210,15 @@ describe('scrubSystemPrompts', () => {
|
||||
reason: 'initial',
|
||||
},
|
||||
})
|
||||
const delta = JSON.stringify({
|
||||
type: 'request/header-delta', seq: 2, time: 3,
|
||||
const changed = JSON.stringify({
|
||||
type: 'request/header', seq: 2, time: 3,
|
||||
data: {
|
||||
system: { keepStart: 1, keepEnd: 2, insert: ['new prompt line'] },
|
||||
tools: { changed: [{ name: 'read', description: 'changed schema' }] },
|
||||
messagePrefix: [{ role: 'user', content: [{ type: 'text', text: 'changed prefix' }] }],
|
||||
header: {
|
||||
system: 'new prompt',
|
||||
tools: [{ name: 'read', description: 'changed schema' }],
|
||||
messagePrefix: [{ role: 'user', content: [{ type: 'text', text: 'changed prefix' }] }],
|
||||
},
|
||||
reason: 'change',
|
||||
},
|
||||
})
|
||||
const toolsOnly = JSON.stringify({
|
||||
@@ -293,11 +226,10 @@ describe('scrubSystemPrompts', () => {
|
||||
data: { header: { tools: [{ name: 'read', description: 'schema only' }] }, reason: 'resume' },
|
||||
})
|
||||
|
||||
const out = scrubSystemPrompts(`${header}\n${delta}\n${toolsOnly}\n`)
|
||||
const out = scrubSystemPrompts(`${header}\n${changed}\n${toolsOnly}\n`)
|
||||
expect(out).toContain('"system":"{{system}}"')
|
||||
expect(out).toContain('"insert":["{{system}}"]')
|
||||
expect(out).not.toContain('full prompt')
|
||||
expect(out).not.toContain('new prompt line')
|
||||
expect(out).not.toContain('new prompt')
|
||||
expect(out).toContain('full schema')
|
||||
expect(out).toContain('full prefix')
|
||||
expect(out).toContain('changed schema')
|
||||
|
||||
@@ -9,9 +9,8 @@ import {
|
||||
childFixturePaths,
|
||||
fixtureContext,
|
||||
formatSystemPromptSnapshot,
|
||||
headerDeltaCount,
|
||||
headerChangeCount,
|
||||
normalizedHeaders,
|
||||
normalizedSystemPromptDeltas,
|
||||
normalizedSystemPrompts,
|
||||
refreshFixtureReplacements,
|
||||
stabilizeRefreshLog,
|
||||
@@ -51,7 +50,7 @@ const RECORD_SRC = fileURLToPath(new URL('./fixtures/record-suite', import.meta.
|
||||
// is what this suite can exercise; the real overlay boot is the acp-agent
|
||||
// example's code-mode scenarios).
|
||||
const REPLAY_SCENARIOS: Scenario[] = [
|
||||
{ name: 'pin-turn', hasModelTurn: true, recorded: true, pinsHeader: true, expectedHeaderDeltas: 1, headerClass: 'main' },
|
||||
{ name: 'pin-turn', hasModelTurn: true, recorded: true, pinsHeader: true, expectedHeaderChanges: 1, headerClass: 'main' },
|
||||
{ name: 'plain-turn', hasModelTurn: true, recorded: true, childSessions: 1, headerClass: 'main', configPath: AGENT.configPath },
|
||||
{ name: 'no-model', hasModelTurn: false, recorded: false, headerClass: 'main' },
|
||||
{ name: 'blocked-log', hasModelTurn: false, comparesLog: true, recorded: false, headerClass: 'main' },
|
||||
@@ -132,7 +131,9 @@ describe('defineAcpSnapshotSuite: refresh write-back', () => {
|
||||
expect(readFileSync(join(refreshDir, 'pin-turn', 'system-prompt.golden.md'), 'utf8')).toBe([
|
||||
'SYS PROMPT',
|
||||
'',
|
||||
'<!-- request/header-delta 1: keepStart=1, keepEnd=0 -->',
|
||||
'<!-- request/header change 1 -->',
|
||||
'',
|
||||
'SYS PROMPT',
|
||||
'',
|
||||
'NEW PROMPT LINE',
|
||||
'',
|
||||
@@ -247,46 +248,30 @@ describe('normalizedSystemPrompts', () => {
|
||||
})
|
||||
})
|
||||
|
||||
describe('normalizedSystemPromptDeltas', () => {
|
||||
it('extracts and normalizes well-formed system edits', () => {
|
||||
const log = [
|
||||
'{"type":"request/header-delta","data":{"system":{"keepStart":1,"keepEnd":0,"insert":["work in /w"]}}}',
|
||||
'{"type":"request/header-delta","data":{"tools":{"replace":[]}}}',
|
||||
'{"type":"request/header-delta","data":{"system":{"keepStart":"1","keepEnd":0,"insert":[]}}}',
|
||||
'{"type":"request/header-delta","data":{"system":{"keepStart":1,"keepEnd":0,"insert":[null]}}}',
|
||||
'',
|
||||
].join('\n')
|
||||
expect(normalizedSystemPromptDeltas(log, { sessionIds: [], cwd: '/w' })).toEqual([
|
||||
{ keepStart: 1, keepEnd: 0, insert: ['work in {{cwd}}'] },
|
||||
])
|
||||
})
|
||||
})
|
||||
|
||||
describe('formatSystemPromptSnapshot', () => {
|
||||
it('adds a missing terminal newline without changing an existing one', () => {
|
||||
expect(formatSystemPromptSnapshot('prompt')).toBe('prompt\n')
|
||||
expect(formatSystemPromptSnapshot('prompt\n')).toBe('prompt\n')
|
||||
})
|
||||
|
||||
it('renders readable system-prompt delta sections', () => {
|
||||
expect(formatSystemPromptSnapshot('prompt', [
|
||||
{ keepStart: 1, keepEnd: 0, insert: ['new', 'lines'] },
|
||||
])).toBe('prompt\n\n<!-- request/header-delta 1: keepStart=1, keepEnd=0 -->\n\nnew\nlines\n')
|
||||
it('renders readable changed-prompt sections', () => {
|
||||
expect(formatSystemPromptSnapshot('prompt', ['new\nlines']))
|
||||
.toBe('prompt\n\n<!-- request/header change 1 -->\n\nnew\nlines\n')
|
||||
})
|
||||
|
||||
it('does not double the newline of a delta insert with a trailing blank line', () => {
|
||||
expect(formatSystemPromptSnapshot('prompt\n', [
|
||||
{ keepStart: 2, keepEnd: 1, insert: ['tail', ''] },
|
||||
])).toBe('prompt\n\n<!-- request/header-delta 1: keepStart=2, keepEnd=1 -->\n\ntail\n')
|
||||
it('does not double the newline of a changed prompt', () => {
|
||||
expect(formatSystemPromptSnapshot('prompt\n', ['changed\n']))
|
||||
.toBe('prompt\n\n<!-- request/header change 1 -->\n\nchanged\n')
|
||||
})
|
||||
})
|
||||
|
||||
describe('headerDeltaCount', () => {
|
||||
it('counts request/header-delta events, ignoring blanks and other lines', () => {
|
||||
const delta = JSON.stringify({ type: 'request/header-delta', seq: 2, time: 9, data: {} })
|
||||
const other = JSON.stringify({ type: 'request/header', seq: 0, time: 9, data: {} })
|
||||
expect(headerDeltaCount(`${other}\n\n${delta}\n${delta}\n`)).toBe(2)
|
||||
expect(headerDeltaCount(`${other}\n`)).toBe(0)
|
||||
describe('headerChangeCount', () => {
|
||||
it('counts changed request headers, ignoring anchors, blanks, and other lines', () => {
|
||||
const change = JSON.stringify({ type: 'request/header', seq: 2, time: 9, data: { reason: 'change' } })
|
||||
const anchor = JSON.stringify({ type: 'request/header', seq: 0, time: 9, data: { reason: 'initial' } })
|
||||
const other = JSON.stringify({ type: 'turn/start', seq: 1, time: 9, data: {} })
|
||||
expect(headerChangeCount(`${anchor}\n${other}\n\n${change}\n${change}\n`)).toBe(2)
|
||||
expect(headerChangeCount(`${anchor}\n`)).toBe(0)
|
||||
})
|
||||
})
|
||||
|
||||
|
||||
@@ -39,7 +39,7 @@ Agent status (per agent):
|
||||
|
||||
Model requests (on `llm/stream`):
|
||||
|
||||
- **a loop-built request is exactly what the log reconstructs** — a frozen request with a live `sessionId` (the loop-built marker; hand-built one-shots like compaction's summarize are unfrozen and skipped) must carry frozen `messages` deep-equal to the derivation over the log prefix strictly before the in-flight step's `step/start` (rebuilt through a FRESH `Session`, so the live cache cannot vouch for itself — and boundary-correct: content logged after `step/start` legitimately belongs to the next request), and every non-content field must equal the fold of the log's `request/header*` events (see [the reconstructability RFC](../../../docs/rfc/implemented/architecture/2026-07-05-reconstructable-requests.md)). Registered with `prepend: true` so a short-circuiting `llm/stream` listener (the replay adapter) cannot silence it; prepend orders it against append-registered listeners only — correctness rests on the seq-bounded rebuild, never listener timing.
|
||||
- **a loop-built request is exactly what the log reconstructs** — a frozen request with a live `sessionId` (the loop-built marker; hand-built one-shots like compaction's summarize are unfrozen and skipped) must carry frozen `messages` deep-equal to the derivation over the log prefix strictly before the in-flight step's `step/start` (rebuilt through a FRESH `Session`, so the live cache cannot vouch for itself — and boundary-correct: content logged after `step/start` legitimately belongs to the next request), and every non-content field must equal the latest logged `request/header` (see [the reconstructability RFC](../../../docs/rfc/implemented/architecture/2026-07-05-reconstructable-requests.md)). Registered with `prepend: true` so a short-circuiting `llm/stream` listener (the replay adapter) cannot silence it; prepend orders it against append-registered listeners only — correctness rests on the seq-bounded rebuild, never listener timing.
|
||||
|
||||
On any violation it throws `InvariantError` (`code: 'INVARIANT'`).
|
||||
|
||||
|
||||
@@ -60,8 +60,8 @@ interface SessionTrace {
|
||||
/** Every seq seen so far — validates `sourceEventSeqs` references. */
|
||||
knownSeqs: Set<number>
|
||||
/**
|
||||
* The seqs currently on the surface linked list, in linked-list order
|
||||
* (head to tail). A replace reorders this relative to seq order (the new
|
||||
* The seqs currently on the surface, in derived-message order. A replace
|
||||
* reorders this relative to seq order (the new
|
||||
* node takes the replaced range's position), so range validation is
|
||||
* positional, not by seq comparison.
|
||||
*/
|
||||
@@ -154,7 +154,7 @@ function validateEvent(trace: SessionTrace, event: SessionEvent): SessionTraceTr
|
||||
}
|
||||
}
|
||||
}
|
||||
// Fold this event into the tracked surface linked list, validating the
|
||||
// Fold this event into the tracked surface order, validating the
|
||||
// replace contract as we go. `append` adds a tail node; `replace` shadows a
|
||||
// positional range — every shadowed node must appear in sourceEventSeqs.
|
||||
if (se.surfaceOp !== undefined) {
|
||||
@@ -497,7 +497,7 @@ export function apply(ctx: Context): void {
|
||||
// the boundary (an `agent/request`-window inject) is legitimately absent
|
||||
// from this request, and a current-surface comparison would false-fire.
|
||||
// - header: every non-content field must equal the fold of the log's
|
||||
// `request/header*` events — the loop logs the header event BEFORE
|
||||
// `request/header` events — the loop logs the header event BEFORE
|
||||
// dispatch, so the fold already covers this request.
|
||||
//
|
||||
// Registered with `prepend: true` so a short-circuiting llm/stream listener
|
||||
|
||||
@@ -635,7 +635,7 @@ describe('surface invariants', () => {
|
||||
session.append('user/message', { content: [{ type: 'text', text: 'a' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) // seq 2
|
||||
session.append('user/message', { content: [{ type: 'text', text: 'b' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) // seq 3
|
||||
// Replace node 2 (position 0) with seq 4 — surface is now [4, 3], so seq 4
|
||||
// precedes seq 3 in linked-list order even though 4 > 3 numerically.
|
||||
// precedes seq 3 in surface order even though 4 > 3 numerically.
|
||||
session.append('assistant/message', { turn: 1, step: 1, content: [{ type: 'text', text: 's' }] }, { surfaceOp: { op: 'replace', start: 2, end: 2 }, sourceEventSeqs: [2] }) // seq 4
|
||||
// A replace with start=3, end=4 passes the seq check (3 <= 4) but is
|
||||
// reversed positionally (3 is at pos 1, 4 is at pos 0).
|
||||
@@ -745,7 +745,7 @@ describe('request-reconstruction cross-check (llm/stream)', () => {
|
||||
it('expects the folded header\'s session prefix ahead of the derivation (prefix + derived)', async () => {
|
||||
const { ctx, session, boundary } = await requestSetup()
|
||||
const prefix = { role: 'user' as const, content: [{ type: 'text' as const, text: '<system-reminder>catalog</system-reminder>' }] }
|
||||
session.append('request/header-delta', { messagePrefix: [prefix] })
|
||||
session.append('request/header', { header: { config: { model: 'm' }, messagePrefix: [prefix] }, reason: 'change' })
|
||||
// The prefixed request matches the fold…
|
||||
const prefixed = Object.freeze({ model: 'm', messages: Object.freeze([prefix, ...boundary]), sessionId: session.id })
|
||||
expect(() => { dispatch(ctx, prefixed) }).not.toThrow()
|
||||
|
||||
Reference in New Issue
Block a user