feat(session): opt-in packed chunk rows in the JSONL log
Providers stream token-sized deltas, so a session log stores hundreds of near-identical assistant/chunk lines whose JSON envelopes dwarf their payloads (~56x measured on a real DeepSeek session, 73% of file bytes). Add a lossless storage codec to dsh-session: packChunkRuns() folds each run of >=3 consecutive same-block delta chunks into one storage row -- text-chunks / reasoning-chunks / tool-call-chunks, bare slash-less tags like the header line's 'session' so rows cannot be confused with session events -- and decodeStorageRecord() expands rows back to the exact original events (seq0/time0 + dt gap array reconstruct every member's seq/time; tool-call rows carry the run-constant id/name). The encoder whitelists exact shapes and stores anything unrecognized verbatim; the decoder validates row-tagged values and fails loud on malformation. The JSONL backend gains a packChunks config (default false). Writing packs only when enabled -- default-off output stays byte-identical to the previous layout, so snapshot goldens are untouched. Reading is layout-blind: scanLog always decodes rows and now checks seq contiguity with a cursor instead of the line index, so packed, unpacked, and mixed files all load identically. Fixture readers (llm-replay parseSessionLog, acp-snapshot normalizeSessionLog) share the codec; the normalizer zeroes a row's time0/dt exactly like an event's time. The two demo bundles plumb packChunks from cordis.yml to the backend. Measured on a real coding session: 105 KB -> 42 KB (-60%), 475 lines -> 74, with reasoning/tool-call heavy sessions saving the most. Covered by example + fast-check round-trip codec tests, backend packed/mixed/torn- tail specs, and an end-to-end demo run loading a packed log through a default-config backend.
This commit is contained in:
@@ -81,8 +81,10 @@ export function normalizeStdout(rawStdout: string, ctx: NormalizeContext): strin
|
||||
* Normalize a session JSONL log into a stable golden: the header line's
|
||||
* volatile fields (`createdAt`, `id`, `cwd`) and every event's `time` are
|
||||
* zeroed/scrubbed, all volatile strings scrubbed, and `seq` is LEFT INTACT
|
||||
* (deterministic by contract). Output is JSONL in the same shape as the input —
|
||||
* one compact record per line.
|
||||
* (deterministic by contract). A packed chunk row's timing (`time0`, the `dt`
|
||||
* gaps) zeroes just like an event `time`; its `seq0` stays, like `seq`.
|
||||
* Output is JSONL in the same shape as the input — one compact record per
|
||||
* line.
|
||||
*
|
||||
* @param rawLog The raw session `.jsonl` content.
|
||||
* @param ctx The run's volatile values to scrub.
|
||||
@@ -95,6 +97,13 @@ export function normalizeSessionLog(rawLog: string, ctx: NormalizeContext): stri
|
||||
// Header line: { type: 'session', createdAt, id, cwd, … }.
|
||||
if (record.type === 'session') {
|
||||
if ('createdAt' in record) record.createdAt = 0
|
||||
} else if ('time0' in record) {
|
||||
// Packed chunk row: zero the anchor timestamp and every member gap.
|
||||
record.time0 = 0
|
||||
const data = record.data
|
||||
if (data !== null && typeof data === 'object' && Array.isArray((data as { dt?: unknown }).dt)) {
|
||||
(data as { dt: unknown[] }).dt = (data as { dt: unknown[] }).dt.map(() => 0)
|
||||
}
|
||||
} else if ('time' in record) {
|
||||
// Event line: zero the epoch-ms timestamp; keep seq (deterministic).
|
||||
record.time = 0
|
||||
|
||||
@@ -109,6 +109,27 @@ describe('normalizeSessionLog', () => {
|
||||
expect(out).toContain('"decision":"block"') // the decision is the behavior — kept
|
||||
})
|
||||
|
||||
it('zeroes a packed chunk row\'s time0 and dt gaps but keeps seq0 and payload', () => {
|
||||
const row = JSON.stringify({
|
||||
type: 'text-chunks', seq0: 7, time0: 999,
|
||||
data: { turn: 1, step: 1, index: 0, dt: [212, 27, 0], texts: ['a', 'b', 'c', 'd'] },
|
||||
})
|
||||
const out = normalizeSessionLog(`${header({})}\n${row}\n`, ctx)
|
||||
expect(out).toContain('"time0":0')
|
||||
expect(out).toContain('"dt":[0,0,0]')
|
||||
expect(out).toContain('"seq0":7') // seq0 is deterministic, like seq — NOT scrubbed
|
||||
expect(out).toContain('"texts":["a","b","c","d"]')
|
||||
expect(out).not.toContain('999')
|
||||
expect(out).not.toContain('212')
|
||||
})
|
||||
|
||||
it('zeroes time0 even when a malformed row carries no dt array', () => {
|
||||
const row = JSON.stringify({ type: 'text-chunks', seq0: 1, time0: 999, data: 'not-an-object' })
|
||||
const out = normalizeSessionLog(`${header({})}\n${row}\n`, ctx)
|
||||
expect(out).toContain('"time0":0')
|
||||
expect(out).not.toContain('999')
|
||||
})
|
||||
|
||||
it('leaves a non-hook event durationMs untouched (only hook/result is scrubbed)', () => {
|
||||
const ev = JSON.stringify({ type: 'tool/result', seq: 2, time: 5, data: { durationMs: 88 } })
|
||||
const out = normalizeSessionLog(`${header({})}\n${ev}\n`, ctx)
|
||||
|
||||
@@ -9,6 +9,7 @@
|
||||
import { existsSync, readFileSync } from 'node:fs'
|
||||
import { delimiter as pathDelimiter } from 'node:path'
|
||||
import type { Context } from 'cordis'
|
||||
import { decodeStorageRecord } from '@deepseek-ai/dsh-session'
|
||||
import type { SessionEvent } from '@deepseek-ai/dsh-session'
|
||||
import type { GenerateOptions, StreamChunk } from '@deepseek-ai/dsh-llm'
|
||||
import { LlmError, assertNever } from '@deepseek-ai/dsh-llm'
|
||||
@@ -68,7 +69,9 @@ export interface SessionScript {
|
||||
/**
|
||||
* Parse a session `.jsonl` buffer into its event list. Line 0 is the session
|
||||
* header (a `{type:'session',…}` record), every subsequent non-empty line is a
|
||||
* {@link SessionEvent}. The header is skipped; malformed lines fail loud.
|
||||
* {@link SessionEvent} or a packed chunk row (expanded back into its events, so
|
||||
* a fixture recorded with `packChunks` on derives the same script). The header
|
||||
* is skipped; malformed lines fail loud.
|
||||
* @param text - the raw `.jsonl` file contents.
|
||||
* @returns every event after the header, in log order.
|
||||
*/
|
||||
@@ -77,8 +80,7 @@ export function parseSessionLog(text: string): SessionEvent[] {
|
||||
const events: SessionEvent[] = []
|
||||
// The JSONL backend guarantees line 0 is the session header.
|
||||
for (let i = 1; i < lines.length; i++) {
|
||||
const parsed: unknown = JSON.parse(lines[i] as string)
|
||||
events.push(parsed as SessionEvent)
|
||||
events.push(...decodeStorageRecord(JSON.parse(lines[i] as string)))
|
||||
}
|
||||
return events
|
||||
}
|
||||
|
||||
@@ -90,6 +90,19 @@ describe('parseSessionLog', () => {
|
||||
const ev = chunkEvent(1, 1, 1, TEXT_CHUNKS[0] as StreamChunk)
|
||||
expect(parseSessionLog(`${header}\n\n${JSON.stringify(ev)}\n\n`)).toEqual([ev])
|
||||
})
|
||||
|
||||
it('expands a packed chunk row into its events (a fixture recorded with packChunks on)', () => {
|
||||
const header = JSON.stringify({ type: 'session', version: 0, id: 's1', createdAt: 0 })
|
||||
const row = JSON.stringify({
|
||||
type: 'text-chunks', seq0: 1, time0: 0,
|
||||
data: { turn: 1, step: 1, index: 0, dt: [0, 0], texts: ['a', 'b', 'c'] },
|
||||
})
|
||||
expect(parseSessionLog(`${header}\n${row}\n`)).toEqual([
|
||||
chunkEvent(1, 1, 1, { type: 'text-delta', index: 0, text: 'a' }),
|
||||
chunkEvent(2, 1, 1, { type: 'text-delta', index: 0, text: 'b' }),
|
||||
chunkEvent(3, 1, 1, { type: 'text-delta', index: 0, text: 'c' }),
|
||||
])
|
||||
})
|
||||
})
|
||||
|
||||
describe('deriveReplayScript', () => {
|
||||
|
||||
Reference in New Issue
Block a user