feat(session): opt-in packed chunk rows in the JSONL log

Providers stream token-sized deltas, so a session log stores hundreds of
near-identical assistant/chunk lines whose JSON envelopes dwarf their
payloads (~56x measured on a real DeepSeek session, 73% of file bytes).

Add a lossless storage codec to dsh-session: packChunkRuns() folds each
run of >=3 consecutive same-block delta chunks into one storage row --
text-chunks / reasoning-chunks / tool-call-chunks, bare slash-less tags
like the header line's 'session' so rows cannot be confused with session
events -- and decodeStorageRecord() expands rows back to the exact
original events (seq0/time0 + dt gap array reconstruct every member's
seq/time; tool-call rows carry the run-constant id/name). The encoder
whitelists exact shapes and stores anything unrecognized verbatim; the
decoder validates row-tagged values and fails loud on malformation.

The JSONL backend gains a packChunks config (default false). Writing
packs only when enabled -- default-off output stays byte-identical to
the previous layout, so snapshot goldens are untouched. Reading is
layout-blind: scanLog always decodes rows and now checks seq contiguity
with a cursor instead of the line index, so packed, unpacked, and mixed
files all load identically. Fixture readers (llm-replay parseSessionLog,
acp-snapshot normalizeSessionLog) share the codec; the normalizer zeroes
a row's time0/dt exactly like an event's time. The two demo bundles
plumb packChunks from cordis.yml to the backend.

Measured on a real coding session: 105 KB -> 42 KB (-60%), 475 lines ->
74, with reasoning/tool-call heavy sessions saving the most. Covered by
example + fast-check round-trip codec tests, backend packed/mixed/torn-
tail specs, and an end-to-end demo run loading a packed log through a
default-config backend.
This commit is contained in:
kingwl
2026-07-15 21:26:36 +08:00
parent b045b553a9
commit 7ef21239ca
21 changed files with 816 additions and 51 deletions

View File

@@ -9,6 +9,7 @@
import { existsSync, readFileSync } from 'node:fs'
import { delimiter as pathDelimiter } from 'node:path'
import type { Context } from 'cordis'
import { decodeStorageRecord } from '@deepseek-ai/dsh-session'
import type { SessionEvent } from '@deepseek-ai/dsh-session'
import type { GenerateOptions, StreamChunk } from '@deepseek-ai/dsh-llm'
import { LlmError, assertNever } from '@deepseek-ai/dsh-llm'
@@ -68,7 +69,9 @@ export interface SessionScript {
/**
* Parse a session `.jsonl` buffer into its event list. Line 0 is the session
* header (a `{type:'session',…}` record), every subsequent non-empty line is a
* {@link SessionEvent}. The header is skipped; malformed lines fail loud.
* {@link SessionEvent} or a packed chunk row (expanded back into its events, so
* a fixture recorded with `packChunks` on derives the same script). The header
* is skipped; malformed lines fail loud.
* @param text - the raw `.jsonl` file contents.
* @returns every event after the header, in log order.
*/
@@ -77,8 +80,7 @@ export function parseSessionLog(text: string): SessionEvent[] {
const events: SessionEvent[] = []
// The JSONL backend guarantees line 0 is the session header.
for (let i = 1; i < lines.length; i++) {
const parsed: unknown = JSON.parse(lines[i] as string)
events.push(parsed as SessionEvent)
events.push(...decodeStorageRecord(JSON.parse(lines[i] as string)))
}
return events
}

View File

@@ -90,6 +90,19 @@ describe('parseSessionLog', () => {
const ev = chunkEvent(1, 1, 1, TEXT_CHUNKS[0] as StreamChunk)
expect(parseSessionLog(`${header}\n\n${JSON.stringify(ev)}\n\n`)).toEqual([ev])
})
it('expands a packed chunk row into its events (a fixture recorded with packChunks on)', () => {
const header = JSON.stringify({ type: 'session', version: 0, id: 's1', createdAt: 0 })
const row = JSON.stringify({
type: 'text-chunks', seq0: 1, time0: 0,
data: { turn: 1, step: 1, index: 0, dt: [0, 0], texts: ['a', 'b', 'c'] },
})
expect(parseSessionLog(`${header}\n${row}\n`)).toEqual([
chunkEvent(1, 1, 1, { type: 'text-delta', index: 0, text: 'a' }),
chunkEvent(2, 1, 1, { type: 'text-delta', index: 0, text: 'b' }),
chunkEvent(3, 1, 1, { type: 'text-delta', index: 0, text: 'c' }),
])
})
})
describe('deriveReplayScript', () => {