Merge refreshed schema DSL into canonical tool outputs

This commit is contained in:
Tianyi Cui
2026-07-22 22:46:10 +08:00
32 changed files with 1208 additions and 61 deletions

View File

@@ -11,7 +11,8 @@ The JSONL durable session-persistence backend — a concrete `SessionPersistence
<encoded-id>.jsonl # only with compression: 'none'
```
- The first logical line is the immutable `SessionHeader` tagged `{ type: 'session', version, id, cwd?, createdAt, parentSession?, seedLength?, delegationDepth }`. `delegationDepth` is required on disk and is `0` for a top-level session; a missing or invalid value rejects the log. Every subsequent line is one `SessionEvent` JSON, **verbatim including `assistant/chunk`** so `seq` stays contiguous (`events[i].seq === i`).
- The first logical line is the immutable `SessionHeader` tagged `{ type: 'session', version, id, cwd?, createdAt, parentSession?, seedLength?, delegationDepth }`. `delegationDepth` is required on disk and is `0` for a top-level session; a missing or invalid value rejects the log. Every subsequent logical line is one storage record; `assistant/chunk` events are never dropped, and `seq` stays contiguous across the decoded log (`events[i].seq === i`).
- A storage record is a `SessionEvent` JSON verbatim, or — written only under `packChunks` — a **packed chunk row** (`text-chunks` / `reasoning-chunks` / `tool-call-chunks`; bare slash-less tags like the header's `session`, so row tags cannot be confused with event types): one line holding a run of ≥3 consecutive same-block `assistant/chunk` delta events, `seq0`/`time0` plus per-member `dt` gaps reconstructing every member's `seq`/`time` exactly. The lossless codec lives in `@deepseek-ai/dsh-session` (`packChunkRuns`/`decodeStorageRecord`) and whitelists exact shapes — anything unrecognized stores verbatim. Reading is layout-blind: `load` always decodes rows, so packed, unpacked, and mixed files load identically.
- Session ids are unvalidated branded strings, so they are injectively escaped to a single safe path segment before use (no traversal, no collision).
## Config
@@ -19,6 +20,7 @@ The JSONL durable session-persistence backend — a concrete `SessionPersistence
| Key | Type | Notes |
|---|---|---|
| `root` | `string` (required) | Root directory for all session files. **No default** — a `process.cwd()` default would scatter files as the process's cwd changes (bash calls, subprocesses). |
| `packChunks` | `boolean` (default `false`) | Write delta-chunk runs as packed rows (~60% smaller logical logs measured on a real coding session). Off, the written logical layout is byte-identical to the pre-packing format; reading packed rows works regardless of this switch. Off by default while the snapshot goldens stay one-event-per-line — recording with packing on rewrites every fixture `session.jsonl`. |
| `compression` | `'zstd' \| 'none'` | Defaults to `'zstd'`; `'none'` retains newline-delimited UTF-8 text. |
`locate(meta)` returns `{ kind: 'jsonl', path }` using the resolved absolute root and the same cwd-bucket/id encoding as materialization. It performs no filesystem I/O: the target can be returned before the file exists, and an existing file contains only the last flushed prefix.

View File

@@ -10,7 +10,8 @@
import { createHash } from 'node:crypto'
import { join } from 'node:path'
import type { SessionEvent, SessionHeader, SessionId } from '@deepseek-ai/dsh-session'
import { decodeStorageRecord, packChunkRuns } from '@deepseek-ai/dsh-session'
import type { SessionEvent, SessionHeader, SessionId, StorageRecord } from '@deepseek-ai/dsh-session'
/** Physical encoding selected for JSONL session artifacts. */
export type JsonlCompression = 'zstd' | 'none'
@@ -151,17 +152,26 @@ export function logPath(
}
/**
* Serialize one event as a JSONL line (no trailing newline).
* @param event - the event to serialize verbatim.
* @returns the event's single-line JSON text; the writer adds the newline.
* Serialize an event batch as JSONL lines (no trailing newline). With
* `packChunks` on, delta-chunk runs pack into `text-chunks` /
* `reasoning-chunks` / `tool-call-chunks` storage rows; off writes one event
* per line, byte-identical to the pre-packing layout. Reading is layout-blind
* either way ({@link scanLog} always decodes rows), so the switch only shapes
* NEW bytes.
* @param events - the batch to serialize, in log order.
* @param packChunks - whether to pack delta runs into storage rows.
* @returns the batch's JSONL text; the writer adds the final newline.
*/
export function eventLine(event: SessionEvent): string {
return JSON.stringify(event)
export function eventLines(events: readonly SessionEvent[], packChunks: boolean): string {
const records: readonly StorageRecord[] = packChunks ? packChunkRuns(events) : events
return records.map(record => JSON.stringify(record)).join('\n')
}
/**
* Parse a JSONL log buffer into its preserved event prefix (the header is line
* 0). Fully written events in an interrupted final turn remain part of the
* 0). Event lines pass through verbatim; packed chunk rows expand back into
* their events, so callers see one contiguous event list regardless of layout.
* Fully written events in an interrupted final turn remain part of the
* prefix. The first unparsable record or seq gap after the last `turn/end`
* marks a tolerated torn tail; the same hole in the committed region rejects.
*
@@ -200,46 +210,60 @@ export function scanLog(buffer: Buffer): { meta: SessionHeader; events: SessionE
}
const headerLine = parsedHeader
// Parse every complete record first so the last valid `turn/end` determines
// whether an earlier hole is committed corruption or an uncommitted tail.
interface Parsed { ok: boolean; event?: SessionEvent; endByte: number }
// Parse and decode every complete line first so the last valid `turn/end`
// determines whether an earlier hole is committed corruption or an
// uncommitted tail. One line yields one event, or a whole run for a packed
// chunk row; a row-tagged line that fails row validation is a hole, exactly
// like unparsable JSON.
interface Parsed { ok: boolean; events?: SessionEvent[]; endByte: number }
const parsed: Parsed[] = eventEntries.map((entry) => {
try {
return { ok: true, event: JSON.parse(entry.text) as SessionEvent, endByte: entry.endByte }
return { ok: true, events: decodeStorageRecord(JSON.parse(entry.text)), endByte: entry.endByte }
} catch {
return { ok: false, endByte: entry.endByte }
}
})
// The last index (into eventEntries) that is a valid `turn/end` — holes
// through a closed turn are always committed corruption.
// The last index (into eventEntries) that ends in a valid `turn/end` — the
// last fully-committed boundary (the loop flushes only at turn/end). A packed
// row never stores a turn/end, so only single-event lines can match.
let lastTurnEnd = -1
for (let i = parsed.length - 1; i >= 0; i--) {
const p = parsed[i]
if (p?.ok && p.event?.type === 'turn/end') { lastTurnEnd = i; break }
if (p?.ok && p.events?.some(e => e.type === 'turn/end')) { lastTurnEnd = i; break }
}
// Preserve the contiguous prefix, including a complete interrupted turn;
// holes through the last committed boundary throw, while later holes stop.
// Contiguity is a cursor over seqs (not the line index): a packed row
// advances the cursor by its whole run.
const preserved: SessionEvent[] = []
for (let i = 0; i < parsed.length; i++) {
let lastPreservedLine = -1
scan: for (let i = 0; i < parsed.length; i++) {
const p = parsed[i]
if (!p?.ok || p.event === undefined) {
if (!p?.ok || p.events === undefined) {
if (i <= lastTurnEnd) throw new Error(`corrupt session log: unparsable committed event at line ${i + 1}`)
break // torn tail fragment after the last turn/end — stop, tolerate
}
if (p.event.seq !== i) {
if (i <= lastTurnEnd) throw new Error(`corrupt session log: seq gap in committed region at line ${i + 1} (expected ${i}, got ${p.event.seq})`)
break // gap after the last turn/end — torn tail, stop
for (const event of p.events) {
if (event.seq !== preserved.length) {
if (i <= lastTurnEnd) {
throw new Error(`corrupt session log: seq gap in committed region at line ${i + 1} (expected ${preserved.length}, got ${event.seq})`)
}
break scan // gap after the last turn/end — torn tail, stop
}
preserved.push(event)
}
preserved.push(p.event)
lastPreservedLine = i
}
// committedBytes = end of the last PRESERVED line (header if none): the next
// append truncates any torn bytes past this point before writing the
// synthetic closers + new events.
const lastPreserved = parsed[preserved.length - 1]
const committedBytes = preserved.length > 0 && lastPreserved ? lastPreserved.endByte : headerEntry.endByte
// committedBytes = end of the last FULLY preserved line (header if none): the
// next append truncates any torn bytes past this point before writing the
// synthetic closers + new events. A line is preserved whole or not at all —
// a mid-row seq gap discards the whole row, keeping the truncation offset on
// a line boundary.
const lastPreserved = parsed[lastPreservedLine]
const committedBytes = lastPreserved !== undefined ? lastPreserved.endByte : headerEntry.endByte
return { meta: fromHeaderLine(headerLine), events: preserved, committedBytes }
}

View File

@@ -17,7 +17,7 @@ import {
} from '@deepseek-ai/dsh-session-persistence'
import type { SessionEvent, SessionId, SessionHeader } from '@deepseek-ai/dsh-session'
import {
encodeSegment, eventLine, logPath, logSuffix, parseHeaderMeta, scanLog, sessionDir, toHeaderLine,
encodeSegment, eventLines, logPath, logSuffix, parseHeaderMeta, scanLog, sessionDir, toHeaderLine,
type JsonlCompression,
} from './format.ts'
import { compressZstdFrame, decompressZstdFrame, scanZstdFrames } from './zstd.ts'
@@ -33,7 +33,7 @@ export const JsonlCompressionSchema: z<JsonlCompression> = z.union([
z.const('none'),
]).default(DEFAULT_COMPRESSION)
/** Plugin config: where the JSONL backend keeps its session logs (`root` is required — no default). */
/** Plugin config: where the JSONL backend keeps its session logs, and the packed-row write switch. */
export interface Config {
/**
* Root directory for all session files. Required (no default): a default of
@@ -41,6 +41,15 @@ export interface Config {
* (bash calls, subprocesses). Sessions group under per-cwd subdirectories.
*/
root: string
/**
* Write runs of consecutive `assistant/chunk` delta events as packed
* `text-chunks`/`reasoning-chunks`/`tool-call-chunks` rows (lossless,
* ~60% smaller logs measured on a real session). Off by default while
* snapshot fixtures stay in the one-event-per-line layout: recording with
* packing on rewrites every golden `session.jsonl`. READING packed rows is
* unconditional — a log's layout never depends on this switch.
*/
packChunks?: boolean
/** Physical encoding; defaults to checksummed Zstandard frames. */
compression?: JsonlCompression
}
@@ -67,6 +76,7 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi
static Config: z<Config> = z.object({
root: z.string().required(),
packChunks: z.boolean().default(false),
compression: JsonlCompressionSchema,
})
@@ -78,6 +88,7 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi
override readonly name = 'session-persistence-jsonl'
private root: string
private packChunks: boolean
private compression: JsonlCompression
private coordinator: PersistenceCoordinator<JsonlTornMarker>
private rootEncodingCheck: Promise<void> | undefined
@@ -86,6 +97,9 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi
super(ctx)
// Resolve once so later process.cwd() changes cannot split one backend across roots.
this.root = resolve(config.root)
// schemastery (static Config) applied the default before construction;
// the cast records that runtime fact for exactOptionalPropertyTypes.
this.packChunks = (config as Required<Config>).packChunks
this.compression = config.compression ?? DEFAULT_COMPRESSION
this.coordinator = new PersistenceCoordinator<JsonlTornMarker>(this.ctx, this)
}
@@ -354,7 +368,7 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi
/** Encode the header and first batch without combining their frame boundaries. */
private async encodeMaterialization(meta: SessionHeader, events: readonly SessionEvent[]): Promise<Buffer | string> {
const header = JSON.stringify(toHeaderLine(meta)) + '\n'
const body = events.map(eventLine).join('\n') + '\n'
const body = eventLines(events, this.packChunks) + '\n'
if (this.compression === 'none') return header + body
const headerFrame = await compressZstdFrame(header)
const eventFrame = await compressZstdFrame(body)
@@ -363,7 +377,7 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi
/** Encode one durable append batch in the configured physical representation. */
private async encodeEventBatch(events: readonly SessionEvent[]): Promise<Buffer | string> {
const body = events.map(eventLine).join('\n') + '\n'
const body = eventLines(events, this.packChunks) + '\n'
return this.compression === 'zstd' ? compressZstdFrame(body) : body
}

View File

@@ -6,7 +6,7 @@ import { isAbsolute, join, relative, resolve } from 'node:path'
import SessionStore, { SessionId } from '@deepseek-ai/dsh-session'
import type { Session, SessionEvent, SessionHeader } from '@deepseek-ai/dsh-session'
import SessionPersistenceJsonl from '@deepseek-ai/dsh-session-persistence-jsonl'
import { encodeSegment, logPath, scanLog, sessionDir, toHeaderLine } from '../src/format.ts'
import { encodeSegment, eventLines, logPath, scanLog, sessionDir, toHeaderLine } from '../src/format.ts'
import { runPersistenceContract, meta, oneTurnLog, appendLog } from '../../session-persistence/tests/contract.ts'
import { runCoordinatorContract, type CoordinatorFixture } from '../../session-persistence/tests/coordinator-contract.ts'
@@ -554,6 +554,121 @@ describe('SessionPersistenceJsonl: scanLog unit', () => {
})
})
describe('SessionPersistenceJsonl: packed chunk rows (packChunks: true)', () => {
let ctx: Context
beforeEach(async () => {
root = await freshRoot()
ctx = new Context()
await ctx.plugin(SessionStore)
// compression: 'none' — these tests assert the textual storage-record layout
// (row tags per line); packing is orthogonal to the physical encoding.
await ctx.plugin(SessionPersistenceJsonl, { root, packChunks: true, compression: 'none' })
})
afterEach(async () => { await ctx.fiber.dispose() })
/** A one-turn log whose step streams a five-member text-delta run. */
function chunkRunLog(): SessionEvent[] {
const deltas: SessionEvent[] = Array.from({ length: 5 }, (_, k) => ({
type: 'assistant/chunk',
seq: 2 + k,
time: 3 + k,
data: { turn: 1, step: 1, chunk: { type: 'text-delta', index: 0, text: `t${k}` } },
}))
return [
{ type: 'turn/start', seq: 0, time: 1, data: { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } } },
{ type: 'step/start', seq: 1, time: 2, data: { turn: 1, step: 1 } },
...deltas,
{ type: 'assistant/message', seq: 7, time: 8, data: { turn: 1, step: 1, content: [{ type: 'text', text: 't0t1t2t3t4' }], provenance: { provider: 'mock', model: 'mock' } }, surfaceOp: 'append', sourceEventSeqs: [2, 3, 4, 5, 6] },
{ type: 'step/end', seq: 8, time: 9, data: { turn: 1, step: 1 } },
{ type: 'turn/end', seq: 9, time: 10, data: { turn: 1, reason: { kind: 'completed' } } },
]
}
it('writes a delta run as one text-chunks row and loads back identical events', async () => {
const m = meta('packed', '/work')
const log = chunkRunLog()
await ctx.sessionPersistence.create(m)
await ctx.sessionPersistence.append(m.id, log)
const raw = (await readFile(rawLogPath(root, '/work', m.id), 'utf8')).split('\n').filter(Boolean)
const tags = raw.slice(1).map(line => (JSON.parse(line) as { type: string }).type)
expect(tags).toEqual(['turn/start', 'step/start', 'text-chunks', 'assistant/message', 'step/end', 'turn/end'])
const loaded = await ctx.sessionPersistence.load(m.id)
expect(loaded.events).toEqual(log)
})
it('loads a mixed file: verbatim lines from an unpacked writer, then packed appends', async () => {
const m = meta('mixed', '/work')
const log = chunkRunLog()
// First turn written line-per-event by an unpacked-config writer (an old
// file, hand-planted so this packed-config backend adopts it on load).
await mkdir(sessionDir(root, '/work'), { recursive: true })
await writeFile(rawLogPath(root, '/work', m.id), [
JSON.stringify({ type: 'session', version: 0, id: 'mixed', createdAt: 1000, cwd: '/work', delegationDepth: 0 }),
...log.map(e => JSON.stringify(e)),
].join('\n') + '\n')
// Adopt the stored log (cursor = stored length), then append a second turn
// through THIS packed-config backend.
expect((await ctx.sessionPersistence.load(m.id)).events).toEqual(log)
const secondTurn: SessionEvent[] = JSON.parse(JSON.stringify(log)) as SessionEvent[]
for (const [k, e] of secondTurn.entries()) {
;(e as { seq: number }).seq = 10 + k
;(e.data as { turn: number }).turn = 2
}
await ctx.sessionPersistence.append(m.id, secondTurn)
const loaded = await ctx.sessionPersistence.load(m.id)
expect(loaded.events).toEqual([...log, ...secondTurn])
// The packed append really packed: the file's tail carries a text-chunks row.
const tags = (await readFile(rawLogPath(root, '/work', m.id), 'utf8')).split('\n').filter(Boolean)
.map(line => (JSON.parse(line) as { type: string }).type)
expect(tags.filter(t => t === 'text-chunks')).toHaveLength(1)
expect(tags.filter(t => t === 'assistant/chunk')).toHaveLength(5)
})
it('scanLog: a packed row advances the seq cursor by its whole run', () => {
const logText = [
JSON.stringify({ type: 'session', version: 0, id: 'rows', createdAt: 1, delegationDepth: 0 }),
JSON.stringify({ type: 'turn/start', seq: 0, time: 1, data: { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } } }),
JSON.stringify({ type: 'text-chunks', seq0: 1, time0: 2, data: { turn: 1, step: 1, index: 0, dt: [1, 1], texts: ['a', 'b', 'c'] } }),
JSON.stringify({ type: 'turn/end', seq: 4, time: 5, data: { turn: 1, reason: { kind: 'completed' } } }),
].join('\n') + '\n'
const { events } = scanLog(Buffer.from(logText))
expect(events.map(e => e.seq)).toEqual([0, 1, 2, 3, 4])
expect(events[2]).toEqual({ type: 'assistant/chunk', seq: 2, time: 3, data: { turn: 1, step: 1, chunk: { type: 'text-delta', index: 0, text: 'b' } } })
})
it('scanLog: a malformed packed row in the committed region rejects like corrupt JSON', () => {
const logText = [
JSON.stringify({ type: 'session', version: 0, id: 'bad-row', createdAt: 1, delegationDepth: 0 }),
// dt arity mismatch — row validation throws, so the line is a committed hole.
JSON.stringify({ type: 'text-chunks', seq0: 0, time0: 1, data: { turn: 1, step: 1, index: 0, dt: [], texts: ['a', 'b'] } }),
JSON.stringify({ type: 'turn/end', seq: 2, time: 3, data: { turn: 1, reason: { kind: 'completed' } } }),
].join('\n') + '\n'
expect(() => scanLog(Buffer.from(logText))).toThrow(/unparsable committed event/)
})
it('scanLog: a packed row with a mid-run seq gap after the last turn/end drops the whole row', () => {
const logText = [
JSON.stringify({ type: 'session', version: 0, id: 'row-gap', createdAt: 1, delegationDepth: 0 }),
JSON.stringify({ type: 'turn/start', seq: 0, time: 1, data: { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } } }),
// seq0 skips 1 — the run's first member is already a gap; no turn/end follows.
JSON.stringify({ type: 'text-chunks', seq0: 2, time0: 2, data: { turn: 1, step: 1, index: 0, dt: [1, 1], texts: ['a', 'b', 'c'] } }),
].join('\n') + '\n'
const scanned = scanLog(Buffer.from(logText))
expect(scanned.events.map(e => e.seq)).toEqual([0])
// committedBytes stays on the line boundary BEFORE the dropped row.
const headerAndTurn = logText.split('\n').slice(0, 2).join('\n') + '\n'
expect(scanned.committedBytes).toBe(Buffer.byteLength(headerAndTurn, 'utf8'))
})
it('eventLines(packChunks: false) is byte-identical to the pre-packing layout', () => {
const log = chunkRunLog()
expect(eventLines(log, false)).toBe(log.map(e => JSON.stringify(e)).join('\n'))
})
})
describe('SessionPersistenceJsonl: edge cases', () => {
let ctx: Context
beforeEach(async () => {

View File

@@ -7,7 +7,7 @@ import { join } from 'node:path'
import SessionStore, { SessionId } from '@deepseek-ai/dsh-session'
import type { SessionEvent } from '@deepseek-ai/dsh-session'
import SessionPersistenceJsonl from '@deepseek-ai/dsh-session-persistence-jsonl'
import { eventLine, logPath, scanLog, sessionDir, toHeaderLine, type JsonlCompression } from '../src/format.ts'
import { logPath, scanLog, sessionDir, toHeaderLine, type JsonlCompression } from '../src/format.ts'
import { compressZstdFrame, decompressZstdFrame, scanZstdFrames } from '../src/zstd.ts'
import { runPersistenceContract, meta, oneTurnLog } from '../../session-persistence/tests/contract.ts'
import { runCoordinatorContract, type CoordinatorFixture } from '../../session-persistence/tests/coordinator-contract.ts'
@@ -217,7 +217,7 @@ describe('SessionPersistenceJsonl: default Zstandard encoding', () => {
const plaintext = await decodeCompleteFrames(buffer)
expect(plaintext.toString()).toBe([
JSON.stringify(toHeaderLine(header)),
...oneTurnLog().map(eventLine),
...oneTurnLog().map(e => JSON.stringify(e)),
'',
].join('\n'))
expect((await ctx.sessionPersistence.load(header.id)).events).toEqual(oneTurnLog())
@@ -288,7 +288,7 @@ describe('SessionPersistenceJsonl: default Zstandard encoding', () => {
{ type: 'step/start', seq: 7, time: 8, data: { turn: 2, step: 1 } },
{ type: 'assistant/chunk', seq: 8, time: 9, data: { turn: 2, step: 1, chunk: { type: 'text-delta', index: 0, text: deterministicNoise(300_000) } } },
] as SessionEvent[]
const plaintext = openTurn.map(eventLine).join('\n') + '\n'
const plaintext = openTurn.map(e => JSON.stringify(e)).join('\n') + '\n'
const partial = await tornFrame(plaintext, (decoded) => {
const newlines = decoded.match(/\n/g)?.length ?? 0
return newlines >= 2 && !decoded.endsWith('\n')
@@ -334,7 +334,7 @@ describe('SessionPersistenceJsonl: default Zstandard encoding', () => {
{ type: 'turn/start', seq: 6, time: 7, data: { turn: 2, trigger: { kind: 'message', source: { kind: 'user' } } } },
{ type: 'turn/end', seq: 7, time: 8, data: { turn: 2, reason: { kind: 'completed' } } },
] as SessionEvent[]
const frame = await compressZstdFrame(secondTurn.map(eventLine).join('\n') + '\n')
const frame = await compressZstdFrame(secondTurn.map(e => JSON.stringify(e)).join('\n') + '\n')
await appendFile(path, frame.subarray(0, -1))
const loaded = await ctx.sessionPersistence.load(header.id)
@@ -456,7 +456,7 @@ describe('SessionPersistenceJsonl: encoding selection', () => {
await mkdir(sessionDir(root, loadHeader.cwd), { recursive: true })
await writeFile(logPath(root, loadHeader.cwd, loadHeader.id, 'none'), [
JSON.stringify(toHeaderLine(loadHeader)),
...oneTurnLog().map(eventLine),
...oneTurnLog().map(e => JSON.stringify(e)),
'',
].join('\n'))
await expect(ctx.sessionPersistence.load(loadHeader.id)).rejects.toThrow(/uses \.jsonl/)
@@ -474,7 +474,7 @@ describe('SessionPersistenceJsonl: encoding selection', () => {
await mkdir(sessionDir(root, header.cwd), { recursive: true })
await writeFile(logPath(root, header.cwd, header.id, 'none'), [
JSON.stringify(toHeaderLine(header)),
...oneTurnLog().map(eventLine),
...oneTurnLog().map(e => JSON.stringify(e)),
'',
].join('\n'))
await expect(ctx.sessionPersistence.append(header.id, oneTurnLog())).rejects.toThrow(/uses \.jsonl/)