Merge remote-tracking branch 'origin/master' into jsonl-packed-chunk-rows

Conflicts: the two generated catalog docs (regenerated over merged sources)
and the jsonl backend README storage-layout bullets — master's required
delegationDepth header field weaves with this branch's storage-record/packed
row wording.
This commit is contained in:
kingwl
2026-07-20 20:04:13 +08:00
138 changed files with 644 additions and 225 deletions

View File

@@ -10,7 +10,7 @@ The JSONL durable session-persistence backend — a concrete `SessionPersistence
<encoded-id>.jsonl # header line + one storage record per line
```
- The first `.jsonl` line is the immutable `SessionHeader` tagged `{ type: 'session', version, id, cwd?, createdAt, parentSession?, seedLength? }`; every subsequent line is one storage record. `assistant/chunk` events are never dropped, and `seq` stays contiguous across the decoded log.
- The first `.jsonl` line is the immutable `SessionHeader` tagged `{ type: 'session', version, id, cwd?, createdAt, parentSession?, seedLength?, delegationDepth }`. `delegationDepth` is required on disk and is `0` for a top-level session; a missing or invalid value rejects the log. Every subsequent line is one storage record; `assistant/chunk` events are never dropped, and `seq` stays contiguous across the decoded log (`events[i].seq === i`).
- A storage record is a `SessionEvent` JSON verbatim, or — written only under `packChunks` — a **packed chunk row** (`text-chunks` / `reasoning-chunks` / `tool-call-chunks`; bare slash-less tags like the header's `session`, so row tags cannot be confused with event types): one line holding a run of ≥3 consecutive same-block `assistant/chunk` delta events, `seq0`/`time0` plus per-member `dt` gaps reconstructing every member's `seq`/`time` exactly. The lossless codec lives in `@deepseek-ai/dsh-session` (`packChunkRuns`/`decodeStorageRecord`) and whitelists exact shapes — anything unrecognized stores verbatim. Reading is layout-blind: `load` always decodes rows, so packed, unpacked, and mixed files load identically.
- Session ids are unvalidated branded strings, so they are percent-encoded to a single safe path segment before use (no traversal, no collision).

View File

@@ -26,6 +26,7 @@ export interface HeaderLine {
cwd?: string
parentSession?: SessionId
seedLength?: number
delegationDepth: number
}
/**
@@ -42,6 +43,7 @@ export function toHeaderLine(header: SessionHeader): HeaderLine {
...header.cwd !== undefined ? { cwd: header.cwd } : {},
...header.parentSession !== undefined ? { parentSession: header.parentSession } : {},
...header.seedLength !== undefined ? { seedLength: header.seedLength } : {},
delegationDepth: header.delegationDepth ?? 0,
}
}
@@ -58,6 +60,7 @@ export function fromHeaderLine(line: HeaderLine): SessionHeader {
...line.cwd !== undefined ? { cwd: line.cwd } : {},
...line.parentSession !== undefined ? { parentSession: line.parentSession } : {},
...line.seedLength !== undefined ? { seedLength: line.seedLength } : {},
delegationDepth: line.delegationDepth,
}
}
@@ -69,6 +72,10 @@ function isHeaderLine(value: unknown): value is HeaderLine {
&& typeof (value as { version?: unknown }).version === 'number'
&& typeof (value as { id?: unknown }).id === 'string'
&& typeof (value as { createdAt?: unknown }).createdAt === 'number'
&& typeof (value as { delegationDepth?: unknown }).delegationDepth === 'number'
&& Number.isSafeInteger((value as { delegationDepth: number }).delegationDepth)
&& (value as { delegationDepth: number }).delegationDepth >= 0
&& !Object.is((value as { delegationDepth: number }).delegationDepth, -0)
)
}

View File

@@ -461,9 +461,30 @@ describe('SessionPersistenceJsonl: scanLog unit', () => {
expect(() => scanLog(Buffer.from('{"type":"event"}\n'))).toThrow(/session header/)
})
it.each([
['missing', undefined],
['a string', '1'],
['fractional', 1.5],
['negative', -1],
])('rejects a session header with %s delegationDepth', (_label, delegationDepth) => {
const log = JSON.stringify({
type: 'session',
version: 0,
id: 'invalid-depth',
createdAt: 1,
...delegationDepth === undefined ? {} : { delegationDepth },
}) + '\n'
expect(() => scanLog(Buffer.from(log))).toThrow(/session header/)
})
it('rejects a session header with negative-zero delegationDepth', () => {
const log = '{"type":"session","version":0,"id":"invalid-depth","createdAt":1,"delegationDepth":-0}\n'
expect(() => scanLog(Buffer.from(log))).toThrow(/session header/)
})
it('a seq gap after the last turn/end bounds the preserved tail (torn fragment tolerated)', () => {
const log = [
JSON.stringify({ type: 'session', version: 0, id: 'g', createdAt: 1 }),
JSON.stringify({ type: 'session', version: 0, id: 'g', createdAt: 1, delegationDepth: 0 }),
JSON.stringify({ type: 'turn/start', seq: 0, time: 1, data: { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } } }),
JSON.stringify({ type: 'step/start', seq: 2, time: 2, data: { turn: 1, step: 1 } }), // gap: missing seq 1
].join('\n') + '\n'
@@ -475,7 +496,7 @@ describe('SessionPersistenceJsonl: scanLog unit', () => {
it('rejects a seq gap BEFORE a later committed turn/end (committed data damaged)', () => {
const log = [
JSON.stringify({ type: 'session', version: 0, id: 'g2', createdAt: 1 }),
JSON.stringify({ type: 'session', version: 0, id: 'g2', createdAt: 1, delegationDepth: 0 }),
JSON.stringify({ type: 'turn/start', seq: 0, time: 1, data: { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } } }),
JSON.stringify({ type: 'step/start', seq: 2, time: 2, data: { turn: 1, step: 1 } }), // gap: missing seq 1
JSON.stringify({ type: 'turn/end', seq: 3, time: 3, data: { turn: 1, reason: { kind: 'completed' } } }),
@@ -487,7 +508,7 @@ describe('SessionPersistenceJsonl: scanLog unit', () => {
it('rejects a corrupt line BEFORE a later committed turn/end (committed data damaged)', () => {
const log = [
JSON.stringify({ type: 'session', version: 0, id: 'c', createdAt: 1 }),
JSON.stringify({ type: 'session', version: 0, id: 'c', createdAt: 1, delegationDepth: 0 }),
'{not json', // corrupt, sits in the committed region (a turn/end follows)
JSON.stringify({ type: 'turn/end', seq: 1, time: 2, data: { turn: 1, reason: { kind: 'completed' } } }),
].join('\n') + '\n'
@@ -495,7 +516,7 @@ describe('SessionPersistenceJsonl: scanLog unit', () => {
})
it('a header-only log (no event lines at all) preserves nothing — committedBytes is the header', () => {
const log = JSON.stringify({ type: 'session', version: 0, id: 'h0', createdAt: 1 }) + '\n'
const log = JSON.stringify({ type: 'session', version: 0, id: 'h0', createdAt: 1, delegationDepth: 0 }) + '\n'
const scanned = scanLog(Buffer.from(log))
expect(scanned.events).toEqual([])
// committedBytes falls back to the header line's end (no preserved events).
@@ -504,7 +525,7 @@ describe('SessionPersistenceJsonl: scanLog unit', () => {
it('a corrupt line after the last turn/end bounds the preserved tail', () => {
const log = [
JSON.stringify({ type: 'session', version: 0, id: 'c2', createdAt: 1 }),
JSON.stringify({ type: 'session', version: 0, id: 'c2', createdAt: 1, delegationDepth: 0 }),
JSON.stringify({ type: 'turn/start', seq: 0, time: 1, data: { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } } }),
'{not json', // corrupt crash fragment, no turn/end committed
].join('\n') + '\n'
@@ -515,7 +536,7 @@ describe('SessionPersistenceJsonl: scanLog unit', () => {
it('tolerates a seq gap AFTER a turn/end (uncommitted tail)', () => {
const log = [
JSON.stringify({ type: 'session', version: 0, id: 't', createdAt: 1 }),
JSON.stringify({ type: 'session', version: 0, id: 't', createdAt: 1, delegationDepth: 0 }),
JSON.stringify({ type: 'turn/start', seq: 0, time: 1, data: { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } } }),
JSON.stringify({ type: 'turn/end', seq: 1, time: 2, data: { turn: 1, reason: { kind: 'completed' } } }),
JSON.stringify({ type: 'step/start', seq: 9, time: 3, data: { turn: 2, step: 1 } }), // gap in uncommitted tail
@@ -706,7 +727,7 @@ describe('SessionPersistenceJsonl: edge cases', () => {
// `readFirstLine` accumulates chunks before `list()` parses it.
const bucket = join(root, '_no-cwd')
await mkdir(bucket, { recursive: true })
const bigHeader = JSON.stringify({ type: 'session', version: 0, id: 'big', createdAt: 1, pad: 'x'.repeat(9000) })
const bigHeader = JSON.stringify({ type: 'session', version: 0, id: 'big', createdAt: 1, delegationDepth: 0, pad: 'x'.repeat(9000) })
await writeFile(join(bucket, 'big.jsonl'), bigHeader + '\n')
const ids = (await ctx.sessionPersistence.list()).map(x => x.id)
expect(ids).toContain('big')