feat(session): opt-in packed chunk rows in the JSONL log
Providers stream token-sized deltas, so a session log stores hundreds of near-identical assistant/chunk lines whose JSON envelopes dwarf their payloads (~56x measured on a real DeepSeek session, 73% of file bytes). Add a lossless storage codec to dsh-session: packChunkRuns() folds each run of >=3 consecutive same-block delta chunks into one storage row -- text-chunks / reasoning-chunks / tool-call-chunks, bare slash-less tags like the header line's 'session' so rows cannot be confused with session events -- and decodeStorageRecord() expands rows back to the exact original events (seq0/time0 + dt gap array reconstruct every member's seq/time; tool-call rows carry the run-constant id/name). The encoder whitelists exact shapes and stores anything unrecognized verbatim; the decoder validates row-tagged values and fails loud on malformation. The JSONL backend gains a packChunks config (default false). Writing packs only when enabled -- default-off output stays byte-identical to the previous layout, so snapshot goldens are untouched. Reading is layout-blind: scanLog always decodes rows and now checks seq contiguity with a cursor instead of the line index, so packed, unpacked, and mixed files all load identically. Fixture readers (llm-replay parseSessionLog, acp-snapshot normalizeSessionLog) share the codec; the normalizer zeroes a row's time0/dt exactly like an event's time. The two demo bundles plumb packChunks from cordis.yml to the backend. Measured on a real coding session: 105 KB -> 42 KB (-60%), 475 lines -> 74, with reasoning/tool-call heavy sessions saving the most. Covered by example + fast-check round-trip codec tests, backend packed/mixed/torn- tail specs, and an end-to-end demo run loading a packed log through a default-config backend.
This commit is contained in:
@@ -50,6 +50,8 @@ export interface Config {
|
|||||||
tools?: ToolsConfig
|
tools?: ToolsConfig
|
||||||
/** Directory the JSONL session backend writes under. Defaults to `./.sessions`. */
|
/** Directory the JSONL session backend writes under. Defaults to `./.sessions`. */
|
||||||
persistenceRoot?: string
|
persistenceRoot?: string
|
||||||
|
/** Write delta-chunk runs as packed storage rows (the JSONL backend's `packChunks`). Defaults to `false`. */
|
||||||
|
packChunks?: boolean
|
||||||
/** Skill registry, local-provider, and model-facing consumer config forwarded to agent-spine-demo. */
|
/** Skill registry, local-provider, and model-facing consumer config forwarded to agent-spine-demo. */
|
||||||
skills?: agentCore.SkillConfig
|
skills?: agentCore.SkillConfig
|
||||||
}
|
}
|
||||||
@@ -417,7 +419,7 @@ export interface Config {
|
|||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
Source: [`packages/support/llm-replay/src/index.ts:306`](../packages/support/llm-replay/src/index.ts)
|
Source: [`packages/support/llm-replay/src/index.ts:308`](../packages/support/llm-replay/src/index.ts)
|
||||||
|
|
||||||
## `@deepseek-ai/dsh-mcp-client`
|
## `@deepseek-ai/dsh-mcp-client`
|
||||||
|
|
||||||
@@ -567,7 +569,7 @@ Source: [`packages/sandbox/sandbox-local/src/index.ts:20`](../packages/sandbox/s
|
|||||||
Requires: `sessions`
|
Requires: `sessions`
|
||||||
|
|
||||||
```ts config-catalog
|
```ts config-catalog
|
||||||
/** Plugin config: where the JSONL backend keeps its session logs (`root` is required — no default). */
|
/** Plugin config: where the JSONL backend keeps its session logs, and the packed-row write switch. */
|
||||||
export interface Config {
|
export interface Config {
|
||||||
/**
|
/**
|
||||||
* Root directory for all session files. Required (no default): a default of
|
* Root directory for all session files. Required (no default): a default of
|
||||||
@@ -575,6 +577,15 @@ export interface Config {
|
|||||||
* (bash calls, subprocesses). Sessions group under per-cwd subdirectories.
|
* (bash calls, subprocesses). Sessions group under per-cwd subdirectories.
|
||||||
*/
|
*/
|
||||||
root: string
|
root: string
|
||||||
|
/**
|
||||||
|
* Write runs of consecutive `assistant/chunk` delta events as packed
|
||||||
|
* `text-chunks`/`reasoning-chunks`/`tool-call-chunks` rows (lossless,
|
||||||
|
* ~60% smaller logs measured on a real session). Off by default while
|
||||||
|
* snapshot fixtures stay in the one-event-per-line layout: recording with
|
||||||
|
* packing on rewrites every golden `session.jsonl`. READING packed rows is
|
||||||
|
* unconditional — a log's layout never depends on this switch.
|
||||||
|
*/
|
||||||
|
packChunks?: boolean
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -699,6 +710,8 @@ export interface Config {
|
|||||||
tools?: ToolsConfig
|
tools?: ToolsConfig
|
||||||
/** Directory the JSONL session backend writes under. Defaults to `./.sessions`. */
|
/** Directory the JSONL session backend writes under. Defaults to `./.sessions`. */
|
||||||
persistenceRoot?: string
|
persistenceRoot?: string
|
||||||
|
/** Write delta-chunk runs as packed storage rows (the JSONL backend's `packChunks`). Defaults to `false`. */
|
||||||
|
packChunks?: boolean
|
||||||
/** stdin-chat banner printed once on start. Defaults to `'ready.'`. */
|
/** stdin-chat banner printed once on start. Defaults to `'ready.'`. */
|
||||||
welcome?: string
|
welcome?: string
|
||||||
/** Skill registry, local-provider, and model-facing consumer config forwarded to agent-spine-demo. */
|
/** Skill registry, local-provider, and model-facing consumer config forwarded to agent-spine-demo. */
|
||||||
|
|||||||
@@ -245,7 +245,7 @@ Creation announcement during session publication. A synchronous throw vetoes and
|
|||||||
'session/created'(this: Scoped<Session>, session: Session): void
|
'session/created'(this: Scoped<Session>, session: Session): void
|
||||||
```
|
```
|
||||||
|
|
||||||
Source: [`packages/core/session/src/index.ts:47`](../../packages/core/session/src/index.ts)
|
Source: [`packages/core/session/src/index.ts:49`](../../packages/core/session/src/index.ts)
|
||||||
|
|
||||||
### `session/disposed` — emit
|
### `session/disposed` — emit
|
||||||
|
|
||||||
@@ -255,7 +255,7 @@ Emitted once when an announced session leaves the store, including publication r
|
|||||||
'session/disposed'(this: Scoped<Session>, session: Session): void
|
'session/disposed'(this: Scoped<Session>, session: Session): void
|
||||||
```
|
```
|
||||||
|
|
||||||
Source: [`packages/core/session/src/index.ts:57`](../../packages/core/session/src/index.ts)
|
Source: [`packages/core/session/src/index.ts:59`](../../packages/core/session/src/index.ts)
|
||||||
|
|
||||||
### `session/event` — emit
|
### `session/event` — emit
|
||||||
|
|
||||||
@@ -267,7 +267,7 @@ Post-commit, fire-and-forget append feed. The listener snapshot resolves before
|
|||||||
|
|
||||||
Types: [SessionEvent](../core-data-structures/core.md)
|
Types: [SessionEvent](../core-data-structures/core.md)
|
||||||
|
|
||||||
Source: [`packages/core/session/src/index.ts:69`](../../packages/core/session/src/index.ts)
|
Source: [`packages/core/session/src/index.ts:71`](../../packages/core/session/src/index.ts)
|
||||||
|
|
||||||
### `session/flush` — parallel
|
### `session/flush` — parallel
|
||||||
|
|
||||||
@@ -277,7 +277,7 @@ Awaited parallel durability checkpoint: every listener runs and the caller await
|
|||||||
'session/flush'(this: Scoped<Session>, session: Session): Promise<void> | void
|
'session/flush'(this: Scoped<Session>, session: Session): Promise<void> | void
|
||||||
```
|
```
|
||||||
|
|
||||||
Source: [`packages/core/session/src/index.ts:79`](../../packages/core/session/src/index.ts)
|
Source: [`packages/core/session/src/index.ts:81`](../../packages/core/session/src/index.ts)
|
||||||
|
|
||||||
## `subagent/*`
|
## `subagent/*`
|
||||||
|
|
||||||
|
|||||||
@@ -200,7 +200,7 @@ list(): Session[]
|
|||||||
fork(source: SessionForkSource, boundary?: number, childSessionId?: SessionId): Session
|
fork(source: SessionForkSource, boundary?: number, childSessionId?: SessionId): Session
|
||||||
```
|
```
|
||||||
|
|
||||||
Source: [`packages/core/session/src/index.ts:564`](../../packages/core/session/src/index.ts)
|
Source: [`packages/core/session/src/index.ts:566`](../../packages/core/session/src/index.ts)
|
||||||
|
|
||||||
## `ctx.skills` — `SkillService`
|
## `ctx.skills` — `SkillService`
|
||||||
|
|
||||||
|
|||||||
@@ -305,6 +305,6 @@ The hook bridges' `hook/invoked` / `hook/result` provenance pairs (from `@deepse
|
|||||||
|
|
||||||
## Durability contract
|
## Durability contract
|
||||||
|
|
||||||
What a persistence backend relies on: the durable log persists every event verbatim, **including** `assistant/chunk` — `seq` must stay contiguous, so chunks cannot be filtered out of the canonical log. All `event.data` must be JSON-serializable; `Session.append` enforces this at the source (throwing on non-serializable data), so a bad event never enters the log and `session.events` always equals what a backend can persist. Adding an event type that carries non-serializable data, or that breaks the turn/step nesting the invariants plugin checks, is a breaking change to the on-disk format.
|
What a persistence backend relies on: the durable log persists every event losslessly, **including** `assistant/chunk` — `seq` must stay contiguous, so chunks cannot be filtered out of the canonical log. A backend may choose its own storage encoding for an event batch as long as `load` returns the exact appended events (the JSONL backend's opt-in packed chunk rows are such an encoding — see [persistence.md](persistence.md)). All `event.data` must be JSON-serializable; `Session.append` enforces this at the source (throwing on non-serializable data), so a bad event never enters the log and `session.events` always equals what a backend can persist. Adding an event type that carries non-serializable data, or that breaks the turn/step nesting the invariants plugin checks, is a breaking change to the on-disk format.
|
||||||
|
|
||||||
The backends that consume this contract are on [persistence.md](persistence.md).
|
The backends that consume this contract are on [persistence.md](persistence.md).
|
||||||
|
|||||||
@@ -25,10 +25,10 @@ This matrix shows which packages dispatch each harness-owned event and which pac
|
|||||||
| `fs/observed` | `emit` | [`packages/fs/fs/src/index.ts:68`](../packages/fs/fs/src/index.ts) | [`tool-fs`](../packages/fs/tool-fs) (`emit`) | [`fs-policy`](../packages/fs/fs-policy) |
|
| `fs/observed` | `emit` | [`packages/fs/fs/src/index.ts:68`](../packages/fs/fs/src/index.ts) | [`tool-fs`](../packages/fs/tool-fs) (`emit`) | [`fs-policy`](../packages/fs/fs-policy) |
|
||||||
| `fs/write-intent` | `waterfall` | [`packages/fs/fs/src/index.ts:51`](../packages/fs/fs/src/index.ts) | [`tool-fs`](../packages/fs/tool-fs) (`waterfall`) | [`fs-policy`](../packages/fs/fs-policy) |
|
| `fs/write-intent` | `waterfall` | [`packages/fs/fs/src/index.ts:51`](../packages/fs/fs/src/index.ts) | [`tool-fs`](../packages/fs/tool-fs) (`waterfall`) | [`fs-policy`](../packages/fs/fs-policy) |
|
||||||
| `llm/stream` | `waterfall` | [`packages/llm/llm/src/index.ts:39`](../packages/llm/llm/src/index.ts) | [`llm`](../packages/llm/llm) (`waterfall`) | [`invariants`](../packages/support/invariants), [`llm-replay`](../packages/support/llm-replay) |
|
| `llm/stream` | `waterfall` | [`packages/llm/llm/src/index.ts:39`](../packages/llm/llm/src/index.ts) | [`llm`](../packages/llm/llm) (`waterfall`) | [`invariants`](../packages/support/invariants), [`llm-replay`](../packages/support/llm-replay) |
|
||||||
| `session/created` | `emit` | [`packages/core/session/src/index.ts:47`](../packages/core/session/src/index.ts) | [`session`](../packages/core/session) (`events.dispatch`) | [`invariants`](../packages/support/invariants), [`jsonrpc`](../packages/ui/jsonrpc), [`session-persistence`](../packages/session-persistence/session-persistence) |
|
| `session/created` | `emit` | [`packages/core/session/src/index.ts:49`](../packages/core/session/src/index.ts) | [`session`](../packages/core/session) (`events.dispatch`) | [`invariants`](../packages/support/invariants), [`jsonrpc`](../packages/ui/jsonrpc), [`session-persistence`](../packages/session-persistence/session-persistence) |
|
||||||
| `session/disposed` | `emit` | [`packages/core/session/src/index.ts:57`](../packages/core/session/src/index.ts) | [`session`](../packages/core/session) (`events.dispatch`) | - |
|
| `session/disposed` | `emit` | [`packages/core/session/src/index.ts:59`](../packages/core/session/src/index.ts) | [`session`](../packages/core/session) (`events.dispatch`) | - |
|
||||||
| `session/event` | `emit` | [`packages/core/session/src/index.ts:69`](../packages/core/session/src/index.ts) | [`session`](../packages/core/session) (`events.dispatch`) | [`acp`](../packages/ui/acp), [`invariants`](../packages/support/invariants), [`jsonrpc`](../packages/ui/jsonrpc), [`session-persistence`](../packages/session-persistence/session-persistence), [`stdio`](../packages/ui/stdio) |
|
| `session/event` | `emit` | [`packages/core/session/src/index.ts:71`](../packages/core/session/src/index.ts) | [`session`](../packages/core/session) (`events.dispatch`) | [`acp`](../packages/ui/acp), [`invariants`](../packages/support/invariants), [`jsonrpc`](../packages/ui/jsonrpc), [`session-persistence`](../packages/session-persistence/session-persistence), [`stdio`](../packages/ui/stdio) |
|
||||||
| `session/flush` | `parallel` | [`packages/core/session/src/index.ts:79`](../packages/core/session/src/index.ts) | [`session`](../packages/core/session) (`events.dispatch`) | [`session-persistence`](../packages/session-persistence/session-persistence) |
|
| `session/flush` | `parallel` | [`packages/core/session/src/index.ts:81`](../packages/core/session/src/index.ts) | [`session`](../packages/core/session) (`events.dispatch`) | [`session-persistence`](../packages/session-persistence/session-persistence) |
|
||||||
| `subagent/end` | `emit` | [`packages/subagent/subagent/src/index.ts:92`](../packages/subagent/subagent/src/index.ts) | [`subagent`](../packages/subagent/subagent) (`events.dispatch`) | [`hooks-claude`](../packages/hooks/hooks-claude), [`jsonrpc`](../packages/ui/jsonrpc) |
|
| `subagent/end` | `emit` | [`packages/subagent/subagent/src/index.ts:92`](../packages/subagent/subagent/src/index.ts) | [`subagent`](../packages/subagent/subagent) (`events.dispatch`) | [`hooks-claude`](../packages/hooks/hooks-claude), [`jsonrpc`](../packages/ui/jsonrpc) |
|
||||||
| `subagent/provider-added` | `emit` | [`packages/subagent/subagent/src/index.ts:66`](../packages/subagent/subagent/src/index.ts) | [`subagent`](../packages/subagent/subagent) (`emit`) | [`tool-subagent`](../packages/subagent/tool-subagent) |
|
| `subagent/provider-added` | `emit` | [`packages/subagent/subagent/src/index.ts:66`](../packages/subagent/subagent/src/index.ts) | [`subagent`](../packages/subagent/subagent) (`emit`) | [`tool-subagent`](../packages/subagent/tool-subagent) |
|
||||||
| `subagent/provider-removed` | `emit` | [`packages/subagent/subagent/src/index.ts:72`](../packages/subagent/subagent/src/index.ts) | [`subagent`](../packages/subagent/subagent) (`events.dispatch`) | [`tool-subagent`](../packages/subagent/tool-subagent) |
|
| `subagent/provider-removed` | `emit` | [`packages/subagent/subagent/src/index.ts:72`](../packages/subagent/subagent/src/index.ts) | [`subagent`](../packages/subagent/subagent) (`events.dispatch`) | [`tool-subagent`](../packages/subagent/tool-subagent) |
|
||||||
|
|||||||
@@ -44,6 +44,10 @@ Plain class (not a Cordis Service). Create via `ctx.sessions.create()`.
|
|||||||
|
|
||||||
Durable values need one accepted representation, not a check followed by a second read. `isJsonValue(value)` is the boolean predicate; `snapshotJsonValue(value)` recursively validates and copies a plain value in one pass, returning `undefined` for invalid input and propagating a throwing getter. The snapshot helper accepts finite JSON numbers except `-0` (JSON rewrites it to `0`), dense ordinary arrays, and plain or null-prototype objects; it rejects cycles, unsupported scalars, and exotic prototypes before normalization.
|
Durable values need one accepted representation, not a check followed by a second read. `isJsonValue(value)` is the boolean predicate; `snapshotJsonValue(value)` recursively validates and copies a plain value in one pass, returning `undefined` for invalid input and propagating a throwing getter. The snapshot helper accepts finite JSON numbers except `-0` (JSON rewrites it to `0`), dense ordinary arrays, and plain or null-prototype objects; it rejects cycles, unsupported scalars, and exotic prototypes before normalization.
|
||||||
|
|
||||||
|
### Chunk-row storage codec (`chunk-rows.ts`)
|
||||||
|
|
||||||
|
Providers stream token-sized deltas, so a raw log stores hundreds of `assistant/chunk` lines whose JSON envelopes dwarf their payloads. `packChunkRuns(events)` packs each run of ≥3 consecutive same-block delta chunks into one storage row — `text-chunks`, `reasoning-chunks`, or `tool-call-chunks` (bare slash-less tags: storage vocabulary, not `SessionEventMap` members) — and `decodeStorageRecord(value)` expands a parsed line back into its exact events (`seq0`/`time0` + per-member `dt` gaps reconstruct every `seq`/`time`). The encoder whitelists exact shapes and stores anything unrecognized verbatim; the decoder validates row-tagged values and throws on malformation. Owned here so the JSONL backend and the fixture readers (`dsh-llm-replay`, `dsh-acp-snapshot`) share one codec; the write-side switch is the backend's `packChunks` config.
|
||||||
|
|
||||||
### Surface types
|
### Surface types
|
||||||
|
|
||||||
- `SurfaceOp` — how a surface node entered the linked list: `'append'` (normal tail append) or `{ op: 'replace', start, end }` (replace nodes from `start` through `end` inclusive — both must be valid surface node seqs; `start === end` replaces a single node). Used by compaction to shadow old nodes without deleting them.
|
- `SurfaceOp` — how a surface node entered the linked list: `'append'` (normal tail append) or `{ op: 'replace', start, end }` (replace nodes from `start` through `end` inclusive — both must be valid surface node seqs; `start === end` replaces a single node). Used by compaction to shadow old nodes without deleting them.
|
||||||
|
|||||||
326
packages/core/session/src/chunk-rows.ts
Normal file
326
packages/core/session/src/chunk-rows.ts
Normal file
@@ -0,0 +1,326 @@
|
|||||||
|
/**
|
||||||
|
* Lossless storage packing for `assistant/chunk` delta runs. Providers stream
|
||||||
|
* token-sized deltas, so a log stores hundreds of near-identical event lines
|
||||||
|
* whose JSON envelopes dwarf their payloads (~56× measured on a real DeepSeek
|
||||||
|
* session). This module packs each run of consecutive same-block delta chunks
|
||||||
|
* into ONE storage row — `text-chunks`, `reasoning-chunks`, or
|
||||||
|
* `tool-call-chunks` — and expands rows back to the exact original events.
|
||||||
|
*
|
||||||
|
* Storage rows are a durable-encoding vocabulary, NOT session events: they
|
||||||
|
* never enter `Session.events`, have no `SessionEventMap` entry, and use bare
|
||||||
|
* (slash-less) type tags so a reader cannot confuse them with the event
|
||||||
|
* taxonomy (precedent: the JSONL header line's `session` tag). The encoder
|
||||||
|
* whitelists exact shapes — anything it does not fully recognize is stored
|
||||||
|
* verbatim, so unknown fields or future chunk variants lose compression, never
|
||||||
|
* data. The decoder validates before expanding and fails loud on a malformed
|
||||||
|
* row-tagged value instead of silently dropping a whole run.
|
||||||
|
*
|
||||||
|
* @module @deepseek-ai/dsh-session/chunk-rows
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { CallId, assertNever } from '@deepseek-ai/dsh-llm'
|
||||||
|
import type { StreamChunk } from '@deepseek-ai/dsh-llm'
|
||||||
|
import type { SessionEvent } from './types.ts'
|
||||||
|
|
||||||
|
/** The chunk kinds that may pack; block boundaries, usage, and finish chunks always stay one event per line. */
|
||||||
|
type DeltaKind = 'text-delta' | 'reasoning-delta' | 'tool-call-delta'
|
||||||
|
|
||||||
|
/** A run member: an `assistant/chunk` event whose exact shape the encoder whitelisted. */
|
||||||
|
type DeltaEvent = SessionEvent<'assistant/chunk'>
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Fields shared by every packed run: placement, block correlation, and member
|
||||||
|
* timestamps as gaps. Member `k` reconstructs as seq `seq0 + k` and time
|
||||||
|
* `time0` plus the first `k` gaps; a gap may be negative when the wall clock
|
||||||
|
* stepped backwards between events.
|
||||||
|
*/
|
||||||
|
interface RunDataBase {
|
||||||
|
turn: number
|
||||||
|
step: number
|
||||||
|
/** The stream block index every member shares. */
|
||||||
|
index: number
|
||||||
|
/** Epoch-ms gaps between consecutive members; length is one less than the member count. */
|
||||||
|
dt: number[]
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Payload of a `text-chunks`/`reasoning-chunks` row: one entry per member, never joined — token boundaries are data. */
|
||||||
|
interface TextRunData extends RunDataBase {
|
||||||
|
texts: string[]
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Payload of a `tool-call-chunks` row: the run-constant call identity plus each member's raw arguments fragment. */
|
||||||
|
interface ToolCallRunData extends RunDataBase {
|
||||||
|
id: CallId
|
||||||
|
/** Present iff every member carried it, with one uniform value (a mixed run never packs). */
|
||||||
|
name?: string
|
||||||
|
args: string[]
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A packed run of consecutive delta chunk events, discriminated on `type`.
|
||||||
|
* `seq0`/`time0` anchor the first member; text and reasoning rows share the
|
||||||
|
* {@link TextRunData} payload, tool-call rows carry {@link ToolCallRunData}.
|
||||||
|
*/
|
||||||
|
export type ChunkRow =
|
||||||
|
| { type: 'text-chunks'; seq0: number; time0: number; data: TextRunData }
|
||||||
|
| { type: 'reasoning-chunks'; seq0: number; time0: number; data: TextRunData }
|
||||||
|
| { type: 'tool-call-chunks'; seq0: number; time0: number; data: ToolCallRunData }
|
||||||
|
|
||||||
|
/** One durable log line's JSON value: a session event verbatim, or a packed chunk row. */
|
||||||
|
export type StorageRecord = SessionEvent | ChunkRow
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Minimum members before a run packs. Below it a row's envelope rivals the
|
||||||
|
* event lines it replaces. A format constant, not a tunable: both layouts
|
||||||
|
* decode identically, so changing it never invalidates stored logs.
|
||||||
|
*/
|
||||||
|
const MIN_RUN = 3
|
||||||
|
|
||||||
|
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||||
|
return typeof value === 'object' && value !== null
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Exact-key check: `value` has every key in `keys` and nothing else. */
|
||||||
|
function hasExactKeys(value: object, keys: readonly string[]): boolean {
|
||||||
|
return Object.keys(value).length === keys.length && keys.every(k => Object.hasOwn(value, k))
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Classify an event for packing: its delta kind when the ENTIRE shape
|
||||||
|
* (envelope, data, chunk — exact keys, primitive types, integer seq/time) is
|
||||||
|
* whitelisted, else `undefined` (store verbatim). Inputs come from live typed
|
||||||
|
* appends AND parsed fixture files, so the checks are structural, not
|
||||||
|
* type-trusted. Integer times keep gap encoding exact: a fractional time would
|
||||||
|
* reconstruct through float subtraction/addition, which need not round-trip.
|
||||||
|
*/
|
||||||
|
function classify(event: SessionEvent): DeltaKind | undefined {
|
||||||
|
if (event.type !== 'assistant/chunk') return undefined
|
||||||
|
if (!hasExactKeys(event, ['type', 'seq', 'time', 'data'])) return undefined
|
||||||
|
if (!Number.isSafeInteger(event.seq) || event.seq < 0 || !Number.isSafeInteger(event.time)) return undefined
|
||||||
|
const data: unknown = event.data
|
||||||
|
if (!isRecord(data) || !hasExactKeys(data, ['turn', 'step', 'chunk'])) return undefined
|
||||||
|
if (typeof data.turn !== 'number' || typeof data.step !== 'number') return undefined
|
||||||
|
const chunk = data.chunk
|
||||||
|
if (!isRecord(chunk) || typeof chunk.index !== 'number') return undefined
|
||||||
|
switch (chunk.type) {
|
||||||
|
case 'text-delta':
|
||||||
|
case 'reasoning-delta':
|
||||||
|
return hasExactKeys(chunk, ['type', 'index', 'text']) && typeof chunk.text === 'string'
|
||||||
|
? chunk.type
|
||||||
|
: undefined
|
||||||
|
case 'tool-call-delta': {
|
||||||
|
const shapeOk = hasExactKeys(chunk, ['type', 'index', 'id', 'argumentsDelta'])
|
||||||
|
|| (hasExactKeys(chunk, ['type', 'index', 'id', 'name', 'argumentsDelta']) && typeof chunk.name === 'string')
|
||||||
|
return shapeOk && typeof chunk.id === 'string' && typeof chunk.argumentsDelta === 'string'
|
||||||
|
? chunk.type
|
||||||
|
: undefined
|
||||||
|
}
|
||||||
|
// Whitelist fall-through over parsed data: block-start/end, usage, finish,
|
||||||
|
// and any future chunk variant stay one event per line.
|
||||||
|
default:
|
||||||
|
return undefined
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The tool-call fields of a whitelisted delta chunk (only after {@link classify} returned `'tool-call-delta'`). */
|
||||||
|
function toolCallOf(event: DeltaEvent): { id: string; name?: string } {
|
||||||
|
return event.data.chunk as { id: string; name?: string }
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The block index of a whitelisted delta chunk (not every {@link StreamChunk} variant carries one). */
|
||||||
|
function indexOf(event: DeltaEvent): number {
|
||||||
|
return (event.data.chunk as { index: number }).index
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Whether `next` extends a run ending in `prev` (same kind already checked by the caller). */
|
||||||
|
function continues(prev: DeltaEvent, next: DeltaEvent, kind: DeltaKind): boolean {
|
||||||
|
if (next.seq !== prev.seq + 1) return false
|
||||||
|
if (next.data.turn !== prev.data.turn || next.data.step !== prev.data.step) return false
|
||||||
|
if (indexOf(next) !== indexOf(prev)) return false
|
||||||
|
if (kind !== 'tool-call-delta') return true
|
||||||
|
const a = toolCallOf(prev)
|
||||||
|
const b = toolCallOf(next)
|
||||||
|
// `name` must match in presence AND value — a mixed run is not representable.
|
||||||
|
return a.id === b.id && Object.hasOwn(a, 'name') === Object.hasOwn(b, 'name') && a.name === b.name
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Build the row for a completed run (`run.length >= MIN_RUN`, uniform per {@link continues}). */
|
||||||
|
function buildRow(kind: DeltaKind, run: readonly DeltaEvent[]): ChunkRow {
|
||||||
|
const first = run[0] as DeltaEvent
|
||||||
|
const base = {
|
||||||
|
turn: first.data.turn,
|
||||||
|
step: first.data.step,
|
||||||
|
index: indexOf(first),
|
||||||
|
dt: run.slice(1).map((event, i) => event.time - (run[i] as DeltaEvent).time),
|
||||||
|
}
|
||||||
|
const envelope = { seq0: first.seq, time0: first.time }
|
||||||
|
if (kind === 'tool-call-delta') {
|
||||||
|
const call = toolCallOf(first)
|
||||||
|
return {
|
||||||
|
type: 'tool-call-chunks',
|
||||||
|
...envelope,
|
||||||
|
data: {
|
||||||
|
...base,
|
||||||
|
id: CallId(call.id),
|
||||||
|
...Object.hasOwn(call, 'name') ? { name: call.name as string } : {},
|
||||||
|
args: run.map(event => (event.data.chunk as { argumentsDelta: string }).argumentsDelta),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const data = { ...base, texts: run.map(event => (event.data.chunk as { text: string }).text) }
|
||||||
|
return kind === 'text-delta'
|
||||||
|
? { type: 'text-chunks', ...envelope, data }
|
||||||
|
: { type: 'reasoning-chunks', ...envelope, data }
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Pack an event batch for storage: each run of at least {@link MIN_RUN}
|
||||||
|
* consecutive whitelisted same-kind, same-block delta chunk events becomes one
|
||||||
|
* {@link ChunkRow}; every other event passes through verbatim, in order.
|
||||||
|
* Pure and stateless — safe over any array, including a batch whose runs were
|
||||||
|
* split by flush boundaries (the split runs simply pack per batch).
|
||||||
|
*
|
||||||
|
* @param events - the batch to encode, in log order.
|
||||||
|
* @returns the storage records to write, one JSONL line each.
|
||||||
|
*/
|
||||||
|
export function packChunkRuns(events: readonly SessionEvent[]): StorageRecord[] {
|
||||||
|
const out: StorageRecord[] = []
|
||||||
|
let kind: DeltaKind | undefined
|
||||||
|
let run: DeltaEvent[] = []
|
||||||
|
const flush = (): void => {
|
||||||
|
if (kind !== undefined && run.length >= MIN_RUN) out.push(buildRow(kind, run))
|
||||||
|
else out.push(...run)
|
||||||
|
kind = undefined
|
||||||
|
run = []
|
||||||
|
}
|
||||||
|
for (const event of events) {
|
||||||
|
const k = classify(event)
|
||||||
|
if (k === undefined) {
|
||||||
|
flush()
|
||||||
|
out.push(event)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
const delta = event as DeltaEvent
|
||||||
|
const last = run[run.length - 1]
|
||||||
|
if (k === kind && last !== undefined && continues(last, delta, k)) {
|
||||||
|
run.push(delta)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
flush()
|
||||||
|
kind = k
|
||||||
|
run = [delta]
|
||||||
|
}
|
||||||
|
flush()
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Throw the uniform malformed-row diagnostic. */
|
||||||
|
function malformed(tag: string, why: string): never {
|
||||||
|
throw new Error(`malformed ${tag} storage row: ${why}`)
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Validate the shared run-data fields and the payload/dt arity. */
|
||||||
|
function validateRunData(tag: string, data: Record<string, unknown>, payloadKey: 'texts' | 'args'): void {
|
||||||
|
if (typeof data.turn !== 'number' || typeof data.step !== 'number' || typeof data.index !== 'number') {
|
||||||
|
malformed(tag, 'turn/step/index must be numbers')
|
||||||
|
}
|
||||||
|
const payload = data[payloadKey]
|
||||||
|
if (!Array.isArray(payload) || payload.length === 0 || payload.some(entry => typeof entry !== 'string')) {
|
||||||
|
malformed(tag, `${payloadKey} must be a non-empty string array`)
|
||||||
|
}
|
||||||
|
const dt = data.dt
|
||||||
|
if (!Array.isArray(dt) || dt.some(gap => typeof gap !== 'number' || !Number.isFinite(gap))) {
|
||||||
|
malformed(tag, 'dt must be an array of finite numbers')
|
||||||
|
}
|
||||||
|
if (dt.length !== payload.length - 1) {
|
||||||
|
malformed(tag, `dt length ${dt.length} does not match ${payload.length} members`)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Validate a row-tagged parsed value's envelope and data, throwing on any malformation. */
|
||||||
|
function validateRow(value: Record<string, unknown>, tag: ChunkRow['type']): ChunkRow {
|
||||||
|
if (!hasExactKeys(value, ['type', 'seq0', 'time0', 'data'])) {
|
||||||
|
malformed(tag, 'envelope must be exactly {type, seq0, time0, data}')
|
||||||
|
}
|
||||||
|
if (!Number.isSafeInteger(value.seq0) || (value.seq0 as number) < 0) {
|
||||||
|
malformed(tag, 'seq0 must be a non-negative safe integer')
|
||||||
|
}
|
||||||
|
if (typeof value.time0 !== 'number' || !Number.isFinite(value.time0)) {
|
||||||
|
malformed(tag, 'time0 must be a finite number')
|
||||||
|
}
|
||||||
|
const data = value.data
|
||||||
|
if (!isRecord(data)) malformed(tag, 'data must be an object')
|
||||||
|
if (tag === 'tool-call-chunks') {
|
||||||
|
const withName = hasExactKeys(data, ['turn', 'step', 'index', 'id', 'name', 'dt', 'args'])
|
||||||
|
if (!withName && !hasExactKeys(data, ['turn', 'step', 'index', 'id', 'dt', 'args'])) {
|
||||||
|
malformed(tag, 'data must be exactly {turn, step, index, id, name?, dt, args}')
|
||||||
|
}
|
||||||
|
if (typeof data.id !== 'string' || (withName && typeof data.name !== 'string')) {
|
||||||
|
malformed(tag, 'id (and name when present) must be strings')
|
||||||
|
}
|
||||||
|
validateRunData(tag, data, 'args')
|
||||||
|
} else {
|
||||||
|
if (!hasExactKeys(data, ['turn', 'step', 'index', 'dt', 'texts'])) {
|
||||||
|
malformed(tag, 'data must be exactly {turn, step, index, dt, texts}')
|
||||||
|
}
|
||||||
|
validateRunData(tag, data, 'texts')
|
||||||
|
}
|
||||||
|
return value as unknown as ChunkRow
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Expand a validated row back into its exact original events, in order. */
|
||||||
|
function expandRow(row: ChunkRow): SessionEvent[] {
|
||||||
|
const members = row.type === 'tool-call-chunks' ? row.data.args : row.data.texts
|
||||||
|
const events: SessionEvent[] = []
|
||||||
|
let time = row.time0
|
||||||
|
for (let k = 0; k < members.length; k++) {
|
||||||
|
if (k > 0) time += row.data.dt[k - 1] as number
|
||||||
|
let chunk: StreamChunk
|
||||||
|
switch (row.type) {
|
||||||
|
case 'text-chunks':
|
||||||
|
chunk = { type: 'text-delta', index: row.data.index, text: members[k] as string }
|
||||||
|
break
|
||||||
|
case 'reasoning-chunks':
|
||||||
|
chunk = { type: 'reasoning-delta', index: row.data.index, text: members[k] as string }
|
||||||
|
break
|
||||||
|
case 'tool-call-chunks':
|
||||||
|
chunk = {
|
||||||
|
type: 'tool-call-delta',
|
||||||
|
index: row.data.index,
|
||||||
|
id: row.data.id,
|
||||||
|
...Object.hasOwn(row.data, 'name') ? { name: row.data.name as string } : {},
|
||||||
|
argumentsDelta: members[k] as string,
|
||||||
|
}
|
||||||
|
break
|
||||||
|
/* v8 ignore next 2 -- validateRow only returns the three row tags */
|
||||||
|
default:
|
||||||
|
return assertNever(row, 'chunk-rows expandRow')
|
||||||
|
}
|
||||||
|
events.push({
|
||||||
|
type: 'assistant/chunk',
|
||||||
|
seq: row.seq0 + k,
|
||||||
|
time,
|
||||||
|
data: { turn: row.data.turn, step: row.data.step, chunk },
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return events
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Decode one parsed JSONL line value into the session event(s) it stores.
|
||||||
|
* Chunk-row-tagged values validate and expand (a malformed row throws — it is
|
||||||
|
* corrupt storage, and treating it as an event would silently drop a whole
|
||||||
|
* run); every other value passes through as a single event, unvalidated,
|
||||||
|
* exactly as readers treated event lines before packing existed.
|
||||||
|
*
|
||||||
|
* @param value - one line's `JSON.parse` result.
|
||||||
|
* @returns the stored events, in log order.
|
||||||
|
*/
|
||||||
|
export function decodeStorageRecord(value: unknown): SessionEvent[] {
|
||||||
|
if (!isRecord(value)) return [value as SessionEvent]
|
||||||
|
const tag = value.type
|
||||||
|
if (tag !== 'text-chunks' && tag !== 'reasoning-chunks' && tag !== 'tool-call-chunks') {
|
||||||
|
return [value as SessionEvent]
|
||||||
|
}
|
||||||
|
return expandRow(validateRow(value, tag))
|
||||||
|
}
|
||||||
@@ -22,6 +22,8 @@ export * from './types.ts'
|
|||||||
export { isJsonValue, snapshotJsonValue } from './json.ts'
|
export { isJsonValue, snapshotJsonValue } from './json.ts'
|
||||||
export type { JsonValue } from './json.ts'
|
export type { JsonValue } from './json.ts'
|
||||||
export { interruptedTurnClosers } from './repair.ts'
|
export { interruptedTurnClosers } from './repair.ts'
|
||||||
|
export { decodeStorageRecord, packChunkRuns } from './chunk-rows.ts'
|
||||||
|
export type { ChunkRow, StorageRecord } from './chunk-rows.ts'
|
||||||
export type { SurfaceFoldReplacement, SurfaceFoldResult, SurfaceNode } from './surface.ts'
|
export type { SurfaceFoldReplacement, SurfaceFoldResult, SurfaceNode } from './surface.ts'
|
||||||
export { foldSurface, isSurfaceEvent, isSurfaceEligibleType } from './surface.ts'
|
export { foldSurface, isSurfaceEvent, isSurfaceEligibleType } from './surface.ts'
|
||||||
export { isToolPairingBalanced } from './tool-pairing.ts'
|
export { isToolPairingBalanced } from './tool-pairing.ts'
|
||||||
|
|||||||
208
packages/core/session/tests/chunk-rows.spec.ts
Normal file
208
packages/core/session/tests/chunk-rows.spec.ts
Normal file
@@ -0,0 +1,208 @@
|
|||||||
|
/**
|
||||||
|
* Chunk-row codec tests: pack/expand round-trip losslessness (example-based and
|
||||||
|
* property-based), run-boundary rules, whitelist fall-through, and decoder
|
||||||
|
* validation failures.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { describe, expect, it } from 'vitest'
|
||||||
|
import fc from 'fast-check'
|
||||||
|
import { CallId } from '@deepseek-ai/dsh-llm'
|
||||||
|
import type { StreamChunk } from '@deepseek-ai/dsh-llm'
|
||||||
|
import { decodeStorageRecord, packChunkRuns } from '@deepseek-ai/dsh-session'
|
||||||
|
import type { ChunkRow, SessionEvent, StorageRecord } from '@deepseek-ai/dsh-session'
|
||||||
|
|
||||||
|
/** Build an `assistant/chunk` event with the exact live-append shape. */
|
||||||
|
function chunkEvent(seq: number, time: number, chunk: StreamChunk, turn = 1, step = 1): SessionEvent {
|
||||||
|
return { type: 'assistant/chunk', seq, time, data: { turn, step, chunk } }
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Sequential delta events (contiguous seqs, fixed 10ms gaps) of one kind. */
|
||||||
|
function deltaRun(kind: 'text-delta' | 'reasoning-delta', count: number, seq0 = 0, index = 0): SessionEvent[] {
|
||||||
|
return Array.from({ length: count }, (_, k) =>
|
||||||
|
chunkEvent(seq0 + k, 1000 + 10 * k, { type: kind, index, text: `t${k}` }))
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Decode a packed record list back to a flat event list. */
|
||||||
|
function decodeAll(records: readonly StorageRecord[]): SessionEvent[] {
|
||||||
|
return records.flatMap(record => decodeStorageRecord(JSON.parse(JSON.stringify(record))))
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('packChunkRuns', () => {
|
||||||
|
it('packs a text-delta run into one text-chunks row and round-trips it', () => {
|
||||||
|
const events = deltaRun('text-delta', 5)
|
||||||
|
const packed = packChunkRuns(events)
|
||||||
|
expect(packed).toHaveLength(1)
|
||||||
|
const row = packed[0] as ChunkRow
|
||||||
|
expect(row.type).toBe('text-chunks')
|
||||||
|
expect(row.seq0).toBe(0)
|
||||||
|
expect(row.time0).toBe(1000)
|
||||||
|
expect(row.data).toMatchObject({ turn: 1, step: 1, index: 0, dt: [10, 10, 10, 10], texts: ['t0', 't1', 't2', 't3', 't4'] })
|
||||||
|
expect(decodeAll(packed)).toStrictEqual(events)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('packs reasoning and tool-call runs under their own tags', () => {
|
||||||
|
const reasoning = deltaRun('reasoning-delta', 3)
|
||||||
|
const toolCall = [4, 5, 6].map(seq =>
|
||||||
|
chunkEvent(seq, 1000 + seq, { type: 'tool-call-delta', index: 1, id: CallId('c1'), name: 'write', argumentsDelta: `a${seq}` }))
|
||||||
|
const packed = packChunkRuns([...reasoning, ...toolCall])
|
||||||
|
expect(packed.map(r => (r as ChunkRow).type)).toStrictEqual(['reasoning-chunks', 'tool-call-chunks'])
|
||||||
|
const row = packed[1] as ChunkRow & { type: 'tool-call-chunks' }
|
||||||
|
expect(row.data).toMatchObject({ id: 'c1', name: 'write', args: ['a4', 'a5', 'a6'] })
|
||||||
|
expect(decodeAll(packed)).toStrictEqual([...reasoning, ...toolCall])
|
||||||
|
})
|
||||||
|
|
||||||
|
it('packs a name-less tool-call run and round-trips field absence', () => {
|
||||||
|
const events = [0, 1, 2].map(seq =>
|
||||||
|
chunkEvent(seq, 1000, { type: 'tool-call-delta', index: 0, id: CallId('c1'), argumentsDelta: `a${seq}` }))
|
||||||
|
const packed = packChunkRuns(events)
|
||||||
|
expect(packed).toHaveLength(1)
|
||||||
|
expect(Object.hasOwn((packed[0] as ChunkRow).data, 'name')).toBe(false)
|
||||||
|
const decoded = decodeAll(packed)
|
||||||
|
expect(decoded).toStrictEqual(events)
|
||||||
|
expect(decoded.every(e => !Object.hasOwn((e.data as { chunk: object }).chunk, 'name'))).toBe(true)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('leaves runs shorter than three events verbatim', () => {
|
||||||
|
const events = deltaRun('text-delta', 2)
|
||||||
|
expect(packChunkRuns(events)).toStrictEqual(events)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('leaves non-delta chunks and non-chunk events verbatim between runs', () => {
|
||||||
|
const events: SessionEvent[] = [
|
||||||
|
chunkEvent(0, 1000, { type: 'block-start', index: 0, blockType: 'text' }),
|
||||||
|
...deltaRun('text-delta', 3, 1),
|
||||||
|
chunkEvent(4, 1040, { type: 'block-end', index: 0, block: { type: 'text', text: 't0t1t2' } }),
|
||||||
|
{ type: 'step/end', seq: 5, time: 1050, data: { turn: 1, step: 1 } },
|
||||||
|
]
|
||||||
|
const packed = packChunkRuns(events)
|
||||||
|
expect(packed).toHaveLength(4)
|
||||||
|
expect((packed[1] as ChunkRow).type).toBe('text-chunks')
|
||||||
|
expect(decodeAll(packed)).toStrictEqual(events)
|
||||||
|
})
|
||||||
|
|
||||||
|
it.each([
|
||||||
|
['a seq gap', deltaRun('text-delta', 3).map((e, k) => ({ ...e, seq: k === 2 ? 9 : e.seq }))],
|
||||||
|
['a kind switch', [...deltaRun('text-delta', 2), ...deltaRun('reasoning-delta', 1, 2)]],
|
||||||
|
['a block-index switch', [...deltaRun('text-delta', 2), ...deltaRun('text-delta', 1, 2, 7)]],
|
||||||
|
['a step switch', deltaRun('text-delta', 3).map((e, k) => k === 2 ? chunkEvent(e.seq, e.time, (e.data as { chunk: StreamChunk }).chunk, 1, 2) : e)],
|
||||||
|
])('breaks a run on %s (both halves too short to pack)', (_label, events) => {
|
||||||
|
expect(packChunkRuns(events as SessionEvent[])).toStrictEqual(events)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('breaks a tool-call run on call-id or name change', () => {
|
||||||
|
const call = (seq: number, id: string, name?: string): SessionEvent =>
|
||||||
|
chunkEvent(seq, 1000, { type: 'tool-call-delta', index: 0, id: CallId(id), ...name !== undefined ? { name } : {}, argumentsDelta: 'a' })
|
||||||
|
const idSwitch = [call(0, 'c1', 'w'), call(1, 'c1', 'w'), call(2, 'c2', 'w')]
|
||||||
|
expect(packChunkRuns(idSwitch)).toStrictEqual(idSwitch)
|
||||||
|
const namePresence = [call(0, 'c1', 'w'), call(1, 'c1', 'w'), call(2, 'c1')]
|
||||||
|
expect(packChunkRuns(namePresence)).toStrictEqual(namePresence)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('stores an off-whitelist delta verbatim (extra field, bad type, fractional time)', () => {
|
||||||
|
const extraField = { ...chunkEvent(0, 1000, { type: 'text-delta', index: 0, text: 'x' }), surfaceOp: 'append' }
|
||||||
|
const badText = chunkEvent(1, 1001, { type: 'text-delta', index: 0, text: 7 as unknown as string })
|
||||||
|
const fractionalTime = chunkEvent(2, 1001.5, { type: 'text-delta', index: 0, text: 'y' })
|
||||||
|
const events = [extraField, badText, fractionalTime] as SessionEvent[]
|
||||||
|
expect(packChunkRuns(events)).toStrictEqual(events)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('stores a delta with an off-whitelist data envelope verbatim (parsed-fixture shapes)', () => {
|
||||||
|
const mk = (seq: number, data: unknown): SessionEvent =>
|
||||||
|
({ type: 'assistant/chunk', seq, time: 1000, data } as SessionEvent)
|
||||||
|
const events = [
|
||||||
|
mk(0, 'not-an-object'),
|
||||||
|
mk(1, { turn: 1, step: 1, chunk: { type: 'text-delta', index: 0, text: 'a' }, extra: 1 }),
|
||||||
|
mk(2, { turn: 'x', step: 1, chunk: { type: 'text-delta', index: 0, text: 'a' } }),
|
||||||
|
mk(3, { turn: 1, step: 1, chunk: 'not-an-object' }),
|
||||||
|
mk(4, { turn: 1, step: 1, chunk: { type: 'text-delta', index: 'x', text: 'a' } }),
|
||||||
|
mk(5, { turn: 1, step: 1, chunk: { type: 'tool-call-delta', index: 0, id: 7, argumentsDelta: 'a' } }),
|
||||||
|
mk(6, { turn: 1, step: 1, chunk: { type: 'tool-call-delta', index: 0, id: 'c', name: 7, argumentsDelta: 'a' } }),
|
||||||
|
]
|
||||||
|
expect(packChunkRuns(events)).toStrictEqual(events)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('decodeStorageRecord', () => {
|
||||||
|
it('passes non-row values through as single events, unvalidated', () => {
|
||||||
|
const event = { type: 'turn/start', seq: 0, time: 1, data: { turn: 1 } }
|
||||||
|
expect(decodeStorageRecord(event)).toStrictEqual([event])
|
||||||
|
expect(decodeStorageRecord('junk')).toStrictEqual(['junk'])
|
||||||
|
expect(decodeStorageRecord(null)).toStrictEqual([null])
|
||||||
|
})
|
||||||
|
|
||||||
|
it('reconstructs timestamps through negative dt gaps (clock stepped back)', () => {
|
||||||
|
const events = [
|
||||||
|
chunkEvent(0, 1000, { type: 'text-delta', index: 0, text: 'a' }),
|
||||||
|
chunkEvent(1, 990, { type: 'text-delta', index: 0, text: 'b' }),
|
||||||
|
chunkEvent(2, 995, { type: 'text-delta', index: 0, text: 'c' }),
|
||||||
|
]
|
||||||
|
expect(decodeAll(packChunkRuns(events))).toStrictEqual(events)
|
||||||
|
})
|
||||||
|
|
||||||
|
it.each([
|
||||||
|
['a non-object data', { type: 'text-chunks', seq0: 0, time0: 1, data: 'x' }],
|
||||||
|
['an envelope with extra keys', { type: 'text-chunks', seq0: 0, time0: 1, data: { turn: 1, step: 1, index: 0, dt: [], texts: ['a'] }, extra: 1 }],
|
||||||
|
['a negative seq0', { type: 'text-chunks', seq0: -1, time0: 1, data: { turn: 1, step: 1, index: 0, dt: [], texts: ['a'] } }],
|
||||||
|
['a non-finite time0', { type: 'text-chunks', seq0: 0, time0: Infinity, data: { turn: 1, step: 1, index: 0, dt: [], texts: ['a'] } }],
|
||||||
|
['a data shape mismatch', { type: 'text-chunks', seq0: 0, time0: 1, data: { turn: 1, step: 1, index: 0, dt: [], args: ['a'] } }],
|
||||||
|
['a non-string member', { type: 'text-chunks', seq0: 0, time0: 1, data: { turn: 1, step: 1, index: 0, dt: [], texts: [7] } }],
|
||||||
|
['an empty member list', { type: 'text-chunks', seq0: 0, time0: 1, data: { turn: 1, step: 1, index: 0, dt: [], texts: [] } }],
|
||||||
|
['a dt arity mismatch', { type: 'text-chunks', seq0: 0, time0: 1, data: { turn: 1, step: 1, index: 0, dt: [1, 2], texts: ['a', 'b'] } }],
|
||||||
|
['a non-finite dt gap', { type: 'text-chunks', seq0: 0, time0: 1, data: { turn: 1, step: 1, index: 0, dt: [NaN], texts: ['a', 'b'] } }],
|
||||||
|
['a non-numeric turn', { type: 'text-chunks', seq0: 0, time0: 1, data: { turn: 'x', step: 1, index: 0, dt: [], texts: ['a'] } }],
|
||||||
|
['a tool-call row without id', { type: 'tool-call-chunks', seq0: 0, time0: 1, data: { turn: 1, step: 1, index: 0, dt: [], args: ['a'] } }],
|
||||||
|
['a tool-call row with non-string id', { type: 'tool-call-chunks', seq0: 0, time0: 1, data: { turn: 1, step: 1, index: 0, id: 7, dt: [], args: ['a'] } }],
|
||||||
|
['a tool-call row with non-string name', { type: 'tool-call-chunks', seq0: 0, time0: 1, data: { turn: 1, step: 1, index: 0, id: 'c', name: 7, dt: [], args: ['a'] } }],
|
||||||
|
])('throws on %s', (_label, row) => {
|
||||||
|
expect(() => decodeStorageRecord(row)).toThrow(/malformed .* storage row/)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
// --- Property: pack∘decode is the identity over arbitrary event batches ---
|
||||||
|
|
||||||
|
const deltaChunkArb: fc.Arbitrary<StreamChunk> = fc.oneof(
|
||||||
|
fc.record({ type: fc.constant<'text-delta'>('text-delta'), index: fc.nat(2), text: fc.string() }),
|
||||||
|
fc.record({ type: fc.constant<'reasoning-delta'>('reasoning-delta'), index: fc.nat(2), text: fc.string() }),
|
||||||
|
fc.record({
|
||||||
|
type: fc.constant<'tool-call-delta'>('tool-call-delta'),
|
||||||
|
index: fc.nat(2),
|
||||||
|
id: fc.constantFrom(CallId('c1'), CallId('c2')),
|
||||||
|
argumentsDelta: fc.string(),
|
||||||
|
}),
|
||||||
|
fc.record({
|
||||||
|
type: fc.constant<'tool-call-delta'>('tool-call-delta'),
|
||||||
|
index: fc.nat(2),
|
||||||
|
id: fc.constantFrom(CallId('c1'), CallId('c2')),
|
||||||
|
name: fc.constantFrom('write', 'read'),
|
||||||
|
argumentsDelta: fc.string(),
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
|
||||||
|
const boundaryChunkArb: fc.Arbitrary<StreamChunk> = fc.oneof(
|
||||||
|
fc.record({ type: fc.constant<'block-start'>('block-start'), index: fc.nat(2), blockType: fc.constant<'text'>('text') }),
|
||||||
|
fc.record({ type: fc.constant<'finish'>('finish'), reason: fc.constant({ kind: 'stop' as const }) }),
|
||||||
|
)
|
||||||
|
|
||||||
|
/** Batches with contiguous seqs, arbitrary gaps in time, mixed chunk kinds and turn/step placement. */
|
||||||
|
const batchArb: fc.Arbitrary<SessionEvent[]> = fc.array(
|
||||||
|
fc.record({
|
||||||
|
chunk: fc.oneof({ weight: 4, arbitrary: deltaChunkArb }, { weight: 1, arbitrary: boundaryChunkArb }),
|
||||||
|
gap: fc.integer({ min: -5, max: 200 }),
|
||||||
|
turn: fc.nat(1),
|
||||||
|
step: fc.nat(1),
|
||||||
|
}),
|
||||||
|
{ maxLength: 40 },
|
||||||
|
// JSON round-trip normalizes fast-check's null-prototype records into the
|
||||||
|
// plain objects real log events are (the log is JSON), so equality compares
|
||||||
|
// values, not prototypes.
|
||||||
|
).map(entries => JSON.parse(JSON.stringify(
|
||||||
|
entries.map((entry, k) => chunkEvent(k, 1000 + entry.gap * k, entry.chunk, entry.turn, entry.step)),
|
||||||
|
)) as SessionEvent[])
|
||||||
|
|
||||||
|
describe('chunk-row codec properties', () => {
|
||||||
|
it('JSON-serialized pack∘decode reproduces every batch exactly', () => {
|
||||||
|
fc.assert(fc.property(batchArb, (events) => {
|
||||||
|
expect(decodeAll(packChunkRuns(events))).toStrictEqual(events)
|
||||||
|
}))
|
||||||
|
})
|
||||||
|
})
|
||||||
@@ -31,6 +31,7 @@ Because the package wires no logger entry, an ACP leaf has **nothing to get wron
|
|||||||
| `tools` | `{ mode: 'native' }` | tool-registry presentation config (`native` / `code` / `both`), routed through `dsh-agent-spine-demo` |
|
| `tools` | `{ mode: 'native' }` | tool-registry presentation config (`native` / `code` / `both`), routed through `dsh-agent-spine-demo` |
|
||||||
| `skills` | owner defaults | registry-cache, local-provider, and model-facing skill-tool config, routed through `dsh-agent-spine-demo` |
|
| `skills` | owner defaults | registry-cache, local-provider, and model-facing skill-tool config, routed through `dsh-agent-spine-demo` |
|
||||||
| `persistenceRoot` | `./.sessions` | the JSONL backend's root directory |
|
| `persistenceRoot` | `./.sessions` | the JSONL backend's root directory |
|
||||||
|
| `packChunks` | `false` | write delta-chunk runs as packed storage rows (the JSONL backend's `packChunks`) |
|
||||||
|
|
||||||
The leaf supplies the swappable backends: an LLM adapter (`llm-deepseek` for the real model, `llm-replay` for keyless snapshot replay) and a bash executor.
|
The leaf supplies the swappable backends: an LLM adapter (`llm-deepseek` for the real model, `llm-replay` for keyless snapshot replay) and a bash executor.
|
||||||
|
|
||||||
|
|||||||
@@ -39,6 +39,8 @@ export interface Config {
|
|||||||
tools?: ToolsConfig
|
tools?: ToolsConfig
|
||||||
/** Directory the JSONL session backend writes under. Defaults to `./.sessions`. */
|
/** Directory the JSONL session backend writes under. Defaults to `./.sessions`. */
|
||||||
persistenceRoot?: string
|
persistenceRoot?: string
|
||||||
|
/** Write delta-chunk runs as packed storage rows (the JSONL backend's `packChunks`). Defaults to `false`. */
|
||||||
|
packChunks?: boolean
|
||||||
/** Skill registry, local-provider, and model-facing consumer config forwarded to agent-spine-demo. */
|
/** Skill registry, local-provider, and model-facing consumer config forwarded to agent-spine-demo. */
|
||||||
skills?: agentCore.SkillConfig
|
skills?: agentCore.SkillConfig
|
||||||
}
|
}
|
||||||
@@ -57,6 +59,7 @@ export const Config: z<Config> = z.object({
|
|||||||
// TODO(single-default-literal): share this schema default and the defensive
|
// TODO(single-default-literal): share this schema default and the defensive
|
||||||
// apply() fallback through one named constant while retaining both boundaries.
|
// apply() fallback through one named constant while retaining both boundaries.
|
||||||
persistenceRoot: z.string().default('./.sessions'),
|
persistenceRoot: z.string().default('./.sessions'),
|
||||||
|
packChunks: z.boolean().default(false),
|
||||||
skills: agentCore.SkillConfigSchema,
|
skills: agentCore.SkillConfigSchema,
|
||||||
})
|
})
|
||||||
/* jscpd:ignore-end */
|
/* jscpd:ignore-end */
|
||||||
@@ -76,6 +79,9 @@ export function apply(ctx: Context, config: Config): void {
|
|||||||
...config.skills !== undefined ? { skills: config.skills } : {},
|
...config.skills !== undefined ? { skills: config.skills } : {},
|
||||||
})
|
})
|
||||||
ctx.plugin(UserInteractionService)
|
ctx.plugin(UserInteractionService)
|
||||||
ctx.plugin(SessionPersistenceJsonl, { root: config.persistenceRoot ?? './.sessions' })
|
ctx.plugin(SessionPersistenceJsonl, {
|
||||||
|
root: config.persistenceRoot ?? './.sessions',
|
||||||
|
...config.packChunks !== undefined ? { packChunks: config.packChunks } : {},
|
||||||
|
})
|
||||||
ctx.plugin(acp, { model: config.model })
|
ctx.plugin(acp, { model: config.model })
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -31,6 +31,7 @@ The leaf `cordis.yml` supplies only the **swappable backends** — an LLM adapte
|
|||||||
| `tools` | `{ mode: 'native' }` | tool-registry presentation config (`native` / `code` / `both`), routed through `dsh-agent-spine-demo` |
|
| `tools` | `{ mode: 'native' }` | tool-registry presentation config (`native` / `code` / `both`), routed through `dsh-agent-spine-demo` |
|
||||||
| `skills` | owner defaults | registry-cache, local-provider, and model-facing skill-tool config, routed through `dsh-agent-spine-demo` |
|
| `skills` | owner defaults | registry-cache, local-provider, and model-facing skill-tool config, routed through `dsh-agent-spine-demo` |
|
||||||
| `persistenceRoot` | `./.sessions` | the JSONL backend's root directory |
|
| `persistenceRoot` | `./.sessions` | the JSONL backend's root directory |
|
||||||
|
| `packChunks` | `false` | write delta-chunk runs as packed storage rows (the JSONL backend's `packChunks`) |
|
||||||
| `welcome` | `ready.` | the stdin-chat banner |
|
| `welcome` | `ready.` | the stdin-chat banner |
|
||||||
| `resumeSessionId` | — | resume a persisted session id instead of starting fresh (sourced from an env var in the leaf) |
|
| `resumeSessionId` | — | resume a persisted session id instead of starting fresh (sourced from an env var in the leaf) |
|
||||||
|
|
||||||
|
|||||||
@@ -44,6 +44,8 @@ export interface Config {
|
|||||||
tools?: ToolsConfig
|
tools?: ToolsConfig
|
||||||
/** Directory the JSONL session backend writes under. Defaults to `./.sessions`. */
|
/** Directory the JSONL session backend writes under. Defaults to `./.sessions`. */
|
||||||
persistenceRoot?: string
|
persistenceRoot?: string
|
||||||
|
/** Write delta-chunk runs as packed storage rows (the JSONL backend's `packChunks`). Defaults to `false`. */
|
||||||
|
packChunks?: boolean
|
||||||
/** stdin-chat banner printed once on start. Defaults to `'ready.'`. */
|
/** stdin-chat banner printed once on start. Defaults to `'ready.'`. */
|
||||||
welcome?: string
|
welcome?: string
|
||||||
/** Skill registry, local-provider, and model-facing consumer config forwarded to agent-spine-demo. */
|
/** Skill registry, local-provider, and model-facing consumer config forwarded to agent-spine-demo. */
|
||||||
@@ -67,6 +69,7 @@ export const Config: z<Config> = z.object({
|
|||||||
// TODO(single-default-literal): share these schema defaults and defensive
|
// TODO(single-default-literal): share these schema defaults and defensive
|
||||||
// apply() fallbacks through named constants while retaining both boundaries.
|
// apply() fallbacks through named constants while retaining both boundaries.
|
||||||
persistenceRoot: z.string().default('./.sessions'),
|
persistenceRoot: z.string().default('./.sessions'),
|
||||||
|
packChunks: z.boolean().default(false),
|
||||||
welcome: z.string().default('ready.'),
|
welcome: z.string().default('ready.'),
|
||||||
skills: agentCore.SkillConfigSchema,
|
skills: agentCore.SkillConfigSchema,
|
||||||
resumeSessionId: z.string(),
|
resumeSessionId: z.string(),
|
||||||
@@ -93,7 +96,10 @@ export function apply(ctx: Context, config: Config): void {
|
|||||||
}],
|
}],
|
||||||
...config.skills !== undefined ? { skills: config.skills } : {},
|
...config.skills !== undefined ? { skills: config.skills } : {},
|
||||||
})
|
})
|
||||||
ctx.plugin(SessionPersistenceJsonl, { root: config.persistenceRoot ?? './.sessions' })
|
ctx.plugin(SessionPersistenceJsonl, {
|
||||||
|
root: config.persistenceRoot ?? './.sessions',
|
||||||
|
...config.packChunks !== undefined ? { packChunks: config.packChunks } : {},
|
||||||
|
})
|
||||||
ctx.plugin(UserInteractionService)
|
ctx.plugin(UserInteractionService)
|
||||||
ctx.plugin(toolAskUser)
|
ctx.plugin(toolAskUser)
|
||||||
ctx.plugin(uiStdio, { welcome: config.welcome ?? 'ready.', agent: 'main' })
|
ctx.plugin(uiStdio, { welcome: config.welcome ?? 'ready.', agent: 'main' })
|
||||||
|
|||||||
@@ -7,10 +7,11 @@ The JSONL durable session-persistence backend — a concrete `SessionPersistence
|
|||||||
```
|
```
|
||||||
<root>/
|
<root>/
|
||||||
cwd-<sha256(cwd)[:12]>/ # per-project bucket (or _no-cwd/ when no cwd)
|
cwd-<sha256(cwd)[:12]>/ # per-project bucket (or _no-cwd/ when no cwd)
|
||||||
<encoded-id>.jsonl # header line + one SessionEvent per line (verbatim)
|
<encoded-id>.jsonl # header line + one storage record per line
|
||||||
```
|
```
|
||||||
|
|
||||||
- The first `.jsonl` line is the immutable `SessionHeader` tagged `{ type: 'session', version, id, cwd?, createdAt, parentSession?, seedLength? }`; every subsequent line is one `SessionEvent` JSON, **verbatim including `assistant/chunk`** so `seq` stays contiguous (`events[i].seq === i`).
|
- The first `.jsonl` line is the immutable `SessionHeader` tagged `{ type: 'session', version, id, cwd?, createdAt, parentSession?, seedLength? }`; every subsequent line is one storage record. `assistant/chunk` events are never dropped, and `seq` stays contiguous across the decoded log.
|
||||||
|
- A storage record is a `SessionEvent` JSON verbatim, or — written only under `packChunks` — a **packed chunk row** (`text-chunks` / `reasoning-chunks` / `tool-call-chunks`; bare slash-less tags like the header's `session`, so row tags cannot be confused with event types): one line holding a run of ≥3 consecutive same-block `assistant/chunk` delta events, `seq0`/`time0` plus per-member `dt` gaps reconstructing every member's `seq`/`time` exactly. The lossless codec lives in `@deepseek-ai/dsh-session` (`packChunkRuns`/`decodeStorageRecord`) and whitelists exact shapes — anything unrecognized stores verbatim. Reading is layout-blind: `load` always decodes rows, so packed, unpacked, and mixed files load identically.
|
||||||
- Session ids are unvalidated branded strings, so they are percent-encoded to a single safe path segment before use (no traversal, no collision).
|
- Session ids are unvalidated branded strings, so they are percent-encoded to a single safe path segment before use (no traversal, no collision).
|
||||||
|
|
||||||
## Config
|
## Config
|
||||||
@@ -18,6 +19,7 @@ The JSONL durable session-persistence backend — a concrete `SessionPersistence
|
|||||||
| Key | Type | Notes |
|
| Key | Type | Notes |
|
||||||
|---|---|---|
|
|---|---|---|
|
||||||
| `root` | `string` (required) | Root directory for all session files. **No default** — a `process.cwd()` default would scatter files as the process's cwd changes (bash calls, subprocesses). |
|
| `root` | `string` (required) | Root directory for all session files. **No default** — a `process.cwd()` default would scatter files as the process's cwd changes (bash calls, subprocesses). |
|
||||||
|
| `packChunks` | `boolean` (default `false`) | Write delta-chunk runs as packed rows (~60% smaller logs measured on a real coding session). Off, the written layout is byte-identical to the pre-packing format; reading packed rows works regardless of this switch. Off by default while the snapshot goldens stay one-event-per-line — recording with packing on rewrites every fixture `session.jsonl`. |
|
||||||
|
|
||||||
## Durability and crash semantics
|
## Durability and crash semantics
|
||||||
|
|
||||||
|
|||||||
@@ -10,7 +10,8 @@
|
|||||||
|
|
||||||
import { createHash } from 'node:crypto'
|
import { createHash } from 'node:crypto'
|
||||||
import { join } from 'node:path'
|
import { join } from 'node:path'
|
||||||
import type { SessionEvent, SessionHeader, SessionId } from '@deepseek-ai/dsh-session'
|
import { decodeStorageRecord, packChunkRuns } from '@deepseek-ai/dsh-session'
|
||||||
|
import type { SessionEvent, SessionHeader, SessionId, StorageRecord } from '@deepseek-ai/dsh-session'
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* The first line of a session's `.jsonl` file: the immutable
|
* The first line of a session's `.jsonl` file: the immutable
|
||||||
@@ -126,17 +127,26 @@ export function logPath(root: string, cwd: string | undefined, id: SessionId): s
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Serialize one event as a JSONL line (no trailing newline).
|
* Serialize an event batch as JSONL lines (no trailing newline). With
|
||||||
* @param event - the event to serialize verbatim.
|
* `packChunks` on, delta-chunk runs pack into `text-chunks` /
|
||||||
* @returns the event's single-line JSON text; the writer adds the newline.
|
* `reasoning-chunks` / `tool-call-chunks` storage rows; off writes one event
|
||||||
|
* per line, byte-identical to the pre-packing layout. Reading is layout-blind
|
||||||
|
* either way ({@link scanLog} always decodes rows), so the switch only shapes
|
||||||
|
* NEW bytes.
|
||||||
|
* @param events - the batch to serialize, in log order.
|
||||||
|
* @param packChunks - whether to pack delta runs into storage rows.
|
||||||
|
* @returns the batch's JSONL text; the writer adds the final newline.
|
||||||
*/
|
*/
|
||||||
export function eventLine(event: SessionEvent): string {
|
export function eventLines(events: readonly SessionEvent[], packChunks: boolean): string {
|
||||||
return JSON.stringify(event)
|
const records: readonly StorageRecord[] = packChunks ? packChunkRuns(events) : events
|
||||||
|
return records.map(record => JSON.stringify(record)).join('\n')
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Parse a JSONL log buffer into its preserved event prefix (the header is line
|
* Parse a JSONL log buffer into its preserved event prefix (the header is line
|
||||||
* 0). Fully written events in an interrupted final turn remain part of the
|
* 0). Event lines pass through verbatim; packed chunk rows expand back into
|
||||||
|
* their events, so callers see one contiguous event list regardless of layout.
|
||||||
|
* Fully written events in an interrupted final turn remain part of the
|
||||||
* prefix. The first unparsable record or seq gap after the last `turn/end`
|
* prefix. The first unparsable record or seq gap after the last `turn/end`
|
||||||
* marks a tolerated torn tail; the same hole in the committed region rejects.
|
* marks a tolerated torn tail; the same hole in the committed region rejects.
|
||||||
*
|
*
|
||||||
@@ -175,46 +185,60 @@ export function scanLog(buffer: Buffer): { meta: SessionHeader; events: SessionE
|
|||||||
}
|
}
|
||||||
const headerLine = parsedHeader
|
const headerLine = parsedHeader
|
||||||
|
|
||||||
// Parse every complete record first so the last valid `turn/end` determines
|
// Parse and decode every complete line first so the last valid `turn/end`
|
||||||
// whether an earlier hole is committed corruption or an uncommitted tail.
|
// determines whether an earlier hole is committed corruption or an
|
||||||
interface Parsed { ok: boolean; event?: SessionEvent; endByte: number }
|
// uncommitted tail. One line yields one event, or a whole run for a packed
|
||||||
|
// chunk row; a row-tagged line that fails row validation is a hole, exactly
|
||||||
|
// like unparsable JSON.
|
||||||
|
interface Parsed { ok: boolean; events?: SessionEvent[]; endByte: number }
|
||||||
const parsed: Parsed[] = eventEntries.map((entry) => {
|
const parsed: Parsed[] = eventEntries.map((entry) => {
|
||||||
try {
|
try {
|
||||||
return { ok: true, event: JSON.parse(entry.text) as SessionEvent, endByte: entry.endByte }
|
return { ok: true, events: decodeStorageRecord(JSON.parse(entry.text)), endByte: entry.endByte }
|
||||||
} catch {
|
} catch {
|
||||||
return { ok: false, endByte: entry.endByte }
|
return { ok: false, endByte: entry.endByte }
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
// The last index (into eventEntries) that is a valid `turn/end` — the last
|
// The last index (into eventEntries) that ends in a valid `turn/end` — the
|
||||||
// fully-committed boundary (the loop flushes only at turn/end).
|
// last fully-committed boundary (the loop flushes only at turn/end). A packed
|
||||||
|
// row never stores a turn/end, so only single-event lines can match.
|
||||||
let lastTurnEnd = -1
|
let lastTurnEnd = -1
|
||||||
for (let i = parsed.length - 1; i >= 0; i--) {
|
for (let i = parsed.length - 1; i >= 0; i--) {
|
||||||
const p = parsed[i]
|
const p = parsed[i]
|
||||||
if (p?.ok && p.event?.type === 'turn/end') { lastTurnEnd = i; break }
|
if (p?.ok && p.events?.some(e => e.type === 'turn/end')) { lastTurnEnd = i; break }
|
||||||
}
|
}
|
||||||
|
|
||||||
// Preserve the contiguous prefix, including a complete interrupted turn;
|
// Preserve the contiguous prefix, including a complete interrupted turn;
|
||||||
// holes through the last committed boundary throw, while later holes stop.
|
// holes through the last committed boundary throw, while later holes stop.
|
||||||
|
// Contiguity is a cursor over seqs (not the line index): a packed row
|
||||||
|
// advances the cursor by its whole run.
|
||||||
const preserved: SessionEvent[] = []
|
const preserved: SessionEvent[] = []
|
||||||
for (let i = 0; i < parsed.length; i++) {
|
let lastPreservedLine = -1
|
||||||
|
scan: for (let i = 0; i < parsed.length; i++) {
|
||||||
const p = parsed[i]
|
const p = parsed[i]
|
||||||
if (!p?.ok || p.event === undefined) {
|
if (!p?.ok || p.events === undefined) {
|
||||||
if (i <= lastTurnEnd) throw new Error(`corrupt session log: unparsable committed event at line ${i + 1}`)
|
if (i <= lastTurnEnd) throw new Error(`corrupt session log: unparsable committed event at line ${i + 1}`)
|
||||||
break // torn tail fragment after the last turn/end — stop, tolerate
|
break // torn tail fragment after the last turn/end — stop, tolerate
|
||||||
}
|
}
|
||||||
if (p.event.seq !== i) {
|
for (const event of p.events) {
|
||||||
if (i <= lastTurnEnd) throw new Error(`corrupt session log: seq gap in committed region at line ${i + 1} (expected ${i}, got ${p.event.seq})`)
|
if (event.seq !== preserved.length) {
|
||||||
break // gap after the last turn/end — torn tail, stop
|
if (i <= lastTurnEnd) {
|
||||||
|
throw new Error(`corrupt session log: seq gap in committed region at line ${i + 1} (expected ${preserved.length}, got ${event.seq})`)
|
||||||
|
}
|
||||||
|
break scan // gap after the last turn/end — torn tail, stop
|
||||||
|
}
|
||||||
|
preserved.push(event)
|
||||||
}
|
}
|
||||||
preserved.push(p.event)
|
lastPreservedLine = i
|
||||||
}
|
}
|
||||||
|
|
||||||
// committedBytes = end of the last PRESERVED line (header if none): the next
|
// committedBytes = end of the last FULLY preserved line (header if none): the
|
||||||
// append truncates any torn bytes past this point before writing the
|
// next append truncates any torn bytes past this point before writing the
|
||||||
// synthetic closers + new events.
|
// synthetic closers + new events. A line is preserved whole or not at all —
|
||||||
const lastPreserved = parsed[preserved.length - 1]
|
// a mid-row seq gap discards the whole row, keeping the truncation offset on
|
||||||
const committedBytes = preserved.length > 0 && lastPreserved ? lastPreserved.endByte : headerEntry.endByte
|
// a line boundary.
|
||||||
|
const lastPreserved = parsed[lastPreservedLine]
|
||||||
|
const committedBytes = lastPreserved !== undefined ? lastPreserved.endByte : headerEntry.endByte
|
||||||
return { meta: fromHeaderLine(headerLine), events: preserved, committedBytes }
|
return { meta: fromHeaderLine(headerLine), events: preserved, committedBytes }
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -16,10 +16,10 @@ import {
|
|||||||
} from '@deepseek-ai/dsh-session-persistence'
|
} from '@deepseek-ai/dsh-session-persistence'
|
||||||
import type { SessionEvent, SessionId, SessionHeader } from '@deepseek-ai/dsh-session'
|
import type { SessionEvent, SessionId, SessionHeader } from '@deepseek-ai/dsh-session'
|
||||||
import {
|
import {
|
||||||
encodeSegment, eventLine, logPath, parseHeaderMeta, scanLog, sessionDir, toHeaderLine,
|
encodeSegment, eventLines, logPath, parseHeaderMeta, scanLog, sessionDir, toHeaderLine,
|
||||||
} from './format.ts'
|
} from './format.ts'
|
||||||
|
|
||||||
/** Plugin config: where the JSONL backend keeps its session logs (`root` is required — no default). */
|
/** Plugin config: where the JSONL backend keeps its session logs, and the packed-row write switch. */
|
||||||
export interface Config {
|
export interface Config {
|
||||||
/**
|
/**
|
||||||
* Root directory for all session files. Required (no default): a default of
|
* Root directory for all session files. Required (no default): a default of
|
||||||
@@ -27,6 +27,15 @@ export interface Config {
|
|||||||
* (bash calls, subprocesses). Sessions group under per-cwd subdirectories.
|
* (bash calls, subprocesses). Sessions group under per-cwd subdirectories.
|
||||||
*/
|
*/
|
||||||
root: string
|
root: string
|
||||||
|
/**
|
||||||
|
* Write runs of consecutive `assistant/chunk` delta events as packed
|
||||||
|
* `text-chunks`/`reasoning-chunks`/`tool-call-chunks` rows (lossless,
|
||||||
|
* ~60% smaller logs measured on a real session). Off by default while
|
||||||
|
* snapshot fixtures stay in the one-event-per-line layout: recording with
|
||||||
|
* packing on rewrites every golden `session.jsonl`. READING packed rows is
|
||||||
|
* unconditional — a log's layout never depends on this switch.
|
||||||
|
*/
|
||||||
|
packChunks?: boolean
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Whether a filesystem error means absence; every non-ENOENT failure must surface. */
|
/** Whether a filesystem error means absence; every non-ENOENT failure must surface. */
|
||||||
@@ -44,6 +53,7 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi
|
|||||||
|
|
||||||
static Config: z<Config> = z.object({
|
static Config: z<Config> = z.object({
|
||||||
root: z.string().required(),
|
root: z.string().required(),
|
||||||
|
packChunks: z.boolean().default(false),
|
||||||
})
|
})
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -54,12 +64,16 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi
|
|||||||
override readonly name = 'session-persistence-jsonl'
|
override readonly name = 'session-persistence-jsonl'
|
||||||
|
|
||||||
private root: string
|
private root: string
|
||||||
|
private packChunks: boolean
|
||||||
private coordinator: PersistenceCoordinator<number>
|
private coordinator: PersistenceCoordinator<number>
|
||||||
|
|
||||||
constructor(ctx: Context, public config: Config) {
|
constructor(ctx: Context, public config: Config) {
|
||||||
super(ctx)
|
super(ctx)
|
||||||
// Resolve once so later process.cwd() changes cannot split one backend across roots.
|
// Resolve once so later process.cwd() changes cannot split one backend across roots.
|
||||||
this.root = resolve(config.root)
|
this.root = resolve(config.root)
|
||||||
|
// schemastery (static Config) applied the default before construction;
|
||||||
|
// the cast records that runtime fact for exactOptionalPropertyTypes.
|
||||||
|
this.packChunks = (config as Required<Config>).packChunks
|
||||||
this.coordinator = new PersistenceCoordinator<number>(this.ctx, this)
|
this.coordinator = new PersistenceCoordinator<number>(this.ctx, this)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -168,7 +182,7 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi
|
|||||||
throw new Error(`refusing to materialize "${meta.id}": a log already exists on disk (load/resume it instead)`)
|
throw new Error(`refusing to materialize "${meta.id}": a log already exists on disk (load/resume it instead)`)
|
||||||
}
|
}
|
||||||
const header = JSON.stringify(toHeaderLine(meta))
|
const header = JSON.stringify(toHeaderLine(meta))
|
||||||
const body = events.map(eventLine).join('\n')
|
const body = eventLines(events, this.packChunks)
|
||||||
const content = header + '\n' + body + '\n'
|
const content = header + '\n' + body + '\n'
|
||||||
|
|
||||||
const tmp = `${finalPath}.${randomBytes(6).toString('hex')}.tmp`
|
const tmp = `${finalPath}.${randomBytes(6).toString('hex')}.tmp`
|
||||||
@@ -223,7 +237,7 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi
|
|||||||
try {
|
try {
|
||||||
const { size: before } = await handle.stat()
|
const { size: before } = await handle.stat()
|
||||||
try {
|
try {
|
||||||
await handle.writeFile(events.map(eventLine).join('\n') + '\n')
|
await handle.writeFile(eventLines(events, this.packChunks) + '\n')
|
||||||
await handle.sync()
|
await handle.sync()
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
// Roll back whatever bytes landed so a retry starts from a clean EOF.
|
// Roll back whatever bytes landed so a retry starts from a clean EOF.
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ import { join } from 'node:path'
|
|||||||
import SessionStore, { SessionId } from '@deepseek-ai/dsh-session'
|
import SessionStore, { SessionId } from '@deepseek-ai/dsh-session'
|
||||||
import type { Session, SessionEvent, SessionHeader } from '@deepseek-ai/dsh-session'
|
import type { Session, SessionEvent, SessionHeader } from '@deepseek-ai/dsh-session'
|
||||||
import SessionPersistenceJsonl from '@deepseek-ai/dsh-session-persistence-jsonl'
|
import SessionPersistenceJsonl from '@deepseek-ai/dsh-session-persistence-jsonl'
|
||||||
import { encodeSegment, logPath, scanLog, sessionDir } from '../src/format.ts'
|
import { encodeSegment, eventLines, logPath, scanLog, sessionDir } from '../src/format.ts'
|
||||||
import { runPersistenceContract, meta, oneTurnLog, appendLog } from '../../session-persistence/tests/contract.ts'
|
import { runPersistenceContract, meta, oneTurnLog, appendLog } from '../../session-persistence/tests/contract.ts'
|
||||||
import { runCoordinatorContract, type CoordinatorFixture } from '../../session-persistence/tests/coordinator-contract.ts'
|
import { runCoordinatorContract, type CoordinatorFixture } from '../../session-persistence/tests/coordinator-contract.ts'
|
||||||
|
|
||||||
@@ -414,6 +414,119 @@ describe('SessionPersistenceJsonl: scanLog unit', () => {
|
|||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
|
describe('SessionPersistenceJsonl: packed chunk rows (packChunks: true)', () => {
|
||||||
|
let ctx: Context
|
||||||
|
beforeEach(async () => {
|
||||||
|
root = await freshRoot()
|
||||||
|
ctx = new Context()
|
||||||
|
await ctx.plugin(SessionStore)
|
||||||
|
await ctx.plugin(SessionPersistenceJsonl, { root, packChunks: true })
|
||||||
|
})
|
||||||
|
afterEach(async () => { await ctx.fiber.dispose() })
|
||||||
|
|
||||||
|
/** A one-turn log whose step streams a five-member text-delta run. */
|
||||||
|
function chunkRunLog(): SessionEvent[] {
|
||||||
|
const deltas: SessionEvent[] = Array.from({ length: 5 }, (_, k) => ({
|
||||||
|
type: 'assistant/chunk',
|
||||||
|
seq: 2 + k,
|
||||||
|
time: 3 + k,
|
||||||
|
data: { turn: 1, step: 1, chunk: { type: 'text-delta', index: 0, text: `t${k}` } },
|
||||||
|
}))
|
||||||
|
return [
|
||||||
|
{ type: 'turn/start', seq: 0, time: 1, data: { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } } },
|
||||||
|
{ type: 'step/start', seq: 1, time: 2, data: { turn: 1, step: 1 } },
|
||||||
|
...deltas,
|
||||||
|
{ type: 'assistant/message', seq: 7, time: 8, data: { turn: 1, step: 1, content: [{ type: 'text', text: 't0t1t2t3t4' }] }, surfaceOp: 'append', sourceEventSeqs: [2, 3, 4, 5, 6] },
|
||||||
|
{ type: 'step/end', seq: 8, time: 9, data: { turn: 1, step: 1 } },
|
||||||
|
{ type: 'turn/end', seq: 9, time: 10, data: { turn: 1, reason: { kind: 'completed' } } },
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
it('writes a delta run as one text-chunks row and loads back identical events', async () => {
|
||||||
|
const m = meta('packed', '/work')
|
||||||
|
const log = chunkRunLog()
|
||||||
|
await ctx.sessionPersistence.create(m)
|
||||||
|
await ctx.sessionPersistence.append(m.id, log)
|
||||||
|
|
||||||
|
const raw = (await readFile(logPath(root, '/work', m.id), 'utf8')).split('\n').filter(Boolean)
|
||||||
|
const tags = raw.slice(1).map(line => (JSON.parse(line) as { type: string }).type)
|
||||||
|
expect(tags).toEqual(['turn/start', 'step/start', 'text-chunks', 'assistant/message', 'step/end', 'turn/end'])
|
||||||
|
|
||||||
|
const loaded = await ctx.sessionPersistence.load(m.id)
|
||||||
|
expect(loaded.events).toEqual(log)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('loads a mixed file: verbatim lines from an unpacked writer, then packed appends', async () => {
|
||||||
|
const m = meta('mixed', '/work')
|
||||||
|
const log = chunkRunLog()
|
||||||
|
// First turn written line-per-event by an unpacked-config writer (an old
|
||||||
|
// file, hand-planted so this packed-config backend adopts it on load).
|
||||||
|
await mkdir(sessionDir(root, '/work'), { recursive: true })
|
||||||
|
await writeFile(logPath(root, '/work', m.id), [
|
||||||
|
JSON.stringify({ type: 'session', version: 0, id: 'mixed', createdAt: 1000, cwd: '/work' }),
|
||||||
|
...log.map(e => JSON.stringify(e)),
|
||||||
|
].join('\n') + '\n')
|
||||||
|
// Adopt the stored log (cursor = stored length), then append a second turn
|
||||||
|
// through THIS packed-config backend.
|
||||||
|
expect((await ctx.sessionPersistence.load(m.id)).events).toEqual(log)
|
||||||
|
const secondTurn: SessionEvent[] = JSON.parse(JSON.stringify(log)) as SessionEvent[]
|
||||||
|
for (const [k, e] of secondTurn.entries()) {
|
||||||
|
;(e as { seq: number }).seq = 10 + k
|
||||||
|
;(e.data as { turn: number }).turn = 2
|
||||||
|
}
|
||||||
|
await ctx.sessionPersistence.append(m.id, secondTurn)
|
||||||
|
|
||||||
|
const loaded = await ctx.sessionPersistence.load(m.id)
|
||||||
|
expect(loaded.events).toEqual([...log, ...secondTurn])
|
||||||
|
// The packed append really packed: the file's tail carries a text-chunks row.
|
||||||
|
const tags = (await readFile(logPath(root, '/work', m.id), 'utf8')).split('\n').filter(Boolean)
|
||||||
|
.map(line => (JSON.parse(line) as { type: string }).type)
|
||||||
|
expect(tags.filter(t => t === 'text-chunks')).toHaveLength(1)
|
||||||
|
expect(tags.filter(t => t === 'assistant/chunk')).toHaveLength(5)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('scanLog: a packed row advances the seq cursor by its whole run', () => {
|
||||||
|
const logText = [
|
||||||
|
JSON.stringify({ type: 'session', version: 0, id: 'rows', createdAt: 1 }),
|
||||||
|
JSON.stringify({ type: 'turn/start', seq: 0, time: 1, data: { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } } }),
|
||||||
|
JSON.stringify({ type: 'text-chunks', seq0: 1, time0: 2, data: { turn: 1, step: 1, index: 0, dt: [1, 1], texts: ['a', 'b', 'c'] } }),
|
||||||
|
JSON.stringify({ type: 'turn/end', seq: 4, time: 5, data: { turn: 1, reason: { kind: 'completed' } } }),
|
||||||
|
].join('\n') + '\n'
|
||||||
|
const { events } = scanLog(Buffer.from(logText))
|
||||||
|
expect(events.map(e => e.seq)).toEqual([0, 1, 2, 3, 4])
|
||||||
|
expect(events[2]).toEqual({ type: 'assistant/chunk', seq: 2, time: 3, data: { turn: 1, step: 1, chunk: { type: 'text-delta', index: 0, text: 'b' } } })
|
||||||
|
})
|
||||||
|
|
||||||
|
it('scanLog: a malformed packed row in the committed region rejects like corrupt JSON', () => {
|
||||||
|
const logText = [
|
||||||
|
JSON.stringify({ type: 'session', version: 0, id: 'bad-row', createdAt: 1 }),
|
||||||
|
// dt arity mismatch — row validation throws, so the line is a committed hole.
|
||||||
|
JSON.stringify({ type: 'text-chunks', seq0: 0, time0: 1, data: { turn: 1, step: 1, index: 0, dt: [], texts: ['a', 'b'] } }),
|
||||||
|
JSON.stringify({ type: 'turn/end', seq: 2, time: 3, data: { turn: 1, reason: { kind: 'completed' } } }),
|
||||||
|
].join('\n') + '\n'
|
||||||
|
expect(() => scanLog(Buffer.from(logText))).toThrow(/unparsable committed event/)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('scanLog: a packed row with a mid-run seq gap after the last turn/end drops the whole row', () => {
|
||||||
|
const logText = [
|
||||||
|
JSON.stringify({ type: 'session', version: 0, id: 'row-gap', createdAt: 1 }),
|
||||||
|
JSON.stringify({ type: 'turn/start', seq: 0, time: 1, data: { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } } }),
|
||||||
|
// seq0 skips 1 — the run's first member is already a gap; no turn/end follows.
|
||||||
|
JSON.stringify({ type: 'text-chunks', seq0: 2, time0: 2, data: { turn: 1, step: 1, index: 0, dt: [1, 1], texts: ['a', 'b', 'c'] } }),
|
||||||
|
].join('\n') + '\n'
|
||||||
|
const scanned = scanLog(Buffer.from(logText))
|
||||||
|
expect(scanned.events.map(e => e.seq)).toEqual([0])
|
||||||
|
// committedBytes stays on the line boundary BEFORE the dropped row.
|
||||||
|
const headerAndTurn = logText.split('\n').slice(0, 2).join('\n') + '\n'
|
||||||
|
expect(scanned.committedBytes).toBe(Buffer.byteLength(headerAndTurn, 'utf8'))
|
||||||
|
})
|
||||||
|
|
||||||
|
it('eventLines(packChunks: false) is byte-identical to the pre-packing layout', () => {
|
||||||
|
const log = chunkRunLog()
|
||||||
|
expect(eventLines(log, false)).toBe(log.map(e => JSON.stringify(e)).join('\n'))
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
describe('SessionPersistenceJsonl: edge cases', () => {
|
describe('SessionPersistenceJsonl: edge cases', () => {
|
||||||
let ctx: Context
|
let ctx: Context
|
||||||
beforeEach(async () => {
|
beforeEach(async () => {
|
||||||
|
|||||||
@@ -81,8 +81,10 @@ export function normalizeStdout(rawStdout: string, ctx: NormalizeContext): strin
|
|||||||
* Normalize a session JSONL log into a stable golden: the header line's
|
* Normalize a session JSONL log into a stable golden: the header line's
|
||||||
* volatile fields (`createdAt`, `id`, `cwd`) and every event's `time` are
|
* volatile fields (`createdAt`, `id`, `cwd`) and every event's `time` are
|
||||||
* zeroed/scrubbed, all volatile strings scrubbed, and `seq` is LEFT INTACT
|
* zeroed/scrubbed, all volatile strings scrubbed, and `seq` is LEFT INTACT
|
||||||
* (deterministic by contract). Output is JSONL in the same shape as the input —
|
* (deterministic by contract). A packed chunk row's timing (`time0`, the `dt`
|
||||||
* one compact record per line.
|
* gaps) zeroes just like an event `time`; its `seq0` stays, like `seq`.
|
||||||
|
* Output is JSONL in the same shape as the input — one compact record per
|
||||||
|
* line.
|
||||||
*
|
*
|
||||||
* @param rawLog The raw session `.jsonl` content.
|
* @param rawLog The raw session `.jsonl` content.
|
||||||
* @param ctx The run's volatile values to scrub.
|
* @param ctx The run's volatile values to scrub.
|
||||||
@@ -95,6 +97,13 @@ export function normalizeSessionLog(rawLog: string, ctx: NormalizeContext): stri
|
|||||||
// Header line: { type: 'session', createdAt, id, cwd, … }.
|
// Header line: { type: 'session', createdAt, id, cwd, … }.
|
||||||
if (record.type === 'session') {
|
if (record.type === 'session') {
|
||||||
if ('createdAt' in record) record.createdAt = 0
|
if ('createdAt' in record) record.createdAt = 0
|
||||||
|
} else if ('time0' in record) {
|
||||||
|
// Packed chunk row: zero the anchor timestamp and every member gap.
|
||||||
|
record.time0 = 0
|
||||||
|
const data = record.data
|
||||||
|
if (data !== null && typeof data === 'object' && Array.isArray((data as { dt?: unknown }).dt)) {
|
||||||
|
(data as { dt: unknown[] }).dt = (data as { dt: unknown[] }).dt.map(() => 0)
|
||||||
|
}
|
||||||
} else if ('time' in record) {
|
} else if ('time' in record) {
|
||||||
// Event line: zero the epoch-ms timestamp; keep seq (deterministic).
|
// Event line: zero the epoch-ms timestamp; keep seq (deterministic).
|
||||||
record.time = 0
|
record.time = 0
|
||||||
|
|||||||
@@ -109,6 +109,27 @@ describe('normalizeSessionLog', () => {
|
|||||||
expect(out).toContain('"decision":"block"') // the decision is the behavior — kept
|
expect(out).toContain('"decision":"block"') // the decision is the behavior — kept
|
||||||
})
|
})
|
||||||
|
|
||||||
|
it('zeroes a packed chunk row\'s time0 and dt gaps but keeps seq0 and payload', () => {
|
||||||
|
const row = JSON.stringify({
|
||||||
|
type: 'text-chunks', seq0: 7, time0: 999,
|
||||||
|
data: { turn: 1, step: 1, index: 0, dt: [212, 27, 0], texts: ['a', 'b', 'c', 'd'] },
|
||||||
|
})
|
||||||
|
const out = normalizeSessionLog(`${header({})}\n${row}\n`, ctx)
|
||||||
|
expect(out).toContain('"time0":0')
|
||||||
|
expect(out).toContain('"dt":[0,0,0]')
|
||||||
|
expect(out).toContain('"seq0":7') // seq0 is deterministic, like seq — NOT scrubbed
|
||||||
|
expect(out).toContain('"texts":["a","b","c","d"]')
|
||||||
|
expect(out).not.toContain('999')
|
||||||
|
expect(out).not.toContain('212')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('zeroes time0 even when a malformed row carries no dt array', () => {
|
||||||
|
const row = JSON.stringify({ type: 'text-chunks', seq0: 1, time0: 999, data: 'not-an-object' })
|
||||||
|
const out = normalizeSessionLog(`${header({})}\n${row}\n`, ctx)
|
||||||
|
expect(out).toContain('"time0":0')
|
||||||
|
expect(out).not.toContain('999')
|
||||||
|
})
|
||||||
|
|
||||||
it('leaves a non-hook event durationMs untouched (only hook/result is scrubbed)', () => {
|
it('leaves a non-hook event durationMs untouched (only hook/result is scrubbed)', () => {
|
||||||
const ev = JSON.stringify({ type: 'tool/result', seq: 2, time: 5, data: { durationMs: 88 } })
|
const ev = JSON.stringify({ type: 'tool/result', seq: 2, time: 5, data: { durationMs: 88 } })
|
||||||
const out = normalizeSessionLog(`${header({})}\n${ev}\n`, ctx)
|
const out = normalizeSessionLog(`${header({})}\n${ev}\n`, ctx)
|
||||||
|
|||||||
@@ -9,6 +9,7 @@
|
|||||||
import { existsSync, readFileSync } from 'node:fs'
|
import { existsSync, readFileSync } from 'node:fs'
|
||||||
import { delimiter as pathDelimiter } from 'node:path'
|
import { delimiter as pathDelimiter } from 'node:path'
|
||||||
import type { Context } from 'cordis'
|
import type { Context } from 'cordis'
|
||||||
|
import { decodeStorageRecord } from '@deepseek-ai/dsh-session'
|
||||||
import type { SessionEvent } from '@deepseek-ai/dsh-session'
|
import type { SessionEvent } from '@deepseek-ai/dsh-session'
|
||||||
import type { GenerateOptions, StreamChunk } from '@deepseek-ai/dsh-llm'
|
import type { GenerateOptions, StreamChunk } from '@deepseek-ai/dsh-llm'
|
||||||
import { LlmError, assertNever } from '@deepseek-ai/dsh-llm'
|
import { LlmError, assertNever } from '@deepseek-ai/dsh-llm'
|
||||||
@@ -68,7 +69,9 @@ export interface SessionScript {
|
|||||||
/**
|
/**
|
||||||
* Parse a session `.jsonl` buffer into its event list. Line 0 is the session
|
* Parse a session `.jsonl` buffer into its event list. Line 0 is the session
|
||||||
* header (a `{type:'session',…}` record), every subsequent non-empty line is a
|
* header (a `{type:'session',…}` record), every subsequent non-empty line is a
|
||||||
* {@link SessionEvent}. The header is skipped; malformed lines fail loud.
|
* {@link SessionEvent} or a packed chunk row (expanded back into its events, so
|
||||||
|
* a fixture recorded with `packChunks` on derives the same script). The header
|
||||||
|
* is skipped; malformed lines fail loud.
|
||||||
* @param text - the raw `.jsonl` file contents.
|
* @param text - the raw `.jsonl` file contents.
|
||||||
* @returns every event after the header, in log order.
|
* @returns every event after the header, in log order.
|
||||||
*/
|
*/
|
||||||
@@ -77,8 +80,7 @@ export function parseSessionLog(text: string): SessionEvent[] {
|
|||||||
const events: SessionEvent[] = []
|
const events: SessionEvent[] = []
|
||||||
// The JSONL backend guarantees line 0 is the session header.
|
// The JSONL backend guarantees line 0 is the session header.
|
||||||
for (let i = 1; i < lines.length; i++) {
|
for (let i = 1; i < lines.length; i++) {
|
||||||
const parsed: unknown = JSON.parse(lines[i] as string)
|
events.push(...decodeStorageRecord(JSON.parse(lines[i] as string)))
|
||||||
events.push(parsed as SessionEvent)
|
|
||||||
}
|
}
|
||||||
return events
|
return events
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -90,6 +90,19 @@ describe('parseSessionLog', () => {
|
|||||||
const ev = chunkEvent(1, 1, 1, TEXT_CHUNKS[0] as StreamChunk)
|
const ev = chunkEvent(1, 1, 1, TEXT_CHUNKS[0] as StreamChunk)
|
||||||
expect(parseSessionLog(`${header}\n\n${JSON.stringify(ev)}\n\n`)).toEqual([ev])
|
expect(parseSessionLog(`${header}\n\n${JSON.stringify(ev)}\n\n`)).toEqual([ev])
|
||||||
})
|
})
|
||||||
|
|
||||||
|
it('expands a packed chunk row into its events (a fixture recorded with packChunks on)', () => {
|
||||||
|
const header = JSON.stringify({ type: 'session', version: 0, id: 's1', createdAt: 0 })
|
||||||
|
const row = JSON.stringify({
|
||||||
|
type: 'text-chunks', seq0: 1, time0: 0,
|
||||||
|
data: { turn: 1, step: 1, index: 0, dt: [0, 0], texts: ['a', 'b', 'c'] },
|
||||||
|
})
|
||||||
|
expect(parseSessionLog(`${header}\n${row}\n`)).toEqual([
|
||||||
|
chunkEvent(1, 1, 1, { type: 'text-delta', index: 0, text: 'a' }),
|
||||||
|
chunkEvent(2, 1, 1, { type: 'text-delta', index: 0, text: 'b' }),
|
||||||
|
chunkEvent(3, 1, 1, { type: 'text-delta', index: 0, text: 'c' }),
|
||||||
|
])
|
||||||
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
describe('deriveReplayScript', () => {
|
describe('deriveReplayScript', () => {
|
||||||
|
|||||||
Reference in New Issue
Block a user