feat(session-persistence): abstract seam + JSONL backend + wiring
Add the durable session-persistence capability seam (ADR 0016): an
abstract SessionPersistence service (dsh-session-persistence,
ctx.sessionPersistence) defining create/append/load/list/has/delete/
update over the existing SessionEvent — no parallel persisted type — and
a first implementation (dsh-session-persistence-jsonl): an append-only
JSONL log per session with crash-safe atomic writes, truncation-repair
of a never-committed crash tail, and a read/replay path. SessionMeta
(format version, cwd, lineage) travels out-of-log via session.header.
A shared runPersistenceContract suite holds every backend to the same
append-only / contiguous-seq / lazy-materialization / serializability
semantics.
Config-driven create() now uses a per-run ${id}-session-<uuid> session
id so a fixed name no longer collides with an on-disk log once a durable
backend is loaded; each run is a new session (a demo simplification). The
examples drop their hand-rolled session-jsonl.ts and load the JSONL
backend via cordis.yml; CI smoke-loads it too.
The agent-facing create/resume factory that consumes load() is a
separate seam, deferred to a follow-up; this change stops at the load
primitive and does not reach into the loop.
This commit is contained in:
Tianyi Cui
@@ -8,7 +8,7 @@ This is the only package in the harness that contains concrete loop logic. Every
|
||||
|
||||
### Public API
|
||||
|
||||
- `ctx.agentLoop.create(id: string, options?: AgentOptions): LoopAgent` Create an agent, start its loop, and register it in `ctx.agents`. Disposed with the calling fiber.
|
||||
- `ctx.agentLoop.create(id: string, options?: AgentOptions): LoopAgent` — create an agent on a fresh per-run session id `${id}-session-<uuid>`, start its loop, and register it in `ctx.agents`. The per-run uuid avoids colliding with the on-disk log a prior run materialized once a durable persistence backend is loaded; each run is a new session (a deliberate demo simplification — a real resume-or-create policy is a TODO). Disposed with the calling fiber.
|
||||
|
||||
### Injected services
|
||||
|
||||
|
||||
@@ -35,6 +35,7 @@
|
||||
"@deepseek-ai/dsh-invariants": "^0.0.1",
|
||||
"@deepseek-ai/dsh-llm": "^0.0.1",
|
||||
"@deepseek-ai/dsh-session": "^0.0.1",
|
||||
"@deepseek-ai/dsh-session-persistence-jsonl": "^0.0.1",
|
||||
"@deepseek-ai/dsh-system-prompt": "^0.0.1",
|
||||
"@deepseek-ai/dsh-tools": "^0.0.1",
|
||||
"cordis": "^4.0.0-rc.6"
|
||||
|
||||
@@ -8,6 +8,7 @@
|
||||
*/
|
||||
|
||||
import { Context, Service } from 'cordis'
|
||||
import { randomUUID } from 'node:crypto'
|
||||
import z from 'schemastery'
|
||||
import { AgentId } from '@deepseek-ai/dsh-agent'
|
||||
import type { AgentOptions } from '@deepseek-ai/dsh-agent'
|
||||
@@ -62,12 +63,24 @@ export class AgentLoop extends Service {
|
||||
* Create an agent, start its loop, and register it. Returns the agent.
|
||||
* Disposed with the calling fiber.
|
||||
*
|
||||
* The session id is per-run (`${id}-session-<uuid>`, no fixed name): once a
|
||||
* durable persistence backend is loaded, a fixed `${id}-session` collides on
|
||||
* the second run — the backend refuses to re-create an id whose log already
|
||||
* exists on disk (the SessionId is the identity). A fresh id means each run
|
||||
* is a new session.
|
||||
*
|
||||
* TODO(demo): each run starting a brand-new session is fine for demos but is
|
||||
* NOT real conversation continuity. A production config-driven agent needs a
|
||||
* deliberate resume-or-create policy (resume the prior session if one exists,
|
||||
* else start fresh) or an explicit caller-chosen session id — revisit when the
|
||||
* UI/ACP path owns session selection.
|
||||
*
|
||||
* TODO(sub-agents): spawn/fork land here — accept a parent agent reference;
|
||||
* fork seeds the new Session with the parent's event log, spawn starts
|
||||
* fresh; the child is returned as a regular Agent handle.
|
||||
*/
|
||||
create(id: string, options: AgentOptions = {}): LoopAgent {
|
||||
const session = this.ctx.sessions.create(`${id}-session`)
|
||||
const session = this.ctx.sessions.create(`${id}-session-${randomUUID()}`, { meta: {} })
|
||||
const agent = new LoopAgent(this.ctx, AgentId(id), options, session)
|
||||
// Generator effect: stop and unregister are independent disposables
|
||||
// (LIFO), so a throwing stop() cannot leak the registry entry.
|
||||
|
||||
65
packages/agent-loop/tests/config-session-id.spec.ts
Normal file
65
packages/agent-loop/tests/config-session-id.spec.ts
Normal file
@@ -0,0 +1,65 @@
|
||||
import { afterEach, describe, expect, it } from 'vitest'
|
||||
import { Context } from 'cordis'
|
||||
import { mkdtemp, rm } from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import LlmService from '@deepseek-ai/dsh-llm'
|
||||
import SessionStore from '@deepseek-ai/dsh-session'
|
||||
import SystemPrompt from '@deepseek-ai/dsh-system-prompt'
|
||||
import ToolRegistry from '@deepseek-ai/dsh-tools'
|
||||
import AgentRegistry from '@deepseek-ai/dsh-agent'
|
||||
import SessionPersistenceJsonl from '@deepseek-ai/dsh-session-persistence-jsonl'
|
||||
import AgentLoop, { LoopAgent } from '@deepseek-ai/dsh-agent-loop'
|
||||
import { MockAdapter, textResponse } from './mock-adapter.ts'
|
||||
|
||||
const dirs: string[] = []
|
||||
afterEach(async () => { for (const d of dirs.splice(0)) await rm(d, { recursive: true, force: true }) })
|
||||
|
||||
function waitForIdle(ctx: Context, agent: LoopAgent): Promise<void> {
|
||||
return new Promise((resolve) => {
|
||||
const dispose = ctx.on('agent/status', (subject, status) => {
|
||||
if (subject === agent && status === 'idle') { dispose(); resolve() }
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
describe('config-driven session id', () => {
|
||||
it('config-driven create uses a fresh ${id}-session-<uuid> per run (restart-safe)', async () => {
|
||||
const root = await mkdtemp(join(tmpdir(), 'dsh-cfg-session-'))
|
||||
dirs.push(root)
|
||||
const idPattern = /^cfg-session-[0-9a-f-]{36}$/
|
||||
// Run 1: a config agent persists a turn under a generated session id.
|
||||
const ctx1 = new Context()
|
||||
await ctx1.plugin(LlmService)
|
||||
await ctx1.plugin(SessionStore)
|
||||
await ctx1.plugin(SystemPrompt)
|
||||
await ctx1.plugin(ToolRegistry)
|
||||
await ctx1.plugin(AgentRegistry)
|
||||
await ctx1.plugin(AgentLoop, { agents: [{ id: 'cfg', model: 'mock', systemPrompt: '' }] })
|
||||
await ctx1.plugin(SessionPersistenceJsonl, { root })
|
||||
ctx1.llm.registerAdapter(['mock'], new MockAdapter([textResponse('cfg')]))
|
||||
const a1 = ctx1.agents.get('cfg') as LoopAgent
|
||||
expect(a1.session.id).toMatch(idPattern)
|
||||
a1.send([{ type: 'text', text: 'q' }], { source: { kind: 'user' } })
|
||||
await waitForIdle(ctx1, a1)
|
||||
await ctx1.fiber.dispose()
|
||||
|
||||
// Run 2 over the SAME root: a fresh id means no on-disk collision (a fixed
|
||||
// ${id}-session would crash here with "already has a persisted log").
|
||||
const ctx2 = new Context()
|
||||
await ctx2.plugin(LlmService)
|
||||
await ctx2.plugin(SessionStore)
|
||||
await ctx2.plugin(SystemPrompt)
|
||||
await ctx2.plugin(ToolRegistry)
|
||||
await ctx2.plugin(AgentRegistry)
|
||||
await ctx2.plugin(AgentLoop, { agents: [{ id: 'cfg', model: 'mock', systemPrompt: '' }] })
|
||||
await ctx2.plugin(SessionPersistenceJsonl, { root })
|
||||
ctx2.llm.registerAdapter(['mock'], new MockAdapter([textResponse('cfg2')]))
|
||||
const a2 = ctx2.agents.get('cfg') as LoopAgent
|
||||
expect(a2.session.id).toMatch(idPattern)
|
||||
expect(a2.session.id).not.toBe(a1.session.id)
|
||||
a2.send([{ type: 'text', text: 'q2' }], { source: { kind: 'user' } })
|
||||
await waitForIdle(ctx2, a2)
|
||||
await ctx2.fiber.dispose()
|
||||
})
|
||||
})
|
||||
33
packages/session-persistence-jsonl/README.md
Normal file
33
packages/session-persistence-jsonl/README.md
Normal file
@@ -0,0 +1,33 @@
|
||||
# @deepseek-ai/dsh-session-persistence-jsonl
|
||||
|
||||
The JSONL durable session-persistence backend — a concrete `SessionPersistence` (the `dsh-session-persistence` seam). One append-only `.jsonl` event log per session plus a small atomic `.summary.json` sidecar for mutable metadata.
|
||||
|
||||
## On-disk layout
|
||||
|
||||
```
|
||||
<root>/
|
||||
cwd-<sha256(cwd)[:12]>/ # per-project bucket (or _no-cwd/ when no cwd)
|
||||
<encoded-id>.jsonl # header line + one SessionEvent per line (verbatim)
|
||||
<encoded-id>.summary.json # mutable SessionSummary (atomic temp-write + rename)
|
||||
```
|
||||
|
||||
- The first `.jsonl` line is the immutable `SessionHeader` tagged `{ type: 'session', version, id, cwd?, createdAt, parentSession? }`; every subsequent line is one `SessionEvent` JSON, **verbatim including `assistant/chunk`** so `seq` stays contiguous (`events[i].seq === i`).
|
||||
- Session ids are unvalidated branded strings, so they are percent-encoded to a single safe path segment before use (no traversal, no collision).
|
||||
|
||||
## Config
|
||||
|
||||
| Key | Type | Notes |
|
||||
|---|---|---|
|
||||
| `root` | `string` (required) | Root directory for all session files. **No default** — a `process.cwd()` default would scatter files as the process's cwd changes (bash calls, subprocesses). |
|
||||
|
||||
## Durability and crash semantics
|
||||
|
||||
- **Lazy materialization.** `create(meta)` writes nothing; the `.jsonl` (header + first batch) is written atomically (temp-write + `fsync` + rename) on the first `append`. A created-but-never-appended session leaves nothing on disk and is absent from `has`/`list`.
|
||||
- **Append-only.** Committed events (at or below a flushed `turn/end`) are never rewritten. Subsequent appends are line appends at EOF + `fsync`.
|
||||
- **Truncation-repair.** `load` returns events only up to the last complete `turn/end` and records the byte offset of any never-committed crash tail; the first post-load `append` `ftruncate`s to that offset (+ `fsync`) before writing, atomically discarding only the uncommitted tail.
|
||||
- **Contiguous-seq.** `load` rejects a mid-log parse error or `seq` gap (unloadable); `append` rejects a batch whose first `seq` does not continue the stored log, and rejects non-JSON-serializable `event.data` naming the offending event type.
|
||||
- **Format version.** Only v1 is supported; `load` rejects an unknown version. A future format change requires a version bump + migration.
|
||||
|
||||
## Write path
|
||||
|
||||
The plugin generalizes the example `session-jsonl.ts`: it subscribes to `session/created` (capture the header; persist a fork's seed once), `session/event` (snapshot each event when buffering — the live `session.events` object is mutable), and `session/flush`/dispose (drain the write-behind buffer through `append`). A per-session write cursor means a resumed session never re-appends already-stored events. Existing live sessions are seeded on plugin apply (HMR does not replay `session/created`). All backend operations for one session are serialized, and disposal awaits quiescence (every init + final drain) before returning, so no write lands after teardown.
|
||||
35
packages/session-persistence-jsonl/package.json
Normal file
35
packages/session-persistence-jsonl/package.json
Normal file
@@ -0,0 +1,35 @@
|
||||
{
|
||||
"name": "@deepseek-ai/dsh-session-persistence-jsonl",
|
||||
"description": "JSONL durable session persistence backend for the DeepSeek Harness",
|
||||
"version": "0.0.1",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"main": "lib/index.js",
|
||||
"types": "lib/index.d.ts",
|
||||
"exports": {
|
||||
".": {
|
||||
"types": "./lib/index.d.ts",
|
||||
"default": "./lib/index.js"
|
||||
},
|
||||
"./src/*": "./src/*",
|
||||
"./package.json": "./package.json"
|
||||
},
|
||||
"files": [
|
||||
"lib",
|
||||
"src"
|
||||
],
|
||||
"license": "BSD-3-Clause",
|
||||
"peerDependencies": {
|
||||
"@deepseek-ai/dsh-session": "^0.0.1",
|
||||
"@deepseek-ai/dsh-session-persistence": "^0.0.1",
|
||||
"cordis": "^4.0.0-rc.6"
|
||||
},
|
||||
"dependencies": {
|
||||
"schemastery": "^3.18.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@deepseek-ai/dsh-session": "^0.0.1",
|
||||
"@deepseek-ai/dsh-session-persistence": "^0.0.1",
|
||||
"cordis": "^4.0.0-rc.6"
|
||||
}
|
||||
}
|
||||
249
packages/session-persistence-jsonl/src/format.ts
Normal file
249
packages/session-persistence-jsonl/src/format.ts
Normal file
@@ -0,0 +1,249 @@
|
||||
/**
|
||||
* On-disk format helpers for the JSONL session-persistence backend: path
|
||||
* sanitization (a {@link SessionId} is an unvalidated branded string, so it
|
||||
* MUST be encoded before use in a path — no traversal, no collision), the
|
||||
* per-cwd directory layout, header-line (de)serialization, the atomic sidecar
|
||||
* for mutable summary fields, and the truncation-repair offset computation.
|
||||
*
|
||||
* @module dsh-session-persistence-jsonl/format
|
||||
*/
|
||||
|
||||
import { createHash } from 'node:crypto'
|
||||
import { join } from 'node:path'
|
||||
import type { SessionEvent, SessionHeader, SessionId, SessionMeta } from '@deepseek-ai/dsh-session'
|
||||
|
||||
/**
|
||||
* The first line of a session's `.jsonl` file: the immutable
|
||||
* {@link SessionHeader} tagged as a `session` record so a reader can tell it
|
||||
* apart from an event line.
|
||||
*/
|
||||
export interface HeaderLine {
|
||||
type: 'session'
|
||||
version: number
|
||||
id: SessionId
|
||||
createdAt: number
|
||||
cwd?: string
|
||||
parentSession?: SessionId
|
||||
}
|
||||
|
||||
/** Build the header line object from a {@link SessionHeader}. */
|
||||
export function toHeaderLine(header: SessionHeader): HeaderLine {
|
||||
return {
|
||||
type: 'session',
|
||||
version: header.version,
|
||||
id: header.id,
|
||||
createdAt: header.createdAt,
|
||||
...header.cwd !== undefined ? { cwd: header.cwd } : {},
|
||||
...header.parentSession !== undefined ? { parentSession: header.parentSession } : {},
|
||||
}
|
||||
}
|
||||
|
||||
/** Parse a header line back into a {@link SessionHeader}. */
|
||||
export function fromHeaderLine(line: HeaderLine): SessionHeader {
|
||||
return {
|
||||
version: line.version,
|
||||
id: line.id,
|
||||
createdAt: line.createdAt,
|
||||
...line.cwd !== undefined ? { cwd: line.cwd } : {},
|
||||
...line.parentSession !== undefined ? { parentSession: line.parentSession } : {},
|
||||
}
|
||||
}
|
||||
|
||||
/** Type guard: a parsed first line is a well-formed session header. */
|
||||
function isHeaderLine(value: unknown): value is HeaderLine {
|
||||
return (
|
||||
typeof value === 'object' && value !== null
|
||||
&& (value as { type?: unknown }).type === 'session'
|
||||
&& typeof (value as { version?: unknown }).version === 'number'
|
||||
&& typeof (value as { id?: unknown }).id === 'string'
|
||||
&& typeof (value as { createdAt?: unknown }).createdAt === 'number'
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Encode an arbitrary string as a single safe path segment, injectively over
|
||||
* ALL JS (UTF-16) strings — including lone surrogates. A {@link SessionId} is
|
||||
* an unvalidated branded string, so this neutralizes `../`, absolute paths,
|
||||
* NUL, and separators before any filesystem use.
|
||||
*
|
||||
* Each UTF-16 code unit is either kept literal (the safe set `[A-Za-z0-9_-]`)
|
||||
* or escaped as `~XXXX` (its 4-hex-digit code unit). `~` is itself escaped, so
|
||||
* the mapping is injective and reversible: distinct inputs never collide. We
|
||||
* iterate code UNITS (`charCodeAt`), not code points, so a lone surrogate
|
||||
* escapes to a distinct `~XXXX` instead of being normalized to U+FFFD (which
|
||||
* `Buffer.from(…, 'utf8')` would do, breaking injectivity). `.` is in the safe
|
||||
* set for readability but the whole-segment tokens `.`/`..` are escaped so they
|
||||
* can never traverse.
|
||||
*/
|
||||
export function encodeSegment(raw: string): string {
|
||||
if (raw.length === 0) throw new Error('cannot encode an empty path segment')
|
||||
if (raw === '.') return '~002E'
|
||||
if (raw === '..') return '~002E~002E'
|
||||
let out = ''
|
||||
for (let i = 0; i < raw.length; i++) {
|
||||
const code = raw.charCodeAt(i)
|
||||
const ch = String.fromCharCode(code)
|
||||
if (ch !== '~' && /^[A-Za-z0-9._-]$/.test(ch)) {
|
||||
out += ch
|
||||
} else {
|
||||
out += '~' + code.toString(16).toUpperCase().padStart(4, '0')
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
/**
|
||||
* The directory a session's files live in: the configured root, then a per-cwd
|
||||
* subdirectory so sessions group by project. The cwd subdir is a stable hash
|
||||
* (short, collision-resistant, filesystem-safe) plus an encoded suffix for
|
||||
* readability; sessions without a cwd go in a shared `_no-cwd` bucket.
|
||||
*/
|
||||
export function sessionDir(root: string, cwd: string | undefined): string {
|
||||
if (cwd === undefined) return join(root, '_no-cwd')
|
||||
const hash = createHash('sha256').update(cwd).digest('hex').slice(0, 12)
|
||||
return join(root, `cwd-${hash}`)
|
||||
}
|
||||
|
||||
/** The append-only event-log file path for a session. */
|
||||
export function logPath(root: string, cwd: string | undefined, id: SessionId): string {
|
||||
return join(sessionDir(root, cwd), `${encodeSegment(id)}.jsonl`)
|
||||
}
|
||||
|
||||
/** The mutable-summary sidecar path for a session (beside its log). */
|
||||
export function sidecarPath(root: string, cwd: string | undefined, id: SessionId): string {
|
||||
return join(sessionDir(root, cwd), `${encodeSegment(id)}.summary.json`)
|
||||
}
|
||||
|
||||
/** Serialize one event as a JSONL line (no trailing newline). */
|
||||
export function eventLine(event: SessionEvent): string {
|
||||
return JSON.stringify(event)
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute the byte offset of the END of the last complete `turn/end` line in a
|
||||
* JSONL log buffer (the header line is index 0). Returns the offset to which a
|
||||
* crash tail should be truncated, and the contiguous events up to and including
|
||||
* that `turn/end`. A parse error or a `seq` gap in the MIDDLE (at or before the
|
||||
* last `turn/end`) makes the session unloadable and throws; trailing garbage
|
||||
* AFTER the last `turn/end` is the tolerated crash tail and is excluded.
|
||||
*
|
||||
* This relies on the session-log invariant that every event lives inside a turn
|
||||
* (`Session.append` enforces it): the last `turn/end` is therefore the last
|
||||
* durable boundary, and nothing committed can sit outside a completed turn.
|
||||
*/
|
||||
export function scanLog(buffer: Buffer): { meta: SessionMeta; events: SessionEvent[]; committedBytes: number } {
|
||||
const text = buffer.toString('utf8')
|
||||
// Split into complete (newline-terminated) lines, tracking the byte offset of
|
||||
// each line's end so the truncation point is exact (multi-byte chars make the
|
||||
// char offset differ from the byte offset). A trailing line with no newline is
|
||||
// an uncommitted crash fragment and is ignored — it is below the last
|
||||
// turn/end by construction (the loop only flushes whole lines).
|
||||
//
|
||||
// Track the byte offset with a RUNNING accumulator (`endByte`), adding each
|
||||
// line's byte length as we go. Recomputing `Buffer.byteLength(text.slice(0, i))`
|
||||
// per newline would rescan the whole prefix every time — O(n²) over a long
|
||||
// log (one assistant/chunk line per token makes that pathological).
|
||||
const lines: { text: string; endByte: number }[] = []
|
||||
let start = 0
|
||||
let byteOffset = 0
|
||||
for (let i = 0; i < text.length; i++) {
|
||||
if (text[i] === '\n') {
|
||||
const lineText = text.slice(start, i)
|
||||
byteOffset += Buffer.byteLength(lineText, 'utf8') + 1 // +1 for the '\n' (a 1-byte char)
|
||||
lines.push({ text: lineText, endByte: byteOffset })
|
||||
start = i + 1
|
||||
}
|
||||
}
|
||||
|
||||
const [headerEntry, ...eventEntries] = lines
|
||||
if (headerEntry === undefined) throw new Error('empty or header-less session log')
|
||||
|
||||
// Line 0 is the header.
|
||||
let parsedHeader: unknown
|
||||
try {
|
||||
parsedHeader = JSON.parse(headerEntry.text)
|
||||
} catch {
|
||||
throw new Error('corrupt session log: header line is not valid JSON')
|
||||
}
|
||||
if (!isHeaderLine(parsedHeader)) {
|
||||
throw new Error('corrupt session log: first line is not a session header')
|
||||
}
|
||||
const headerLine = parsedHeader
|
||||
|
||||
// Find the committed region: the prefix up to and including the LAST complete
|
||||
// `turn/end` in the WHOLE log. Two passes so a crash tail after the last
|
||||
// turn/end is tolerated, but corruption/gaps AT OR BEFORE the last committed
|
||||
// turn/end make the log unloadable (committed data must never be silently
|
||||
// dropped).
|
||||
//
|
||||
// Pass 1: parse every line that parses, recording (parsedOk, seq, isTurnEnd,
|
||||
// endByte) per line index. Lines that fail to parse are holes.
|
||||
interface Parsed { ok: boolean; event?: SessionEvent; endByte: number }
|
||||
const parsed: Parsed[] = eventEntries.map((entry) => {
|
||||
try {
|
||||
return { ok: true, event: JSON.parse(entry.text) as SessionEvent, endByte: entry.endByte }
|
||||
} catch {
|
||||
return { ok: false, endByte: entry.endByte }
|
||||
}
|
||||
})
|
||||
|
||||
// The last index (into eventEntries) that is a valid `turn/end`.
|
||||
let lastTurnEnd = -1
|
||||
for (let i = parsed.length - 1; i >= 0; i--) {
|
||||
const p = parsed[i]
|
||||
if (p?.ok && p.event?.type === 'turn/end') { lastTurnEnd = i; break }
|
||||
}
|
||||
|
||||
// No committed turn/end anywhere: nothing is committed. The whole event
|
||||
// region is an uncommitted (first-turn) tail — committedBytes is the header.
|
||||
if (lastTurnEnd < 0) {
|
||||
const meta = metaFrom(headerLine)
|
||||
return { meta, events: [], committedBytes: headerEntry.endByte }
|
||||
}
|
||||
|
||||
// Pass 2: the committed prefix [0..lastTurnEnd] must be fully intact and
|
||||
// contiguous (line i is a parsed event with seq === i). A hole or seq gap in
|
||||
// the committed region means committed data was damaged → unloadable.
|
||||
const committed: SessionEvent[] = []
|
||||
for (let i = 0; i <= lastTurnEnd; i++) {
|
||||
const p = parsed[i]
|
||||
if (!p?.ok || p.event === undefined) {
|
||||
throw new Error(`corrupt session log: unparsable committed event at line ${i + 1}`)
|
||||
}
|
||||
if (p.event.seq !== i) {
|
||||
throw new Error(`corrupt session log: seq gap in committed region at line ${i + 1} (expected ${i}, got ${p.event.seq})`)
|
||||
}
|
||||
committed.push(p.event)
|
||||
}
|
||||
const lastEntry = parsed[lastTurnEnd]
|
||||
/* v8 ignore next -- lastTurnEnd indexes a parsed entry by construction */
|
||||
const committedBytes = lastEntry ? lastEntry.endByte : headerEntry.endByte
|
||||
return { meta: metaFrom(headerLine), events: committed, committedBytes }
|
||||
}
|
||||
|
||||
/** Build the load-time {@link SessionMeta} from a header line (summary overlaid later). */
|
||||
function metaFrom(headerLine: HeaderLine): SessionMeta {
|
||||
return {
|
||||
...fromHeaderLine(headerLine),
|
||||
updatedAt: headerLine.createdAt, // overlaid by the sidecar in load()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse just the header line of a log into load-time {@link SessionMeta}, or
|
||||
* `undefined` if it is missing/not a header. Used by `list()` to read session
|
||||
* metadata WITHOUT parsing the whole log: a session picker scales with the
|
||||
* number of sessions, not the total size of every conversation. The summary
|
||||
* sidecar is overlaid by the caller; `updatedAt` here mirrors `createdAt` until
|
||||
* then (same as {@link scanLog}'s load-time meta).
|
||||
*/
|
||||
export function parseHeaderMeta(firstLine: string): SessionMeta | undefined {
|
||||
let parsed: unknown
|
||||
try {
|
||||
parsed = JSON.parse(firstLine)
|
||||
} catch {
|
||||
return undefined
|
||||
}
|
||||
if (!isHeaderLine(parsed)) return undefined
|
||||
return metaFrom(parsed)
|
||||
}
|
||||
818
packages/session-persistence-jsonl/src/index.ts
Normal file
818
packages/session-persistence-jsonl/src/index.ts
Normal file
@@ -0,0 +1,818 @@
|
||||
/**
|
||||
* JSONL durable session-persistence backend (`@deepseek-ai/dsh-session-persistence-jsonl`).
|
||||
*
|
||||
* Two concerns in one plugin:
|
||||
*
|
||||
* 1. **The backend** — a concrete {@link SessionPersistence}: one append-only
|
||||
* `.jsonl` event log per session (a header line then one `SessionEvent` per
|
||||
* line, verbatim including `assistant/chunk` so `seq` stays contiguous) plus
|
||||
* a small atomic `.summary.json` sidecar for the mutable `SessionSummary`.
|
||||
* Lazy materialization (no file until the first `append`), atomic first
|
||||
* write, and truncation-repair of a never-committed crash tail on the first
|
||||
* `append` after a `load`.
|
||||
*
|
||||
* 2. **The write path** — the `session/event` → buffer → `session/flush` drain
|
||||
* that generalizes the example `session-jsonl.ts`: snapshot each event when
|
||||
* it is buffered (the live `session.events` object is mutable), persist
|
||||
* forks once on `session/created`, maintain a per-session write cursor so a
|
||||
* resumed session never re-appends already-stored events, and seed existing
|
||||
* live sessions on plugin apply (HMR does not replay `session/created`).
|
||||
*
|
||||
* @module @deepseek-ai/dsh-session-persistence-jsonl
|
||||
*/
|
||||
|
||||
import { Context } from 'cordis'
|
||||
import z from 'schemastery'
|
||||
import { open, mkdir, readFile, readdir, rename, link, rm, writeFile, truncate } from 'node:fs/promises'
|
||||
import { resolve } from 'node:path'
|
||||
import { randomBytes } from 'node:crypto'
|
||||
import { SessionPersistence } from '@deepseek-ai/dsh-session-persistence'
|
||||
import { isJsonValue } from '@deepseek-ai/dsh-session'
|
||||
import type { Session, SessionEvent, SessionId, SessionMeta, SessionSummary } from '@deepseek-ai/dsh-session'
|
||||
import {
|
||||
encodeSegment, eventLine, logPath, parseHeaderMeta, scanLog, sessionDir, sidecarPath, toHeaderLine,
|
||||
} from './format.ts'
|
||||
|
||||
export interface Config {
|
||||
/**
|
||||
* Root directory for all session files. Required (no default): a default of
|
||||
* `process.cwd()` would scatter session files as the process's cwd changes
|
||||
* (bash calls, subprocesses). Sessions group under per-cwd subdirectories.
|
||||
*/
|
||||
root: string
|
||||
}
|
||||
|
||||
/** Per-session write state held by the backend's in-memory bookkeeping. */
|
||||
interface SessionState {
|
||||
meta: SessionMeta
|
||||
/** The next seq the backend expects to append (the stored log length). */
|
||||
cursor: number
|
||||
/** Whether the `.jsonl` file has been physically materialized. */
|
||||
materialized: boolean
|
||||
/**
|
||||
* The live Session this state was bound to via `onCreated`, if any. Used to
|
||||
* detect a DIFFERENT live session reusing a tracked id (a collision): state
|
||||
* created through the public `create()`/`load()` API has no owner, but state
|
||||
* bound to a live session lets `onCreated` reject a second, unrelated session
|
||||
* object on the same id instead of silently no-opping (which would leave the
|
||||
* new session's events to be dropped against the old cursor).
|
||||
*/
|
||||
owner?: Session
|
||||
/**
|
||||
* If a load truncation-repair is pending, the byte offset to truncate the
|
||||
* file to before the next append (discards the never-committed crash tail).
|
||||
*/
|
||||
repairTo?: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether a live session's `seed` reproduces a persisted `prefix` exactly — the
|
||||
* prefix is no longer than the seed, and each prefix event DEEP-equals the seed
|
||||
* event at the same index. Used to tell a session legitimately continuing a
|
||||
* persisted log (HMR re-seeing its own session, or a resume) from a different
|
||||
* session that merely reuses the id: the latter would have its already-counted
|
||||
* seq 0..prefix-1 events filtered out on flush and its conversation silently
|
||||
* grafted onto the old log.
|
||||
*
|
||||
* The comparison is a full structural equality (via canonical JSON) of each
|
||||
* event INCLUDING its `data` payload, not just `seq`/`type`/`time` — a session
|
||||
* built from loaded events but with mutated message/tool payloads (same seq/
|
||||
* type/time) must NOT be accepted, or the live history and durable log diverge.
|
||||
* Both sides are JSON-serializable by contract (Session.append enforces it), so
|
||||
* JSON.stringify is a sound canonical form here.
|
||||
*/
|
||||
function seedCoversPrefix(seed: readonly SessionEvent[], prefix: readonly SessionEvent[]): boolean {
|
||||
return prefix.length <= seed.length
|
||||
&& prefix.every((e, i) => {
|
||||
const s = seed[i]
|
||||
return s !== undefined && JSON.stringify(s) === JSON.stringify(e)
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Reject non-JSON-serializable `event.data`, naming the offending type. Used on
|
||||
* the backend's `append(events)` entry point (replay/fork paths that bypass a
|
||||
* live `Session`); events that flow through `Session.append` are already
|
||||
* validated at the source, so the live write path never needs this.
|
||||
*/
|
||||
function assertSerializable(events: readonly SessionEvent[]): void {
|
||||
for (const event of events) {
|
||||
if (!isJsonValue(event.data)) {
|
||||
throw new Error(`event "${event.type}" carries non-JSON-serializable data (seq ${event.seq})`)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The JSONL persistence backend. Load as a plugin; it registers as
|
||||
* `ctx.sessionPersistence` and installs the write-path listeners.
|
||||
*/
|
||||
export class SessionPersistenceJsonl extends SessionPersistence {
|
||||
static inject = ['sessions']
|
||||
|
||||
static Config: z<Config> = z.object({
|
||||
root: z.string().required(),
|
||||
})
|
||||
|
||||
private root: string
|
||||
/** Backend bookkeeping keyed by session id (NOT the live Session object). */
|
||||
private states = new Map<string, SessionState>()
|
||||
/** Write-behind buffers keyed by the live Session (write path). */
|
||||
private buffers = new Map<Session, SessionEvent[]>()
|
||||
/**
|
||||
* Per-session serialization: every backend operation chains onto the prior
|
||||
* one for the same id, so concurrent flushes / a flush racing onCreated never
|
||||
* interleave file writes or read a half-built state. Keyed by session id.
|
||||
*/
|
||||
private chains = new Map<string, Promise<unknown>>()
|
||||
/**
|
||||
* Per-session init promise (onCreated). Keyed by the LIVE Session OBJECT, not
|
||||
* its id: a disposed fiber's session can be replaced by a different live
|
||||
* Session reusing the same id (HMR, an ACP reconnect), and an id-keyed cache
|
||||
* would hand the new object the old object's init promise — skipping
|
||||
* onCreated for the new session, so its events start at seq 0 while flush
|
||||
* filters against the stale cursor and silently drops them. Keying by object
|
||||
* gives each live Session its own init. flush awaits it before appending.
|
||||
*/
|
||||
private inits = new Map<Session, Promise<void>>()
|
||||
|
||||
constructor(ctx: Context, public config: Config) {
|
||||
super(ctx)
|
||||
// Resolve the configured root to an ABSOLUTE path ONCE, here. A relative
|
||||
// root (the examples use `./.sessions`) would otherwise re-resolve against
|
||||
// `process.cwd()` at every later readdir/open — so if any plugin or test
|
||||
// changed cwd between create, append, and load, one session's files could
|
||||
// split across directories. Pinning it at construction makes all paths
|
||||
// stable regardless of later cwd changes.
|
||||
this.root = resolve(config.root)
|
||||
this.installWritePath()
|
||||
}
|
||||
|
||||
// --- SessionPersistence backend surface (all serialized per session id) ---
|
||||
|
||||
create(meta: SessionMeta): Promise<void> {
|
||||
// Snapshot the metadata at call time: the op runs later (behind the
|
||||
// per-session chain) and the snapshot is also stored as the lazy state, so
|
||||
// keeping the caller's object by reference would let a later mutation of
|
||||
// `id`/`cwd` register under one key but materialize under a different
|
||||
// path/header. A shallow copy is enough — SessionMeta is a flat record.
|
||||
const snapshot: SessionMeta = { ...meta }
|
||||
return this.serialize(snapshot.id, () => this.createCore(snapshot))
|
||||
}
|
||||
|
||||
private async createCore(meta: SessionMeta): Promise<void> {
|
||||
// Do NOT clobber an existing session. If we already track it, or a log
|
||||
// exists on disk under this id, refuse — the SessionId IS the identity, and
|
||||
// silently resetting state (cursor 0, materialized false) over committed
|
||||
// data would let the next append rename over the existing log.
|
||||
if (this.states.has(meta.id)) {
|
||||
throw new Error(`session "${meta.id}" already exists in this backend`)
|
||||
}
|
||||
// Scan ALL cwd buckets (pass undefined), not just meta.cwd's: load/has/adopt
|
||||
// identify a session by id alone and search every bucket, so an id already
|
||||
// persisted under a DIFFERENT cwd must still block creation here. Probing
|
||||
// only meta.cwd's bucket would let two logs share one id and make resume
|
||||
// (which picks the first matching bucket) nondeterministic.
|
||||
if (await this.findLog(meta.id, undefined) !== undefined) {
|
||||
throw new Error(`session "${meta.id}" already has a persisted log on disk; load/resume it instead of creating`)
|
||||
}
|
||||
// Pure lazy: record intent only. No file until the first append, so an
|
||||
// abandoned (never-appended) session leaves nothing on disk and stays
|
||||
// absent from has()/list().
|
||||
this.states.set(meta.id, { meta, cursor: 0, materialized: false })
|
||||
}
|
||||
|
||||
/**
|
||||
* Run `op` after any in-flight operation for the same session id, so writes
|
||||
* for one session never interleave (two flushes, a flush racing a load, an
|
||||
* update racing an append). Errors do not poison the chain — the next op
|
||||
* still runs. NOTE: serialized public methods must NOT call each other (that
|
||||
* would deadlock on the same chain); they call the unserialized `*Core`
|
||||
* helpers instead.
|
||||
*/
|
||||
private serialize<T>(id: SessionId, op: () => Promise<T>): Promise<T> {
|
||||
const prior = this.chains.get(id) ?? Promise.resolve()
|
||||
const next = prior.then(op, op)
|
||||
// Keep the chain alive but swallow this op's rejection for the NEXT waiter
|
||||
// (the caller still sees the real rejection via `next`).
|
||||
this.chains.set(id, next.then(() => undefined, () => undefined))
|
||||
return next
|
||||
}
|
||||
|
||||
// `async` so the synchronous validate/clone below reject (not throw) per the
|
||||
// Promise<void> contract — callers use `await expect(...).rejects`.
|
||||
async append(id: SessionId, events: readonly SessionEvent[]): Promise<void> {
|
||||
// Validate serializability BEFORE cloning, so a bad event surfaces the typed
|
||||
// "non-JSON-serializable" error rather than an opaque DataCloneError from
|
||||
// structuredClone below. (In an async method this throw becomes a rejection,
|
||||
// honoring the Promise<void> contract rather than throwing synchronously.)
|
||||
assertSerializable(events)
|
||||
// Deep-snapshot the batch here, BEFORE the op waits behind the per-session
|
||||
// chain: the op may await before serializing, so a caller that passes a live
|
||||
// array (e.g. session.events) and mutates it — OR mutates an event object
|
||||
// inside it — before the op runs would otherwise have those changes
|
||||
// persisted, or advance the cursor past what was actually written.
|
||||
// structuredClone covers both the array and the event objects (safe now that
|
||||
// serializability is checked above). The clone happens synchronously (before
|
||||
// the first await), so it is taken at call time.
|
||||
const batch = events.map(e => structuredClone(e))
|
||||
return this.serialize(id, () => this.appendCore(id, batch))
|
||||
}
|
||||
|
||||
private async appendCore(id: SessionId, events: readonly SessionEvent[]): Promise<void> {
|
||||
if (events.length === 0) return
|
||||
assertSerializable(events)
|
||||
let state = this.states.get(id)
|
||||
if (state === undefined) state = await this.adopt(id) // calls loadCore, not load
|
||||
|
||||
// Truncation-repair: on the first append after a load that found a crash
|
||||
// tail, physically discard the orphaned bytes before writing.
|
||||
if (state.repairTo !== undefined) {
|
||||
await this.repair(state, state.repairTo)
|
||||
delete state.repairTo
|
||||
}
|
||||
|
||||
// Contiguity contract: each event's seq must continue the stored log.
|
||||
for (const [i, event] of events.entries()) {
|
||||
if (event.seq !== state.cursor + i) {
|
||||
throw new Error(`append seq mismatch for "${id}": expected ${state.cursor + i} at index ${i}, got ${event.seq}`)
|
||||
}
|
||||
}
|
||||
|
||||
if (!state.materialized) {
|
||||
await this.materialize(state, events)
|
||||
} else {
|
||||
await this.appendLines(state, events)
|
||||
}
|
||||
// The durable event log is the transaction: advance the cursor as soon as
|
||||
// the log write commits. The sidecar (mutable summary) is best-effort here
|
||||
// — a failed sidecar write must NOT reject an append whose log already
|
||||
// landed (that would desync the cursor and let a retry duplicate seqs).
|
||||
state.cursor += events.length
|
||||
await this.touchSummary(state).catch(() => { /* sidecar is recoverable metadata; log is durable */ })
|
||||
}
|
||||
|
||||
load(id: SessionId): Promise<{ meta: SessionMeta; events: SessionEvent[] }> {
|
||||
return this.serialize(id, () => this.loadCore(id))
|
||||
}
|
||||
|
||||
private async loadCore(id: SessionId): Promise<{ meta: SessionMeta; events: SessionEvent[] }> {
|
||||
const cwd = this.states.get(id)?.meta.cwd
|
||||
const file = await this.findLog(id, cwd)
|
||||
if (file === undefined) throw new Error(`session "${id}" not found`)
|
||||
const buffer = await readFile(file.path)
|
||||
const { meta, events, committedBytes } = scanLog(buffer)
|
||||
this.assertVersion(meta)
|
||||
|
||||
const summary = await this.readSidecar(id, meta.cwd)
|
||||
const fullMeta: SessionMeta = { ...meta, ...summary }
|
||||
|
||||
// Record the state so the next append repairs the crash tail (if any) and
|
||||
// continues at the committed length. The state keeps its OWN copy of the
|
||||
// meta; the value returned to the caller is a SEPARATE copy so a consumer
|
||||
// mutating `loaded.meta` (e.g. `cwd`) cannot corrupt the backend's pathing
|
||||
// metadata and send later reads/writes to the wrong log.
|
||||
const needsRepair = committedBytes < buffer.byteLength
|
||||
this.states.set(id, {
|
||||
meta: { ...fullMeta },
|
||||
cursor: events.length,
|
||||
materialized: true,
|
||||
...needsRepair ? { repairTo: committedBytes } : {},
|
||||
})
|
||||
return { meta: fullMeta, events }
|
||||
}
|
||||
|
||||
async list(): Promise<SessionMeta[]> {
|
||||
const metas: SessionMeta[] = []
|
||||
for (const dir of await this.listCwdDirs()) {
|
||||
for (const name of await this.listJsonl(dir)) {
|
||||
// Read ONLY the header line, not the whole log: a session picker must
|
||||
// scale with the number of sessions, not the total size of every
|
||||
// conversation (the log persists every assistant/chunk verbatim, so a
|
||||
// full scanLog here would be O(total history)).
|
||||
const first = await this.readFirstLine(`${dir}/${name}`)
|
||||
if (first === undefined) continue // empty/half-written file
|
||||
const meta = parseHeaderMeta(first)
|
||||
if (meta === undefined) continue // not a session header
|
||||
const summary = await this.readSidecar(meta.id, meta.cwd)
|
||||
metas.push({ ...meta, ...summary })
|
||||
}
|
||||
}
|
||||
return metas
|
||||
}
|
||||
|
||||
/**
|
||||
* Read the first newline-terminated line of a file without loading the whole
|
||||
* file. Returns undefined if the file is empty or has no complete first line
|
||||
* (a half-written log). Reads in bounded chunks so a huge log costs only the
|
||||
* header read.
|
||||
*/
|
||||
private async readFirstLine(path: string): Promise<string | undefined> {
|
||||
const handle = await open(path, 'r')
|
||||
try {
|
||||
const chunks: Buffer[] = []
|
||||
const buf = Buffer.alloc(8192)
|
||||
for (;;) {
|
||||
const { bytesRead } = await handle.read(buf, 0, buf.length, null)
|
||||
if (bytesRead === 0) return undefined // EOF with no newline → no complete line
|
||||
const slice = buf.subarray(0, bytesRead)
|
||||
const nl = slice.indexOf(0x0a)
|
||||
if (nl !== -1) {
|
||||
chunks.push(slice.subarray(0, nl))
|
||||
return Buffer.concat(chunks).toString('utf8')
|
||||
}
|
||||
chunks.push(Buffer.from(slice))
|
||||
}
|
||||
} finally {
|
||||
await handle.close()
|
||||
}
|
||||
}
|
||||
|
||||
async has(id: SessionId): Promise<boolean> {
|
||||
const state = this.states.get(id)
|
||||
if (state?.materialized) return true
|
||||
const cwd = state?.meta.cwd
|
||||
return (await this.findLog(id, cwd)) !== undefined
|
||||
}
|
||||
|
||||
delete(id: SessionId): Promise<void> {
|
||||
return this.serialize(id, () => this.deleteCore(id))
|
||||
}
|
||||
|
||||
private async deleteCore(id: SessionId): Promise<void> {
|
||||
const cwd = this.states.get(id)?.meta.cwd
|
||||
const file = await this.findLog(id, cwd)
|
||||
if (file) await rm(file.path, { force: true })
|
||||
// Remove the sidecar too. A lazy session (update() before the first
|
||||
// append()) has a `.summary.json` sidecar but NO `.jsonl` log, and after a
|
||||
// restart the in-memory cwd is gone — so keying sidecar removal off the log
|
||||
// or the in-memory cwd would leak its possibly-sensitive title/firstPrompt.
|
||||
// Scan every cwd bucket for the sidecar by its (sanitized) filename.
|
||||
await this.removeSidecars(id)
|
||||
this.states.delete(id)
|
||||
}
|
||||
|
||||
/** Remove a session's summary sidecar from EVERY cwd bucket (id is unique). */
|
||||
private async removeSidecars(id: SessionId): Promise<void> {
|
||||
const target = `${encodeSegment(id)}.summary.json`
|
||||
for (const dir of await this.listCwdDirs()) {
|
||||
await rm(`${dir}/${target}`, { force: true })
|
||||
}
|
||||
}
|
||||
|
||||
update(id: SessionId, summary: Partial<SessionSummary>): Promise<void> {
|
||||
return this.serialize(id, () => this.updateCore(id, summary))
|
||||
}
|
||||
|
||||
private async updateCore(id: SessionId, summary: Partial<SessionSummary>): Promise<void> {
|
||||
let state = this.states.get(id)
|
||||
if (state === undefined) state = await this.adopt(id)
|
||||
// Build the NEXT meta separately and commit it to in-memory state only AFTER
|
||||
// the sidecar write succeeds. update's only durable effect is the sidecar,
|
||||
// so a failure DOES reject (unlike append, whose log is the transaction and
|
||||
// sidecar is best-effort) — but if we mutated state.meta first, a later
|
||||
// touchSummary() on a successful append would persist the rejected
|
||||
// title/firstPrompt, making a failed update durable after the fact.
|
||||
const nextMeta: SessionMeta = { ...state.meta, ...summary }
|
||||
await this.writeSidecar(nextMeta)
|
||||
state.meta = nextMeta
|
||||
}
|
||||
|
||||
// --- materialization / append / repair ---
|
||||
|
||||
/** Atomically write the header line + first batch (temp-write, fsync, rename). */
|
||||
private async materialize(state: SessionState, events: readonly SessionEvent[]): Promise<void> {
|
||||
const dir = sessionDir(this.root, state.meta.cwd)
|
||||
await mkdir(dir, { recursive: true, mode: 0o700 })
|
||||
const finalPath = logPath(this.root, state.meta.cwd, state.meta.id)
|
||||
// Never rename over an existing committed log: materialize is the FIRST
|
||||
// write of a session the backend believes is new. A file here means a
|
||||
// different session shares this id on disk — reject loudly rather than
|
||||
// clobber committed data. (createCore already guards the create path before
|
||||
// this point, so this is unreachable-in-practice defense-in-depth against a
|
||||
// TOCTOU/fork race; ignored for coverage.)
|
||||
/* v8 ignore next 3 -- createCore guards collisions before materialize; this is a TOCTOU backstop */
|
||||
if (await this.exists(finalPath)) {
|
||||
throw new Error(`refusing to materialize "${state.meta.id}": a log already exists on disk (load/resume it instead)`)
|
||||
}
|
||||
const header = JSON.stringify(toHeaderLine(state.meta))
|
||||
const body = events.map(eventLine).join('\n')
|
||||
const content = header + '\n' + body + '\n'
|
||||
|
||||
const tmp = `${finalPath}.${randomBytes(6).toString('hex')}.tmp`
|
||||
const handle = await open(tmp, 'wx', 0o600)
|
||||
try {
|
||||
await handle.writeFile(content)
|
||||
await handle.sync()
|
||||
} finally {
|
||||
await handle.close()
|
||||
}
|
||||
// Publish via link()+unlink(), NOT rename(): link fails with EEXIST if the
|
||||
// final path already exists, so two processes materializing the same id
|
||||
// concurrently cannot clobber each other (both could pass the exists() check
|
||||
// above, but only one link() wins). rename() would silently overwrite the
|
||||
// log the other process just committed. The temp link is always removed,
|
||||
// whether link() succeeds or throws (EEXIST on a race, or any I/O error).
|
||||
try {
|
||||
await link(tmp, finalPath)
|
||||
} finally {
|
||||
await rm(tmp, { force: true })
|
||||
}
|
||||
// fsync the directory so the new entry survives a power loss: on POSIX
|
||||
// filesystems the new link is not crash-durable until the parent directory's
|
||||
// metadata is synced. The seam contract is "append returns once durable",
|
||||
// and materialize is the first append's write — so the directory entry must
|
||||
// be durable before we return.
|
||||
await this.syncDir(dir)
|
||||
state.materialized = true
|
||||
}
|
||||
|
||||
/** fsync a directory so a just-created/renamed entry inside it is crash-durable. */
|
||||
private async syncDir(dir: string): Promise<void> {
|
||||
const handle = await open(dir, 'r')
|
||||
try {
|
||||
await handle.sync()
|
||||
} finally {
|
||||
await handle.close()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Append event lines at EOF and fsync. On a write/sync failure AFTER the
|
||||
* kernel accepted some bytes (ENOSPC, an fsync error), truncate the file back
|
||||
* to its pre-append size before rethrowing: `cursor` is unchanged, so the
|
||||
* batch will be retried, and without this rollback the retry would append
|
||||
* AFTER the partial bytes — producing duplicate seqs that make `scanLog` see a
|
||||
* gap and render the session unloadable.
|
||||
*/
|
||||
private async appendLines(state: SessionState, events: readonly SessionEvent[]): Promise<void> {
|
||||
const path = logPath(this.root, state.meta.cwd, state.meta.id)
|
||||
const handle = await open(path, 'a')
|
||||
try {
|
||||
const { size: before } = await handle.stat()
|
||||
try {
|
||||
await handle.writeFile(events.map(eventLine).join('\n') + '\n')
|
||||
await handle.sync()
|
||||
} catch (error) {
|
||||
// Roll back whatever bytes landed so a retry starts from a clean EOF.
|
||||
await handle.truncate(before)
|
||||
await handle.sync()
|
||||
throw error
|
||||
}
|
||||
} finally {
|
||||
await handle.close()
|
||||
}
|
||||
}
|
||||
|
||||
/** Truncate the log file to `offset` bytes and fsync (discard the crash tail). */
|
||||
private async repair(state: SessionState, offset: number): Promise<void> {
|
||||
const path = logPath(this.root, state.meta.cwd, state.meta.id)
|
||||
await truncate(path, offset)
|
||||
const handle = await open(path, 'r+')
|
||||
try {
|
||||
await handle.sync()
|
||||
} finally {
|
||||
await handle.close()
|
||||
}
|
||||
}
|
||||
|
||||
// --- sidecar (mutable summary) ---
|
||||
|
||||
private async touchSummary(state: SessionState): Promise<void> {
|
||||
state.meta = { ...state.meta, updatedAt: Date.now() }
|
||||
await this.writeSidecar(state.meta)
|
||||
}
|
||||
|
||||
/**
|
||||
* Atomic sidecar write (temp-write + rename), summary fields only.
|
||||
*
|
||||
* Deliberately NOT directory-fsynced (unlike {@link materialize}): the
|
||||
* sidecar holds mutable, recoverable summary metadata (updatedAt, title,
|
||||
* firstPrompt), not source-of-truth log data. The rename is atomic so a
|
||||
* reader never sees a torn file, but a power loss may lose the most recent
|
||||
* summary — acceptable because it is re-derivable and the durable log (the
|
||||
* transaction) is independently synced. Strict crash-durability is reserved
|
||||
* for the event log.
|
||||
*/
|
||||
private async writeSidecar(meta: SessionMeta): Promise<void> {
|
||||
const dir = sessionDir(this.root, meta.cwd)
|
||||
await mkdir(dir, { recursive: true, mode: 0o700 })
|
||||
const path = sidecarPath(this.root, meta.cwd, meta.id)
|
||||
const summary: SessionSummary = {
|
||||
updatedAt: meta.updatedAt,
|
||||
...meta.title !== undefined ? { title: meta.title } : {},
|
||||
...meta.firstPrompt !== undefined ? { firstPrompt: meta.firstPrompt } : {},
|
||||
}
|
||||
const tmp = `${path}.${randomBytes(6).toString('hex')}.tmp`
|
||||
await writeFile(tmp, JSON.stringify(summary), { mode: 0o600 })
|
||||
await rename(tmp, path)
|
||||
}
|
||||
|
||||
/**
|
||||
* Read the mutable-summary sidecar, or `undefined` if it is absent/unreadable
|
||||
* (a session that has never been `update()`d, or a failed sidecar write). The
|
||||
* caller keeps the header-derived `updatedAt` (the session's createdAt) in
|
||||
* that case rather than overlaying `0` — reporting an active session as
|
||||
* updated at the Unix epoch would be wrong.
|
||||
*/
|
||||
private async readSidecar(id: SessionId, cwd: string | undefined): Promise<SessionSummary | undefined> {
|
||||
try {
|
||||
const raw = await readFile(sidecarPath(this.root, cwd, id), 'utf8')
|
||||
return JSON.parse(raw) as SessionSummary
|
||||
} catch {
|
||||
return undefined
|
||||
}
|
||||
}
|
||||
|
||||
// --- discovery helpers ---
|
||||
|
||||
/** Find a session's log file across cwd buckets (when cwd is unknown). */
|
||||
private async findLog(id: SessionId, cwd: string | undefined): Promise<{ path: string; cwd: string | undefined } | undefined> {
|
||||
if (cwd !== undefined) {
|
||||
const path = logPath(this.root, cwd, id)
|
||||
return (await this.exists(path)) ? { path, cwd } : undefined
|
||||
}
|
||||
// Unknown cwd: scan buckets for a matching file name.
|
||||
const target = encodeSegment(id) + '.jsonl'
|
||||
for (const dir of await this.listCwdDirs()) {
|
||||
const path = `${dir}/${target}`
|
||||
if (await this.exists(path)) {
|
||||
// Recover cwd from the header for accurate sidecar pathing.
|
||||
const { meta } = scanLog(await readFile(path))
|
||||
return { path, cwd: meta.cwd }
|
||||
}
|
||||
}
|
||||
return undefined
|
||||
}
|
||||
|
||||
/** The cwd-bucket directories under the root (absolute paths). */
|
||||
private async listCwdDirs(): Promise<string[]> {
|
||||
try {
|
||||
const entries = await readdir(this.root, { withFileTypes: true })
|
||||
return entries.filter(e => e.isDirectory()).map(e => `${this.root}/${e.name}`)
|
||||
} catch {
|
||||
return [] // root does not exist yet → no sessions
|
||||
}
|
||||
}
|
||||
|
||||
private async listJsonl(dir: string): Promise<string[]> {
|
||||
const entries = await readdir(dir)
|
||||
return entries.filter(n => n.endsWith('.jsonl'))
|
||||
}
|
||||
|
||||
private async exists(path: string): Promise<boolean> {
|
||||
try {
|
||||
const handle = await open(path, 'r')
|
||||
await handle.close()
|
||||
return true
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
/** Build a state for a session discovered on disk but not yet in memory. */
|
||||
private async adopt(id: SessionId): Promise<SessionState> {
|
||||
// loadCore (NOT load) — adopt runs inside an already-serialized op, so
|
||||
// re-entering the chain via the public load() would deadlock.
|
||||
await this.loadCore(id)
|
||||
const state = this.states.get(id)
|
||||
/* v8 ignore next -- loadCore always sets the state for the id */
|
||||
if (!state) throw new Error(`failed to adopt session "${id}"`)
|
||||
return state
|
||||
}
|
||||
|
||||
private assertVersion(meta: SessionMeta): void {
|
||||
if (meta.version !== 1) {
|
||||
throw new Error(`unsupported session format version ${meta.version} for "${meta.id}" (only v1 is supported)`)
|
||||
}
|
||||
}
|
||||
|
||||
// --- write path (session/event → flush drain) ---
|
||||
|
||||
private installWritePath(): void {
|
||||
const ctx = this.ctx
|
||||
|
||||
// Capture the header on creation; persist a fork's seed once. Record the
|
||||
// init promise so flush/dispose can await it (onCreated is async).
|
||||
ctx.on('session/created', (session) => { void this.initFor(session) })
|
||||
|
||||
// Snapshot + buffer every event (the live object is mutable; clone so a
|
||||
// later in-place mutation of session.events cannot rewrite a buffered
|
||||
// event). Serializability is guaranteed at the source — `Session.append`
|
||||
// rejects non-JSON-serializable data before the event ever enters the log
|
||||
// or this emit — so structuredClone here can never hit a non-cloneable
|
||||
// value, and the durable log can never diverge from session.events.
|
||||
ctx.on('session/event', (session, event) => {
|
||||
let buffer = this.buffers.get(session)
|
||||
if (!buffer) this.buffers.set(session, buffer = [])
|
||||
buffer.push(structuredClone(event))
|
||||
})
|
||||
|
||||
// Drain to the backend at the durability checkpoint.
|
||||
ctx.on('session/flush', session => this.flush(session))
|
||||
|
||||
// Dispose must reach quiescence: await every session's init + final drain
|
||||
// BEFORE returning, so no write lands after teardown (orphan rename/ENOENT).
|
||||
ctx.effect(() => async () => {
|
||||
await Promise.allSettled([...this.inits.values()])
|
||||
await Promise.allSettled([...this.buffers.keys()].map(s => this.flush(s)))
|
||||
await Promise.allSettled([...this.chains.values()])
|
||||
}, 'session-persistence-jsonl write path')
|
||||
|
||||
// HMR: a hot reload does not replay session/created, so seed existing live
|
||||
// sessions (mirrors dsh-invariants).
|
||||
for (const session of ctx.sessions.list()) void this.initFor(session)
|
||||
}
|
||||
|
||||
/** Start (once) the async init for a session and remember its promise. */
|
||||
private initFor(session: Session): Promise<void> {
|
||||
const existing = this.inits.get(session)
|
||||
if (existing) return existing
|
||||
// Snapshot the seed SYNCHRONOUSLY here — initFor runs inside the
|
||||
// `session/created` emit, before any later `append` adds non-seed events.
|
||||
// A clone freezes it against later mutation of the live event objects.
|
||||
const seed = session.events.map(e => structuredClone(e))
|
||||
const p = this.onCreated(session, seed)
|
||||
// Attach a no-op rejection handler so a failing init (e.g. an id collision)
|
||||
// does not surface as an unhandled rejection if no flush observes `p` before
|
||||
// it rejects. The REAL error is still delivered: flush/dispose await the
|
||||
// same `p` from the map and see the rejection there.
|
||||
p.catch(() => { /* observed by flush/dispose via the stored promise */ })
|
||||
this.inits.set(session, p)
|
||||
return p
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether a live `session`'s `seed` reproduces the first `cursor` persisted
|
||||
* events. Reads the on-disk committed prefix and compares. A `cursor` of 0
|
||||
* (nothing persisted yet) trivially matches. Used when a live session claims
|
||||
* ownerless state left by a prior `load()`/`create()` — to reject a fresh,
|
||||
* unrelated session that reuses the id and would otherwise have its seq
|
||||
* 0..cursor-1 events filtered as already-written.
|
||||
*/
|
||||
private async seedMatchesPersisted(session: Session, seed: readonly SessionEvent[], cursor: number): Promise<boolean> {
|
||||
if (cursor === 0) return true
|
||||
const onDisk = await this.findLog(session.header.id, session.header.cwd)
|
||||
/* v8 ignore next -- a cursor > 0 means the log was materialized, so it exists */
|
||||
if (onDisk === undefined) return false
|
||||
const { events: diskEvents } = scanLog(await readFile(onDisk.path))
|
||||
return seedCoversPrefix(seed, diskEvents.slice(0, cursor))
|
||||
}
|
||||
|
||||
/**
|
||||
* On session/created: sync the backend's in-memory state to a live Session.
|
||||
*
|
||||
* Cases, by whether this backend tracks the id and whether a log is on disk:
|
||||
* 1. Already in `states` (created here, or a prior load/resume) → no-op.
|
||||
* 2. Not tracked, a log EXISTS on disk, and it is a seq-aligned PREFIX of the
|
||||
* live session's current events → ADOPT it (HMR/reload): a fresh backend
|
||||
* instance (empty `states`) meets a live session whose log a previous
|
||||
* instance materialized; the live object already carries that history (it
|
||||
* is the source of truth this run), so we continue from the stored length
|
||||
* instead of re-creating. This keeps persistence alive across hot reload.
|
||||
* 3. Not tracked, a log EXISTS on disk, but it is NOT a prefix of the live
|
||||
* session's events → REJECT: a different session collides on the id. The
|
||||
* SessionId is the identity, so two unrelated sessions sharing one is a
|
||||
* bug, not a resume — fail loudly rather than clobber committed data.
|
||||
* 4. Not tracked and NO log on disk → a genuinely new session: register its
|
||||
* meta (lazy) and persist its `seed` once.
|
||||
*
|
||||
* The public `create(meta)` API is stricter still (rejects ANY on-disk id):
|
||||
* there the caller asserts "brand new", so even a prefix match is a bug.
|
||||
*
|
||||
* The seed events were copied into the Session by its constructor WITHOUT
|
||||
* emitting session/event, so the write-behind buffer never sees them — the
|
||||
* one explicit `append(seed)` below is the only persistence of the seed.
|
||||
* Events appended AFTER creation flow through the session/event buffer and
|
||||
* are persisted by flush (filtered by the write cursor), never here.
|
||||
*/
|
||||
private async onCreated(session: Session, seed: readonly SessionEvent[]): Promise<void> {
|
||||
const id = session.header.id
|
||||
const tracked = this.states.get(id)
|
||||
if (tracked !== undefined) {
|
||||
// case 1: already tracked.
|
||||
// (owner === session is a defensive same-object guard: initFor dedupes by
|
||||
// session object, so onCreated never actually runs twice for one session.)
|
||||
/* v8 ignore next -- initFor dedupes per session object; same-object re-entry can't occur */
|
||||
if (tracked.owner === session) return
|
||||
if (tracked.owner === undefined) {
|
||||
// Ownerless state was created via the public create()/load() API. The
|
||||
// FIRST live session to arrive claims it — but ONLY if its seed is the
|
||||
// already-persisted prefix. A load() for preview leaves cursor at the
|
||||
// persisted length; a fresh, unrelated session reusing that id has a
|
||||
// seed shorter than (or not matching) that prefix, so flush would filter
|
||||
// its seq 0..cursor-1 events as already-written and silently graft the
|
||||
// new conversation onto the old log. Verify the seed covers the cursor.
|
||||
if (!await this.seedMatchesPersisted(session, seed, tracked.cursor)) {
|
||||
throw new Error(`session "${id}" is already persisted with ${tracked.cursor} event(s) that do not match this live session (id collision)`)
|
||||
}
|
||||
tracked.owner = session
|
||||
// Persist the live seed SUFFIX beyond the persisted prefix. Constructor
|
||||
// seed events (from sessions.create(id, { seed })) never emit
|
||||
// session/event, so the write-behind buffer never sees them — without
|
||||
// this they would be lost and a later flush would seq-mismatch. (cursor
|
||||
// is 0 for a public create(), so this covers the whole seed there.)
|
||||
const suffix = seed.slice(tracked.cursor)
|
||||
if (suffix.length > 0) await this.append(id, suffix)
|
||||
return
|
||||
}
|
||||
// The state is owned by a DIFFERENT live session. We may reclaim the id
|
||||
// ONLY if that owner left nothing behind: never materialized a log (cursor
|
||||
// 0, not materialized) AND has no write-behind buffer still pending. A
|
||||
// session that appended events but was disposed before its first flush is
|
||||
// NOT materialized yet but DOES have buffered events — reclaiming then
|
||||
// would let that stale buffer drain against the new session's state
|
||||
// (persisting old events under the new id, or dropping the new session's
|
||||
// seq-0 events). Such an owner, and any materialized owner, is a real
|
||||
// collision and rejects; only a truly-abandoned (artifact-free) id is
|
||||
// freed, honoring lazy materialization's "leaves nothing behind" promise.
|
||||
const ownerBuffer = this.buffers.get(tracked.owner)
|
||||
if (!tracked.materialized && !ownerBuffer?.length) {
|
||||
this.states.delete(id)
|
||||
} else {
|
||||
throw new Error(`session "${id}" is already bound to a different live session in this backend (id collision)`)
|
||||
}
|
||||
}
|
||||
|
||||
const onDisk = await this.findLog(id, session.header.cwd)
|
||||
if (onDisk !== undefined) {
|
||||
// Read the committed on-disk events and check they are a seq-aligned
|
||||
// prefix of the live session (HMR re-seeing its own session) vs. an
|
||||
// unrelated session colliding on the id.
|
||||
const { events: diskEvents } = scanLog(await readFile(onDisk.path))
|
||||
if (!seedCoversPrefix(seed, diskEvents)) {
|
||||
// case 3: genuine collision — fail loudly rather than clobber.
|
||||
throw new Error(`session "${id}" already has a persisted log on disk that does not match this live session (id collision)`)
|
||||
}
|
||||
// case 2: adopt. loadCore sets the state (cursor = committed length,
|
||||
// repair offset if a crash tail exists).
|
||||
await this.serialize(id, () => this.loadCore(id))
|
||||
const adopted = this.states.get(id)
|
||||
/* v8 ignore next -- loadCore always sets the state for the id */
|
||||
if (adopted !== undefined) adopted.owner = session
|
||||
// Persist the live SUFFIX beyond the on-disk prefix. These events live
|
||||
// ONLY in `seed` (the live session was ahead of disk — mid-turn at
|
||||
// reload, or events appended while the previous backend was disposed);
|
||||
// this backend never buffered them via session/event, so without this
|
||||
// they would be lost and the next flush (starting at a later seq) would
|
||||
// mismatch or skip them.
|
||||
const suffix = seed.slice(diskEvents.length)
|
||||
if (suffix.length > 0) await this.append(id, suffix)
|
||||
return
|
||||
}
|
||||
|
||||
// case 4: a genuinely new session. Register its meta (lazy), then persist
|
||||
// its seed (events present at creation time) once.
|
||||
const meta: SessionMeta = { ...session.header, updatedAt: Date.now() }
|
||||
await this.create(meta)
|
||||
// Bind this state to the live session so a later DIFFERENT session reusing
|
||||
// the id is detected as a collision (case 1) rather than silently no-opped.
|
||||
const created = this.states.get(id)
|
||||
/* v8 ignore next -- create() always sets the state for the id */
|
||||
if (created !== undefined) created.owner = session
|
||||
if (seed.length > 0) {
|
||||
await this.append(id, seed)
|
||||
}
|
||||
}
|
||||
|
||||
private async flush(session: Session): Promise<void> {
|
||||
// Wait for the session's init (onCreated) to finish so the state/cursor and
|
||||
// any fork-seed persistence are in place before we drain. Awaiting the same
|
||||
// promise initFor stored also surfaces an init failure (e.g. an id
|
||||
// collision) here, where the caller of session/flush observes it.
|
||||
await this.inits.get(session)
|
||||
// Serialize the WHOLE drain (read cursor → append → splice) on the
|
||||
// per-session chain. Two concurrent flushes (e.g. an idle inject()'s
|
||||
// fire-and-forget flush racing an explicit checkpoint) would otherwise both
|
||||
// read the same cursor, both compute the same `fresh` slice, and the second
|
||||
// append would seq-mismatch against the cursor the first already advanced.
|
||||
await this.serialize(session.header.id, () => this.drain(session))
|
||||
}
|
||||
|
||||
/** Drain a session's write buffer to disk. Caller serializes this per id. */
|
||||
private async drain(session: Session): Promise<void> {
|
||||
const buffer = this.buffers.get(session)
|
||||
if (!buffer?.length) return
|
||||
// Copy WITHOUT removing: the buffer is the only durable-pending copy of
|
||||
// these events (session/event does not re-emit). Splicing before the append
|
||||
// means a failed append (disk error, or a seq mismatch after a dropped bad
|
||||
// event) permanently loses a completed turn. Drain the buffer only AFTER
|
||||
// the append commits; events pushed during the await sit past batch.length
|
||||
// and survive the prefix splice, so a retry/dispose re-drains the rest.
|
||||
const batch = buffer.slice()
|
||||
const state = this.states.get(session.header.id)
|
||||
// Only append events at or beyond the write cursor (a resumed session's
|
||||
// seed is already on disk; the cursor was set to the loaded length). flush
|
||||
// awaits the init above, which always sets state, so the `?? 0` fallback is
|
||||
// a defensive guard that never fires in practice.
|
||||
/* v8 ignore next -- state is always set by the awaited init before flush */
|
||||
const cursor = state?.cursor ?? 0
|
||||
const fresh = batch.filter(e => e.seq >= cursor)
|
||||
// appendCore (NOT the serialized append) — drain already runs inside the
|
||||
// per-session chain, so re-entering it via append() would deadlock.
|
||||
if (fresh.length > 0) await this.appendCore(session.header.id, fresh)
|
||||
buffer.splice(0, batch.length)
|
||||
}
|
||||
}
|
||||
|
||||
export default SessionPersistenceJsonl
|
||||
1168
packages/session-persistence-jsonl/tests/jsonl.spec.ts
Normal file
1168
packages/session-persistence-jsonl/tests/jsonl.spec.ts
Normal file
File diff suppressed because it is too large
Load Diff
15
packages/session-persistence-jsonl/tsconfig.json
Normal file
15
packages/session-persistence-jsonl/tsconfig.json
Normal file
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"extends": "../../tsconfig.base.json",
|
||||
"compilerOptions": {
|
||||
"rootDir": "src",
|
||||
"outDir": "lib"
|
||||
},
|
||||
"include": ["src"],
|
||||
"references": [
|
||||
{ "path": "../../vendor/cosmokit" },
|
||||
{ "path": "../../vendor/cordis" },
|
||||
{ "path": "../../vendor/schemastery" },
|
||||
{ "path": "../session" },
|
||||
{ "path": "../session-persistence" }
|
||||
]
|
||||
}
|
||||
33
packages/session-persistence/README.md
Normal file
33
packages/session-persistence/README.md
Normal file
@@ -0,0 +1,33 @@
|
||||
# @deepseek-ai/dsh-session-persistence
|
||||
|
||||
The abstract durable session-persistence seam (`ctx.sessionPersistence`). Defines WHAT a persistence backend does — durably store, reload, list, and update sessions — without saying HOW. Mirrors the `dsh-bash` capability-seam template ([ADR 0009](../../docs/adr/0009-capability-seams.md)): an abstract service here, a concrete implementation in a sibling package, consumers that inject the interface.
|
||||
|
||||
The persisted unit IS the existing `SessionEvent` (event-sourced model — the log is the single source of truth), so there is no parallel "persisted message" type. Metadata that is NOT replayable conversation state (format version, cwd, lineage) travels separately as `SessionMeta`, owned by `dsh-session` and re-exported here.
|
||||
|
||||
## Service API (`ctx.sessionPersistence`)
|
||||
|
||||
| Method | Contract |
|
||||
|---|---|
|
||||
| `create(meta): Promise<void>` | Register a new session's metadata. MAY defer the physical write until the first `append` (lazy materialization). |
|
||||
| `append(id, events): Promise<void>` | Durably persist a batch (from the `session/flush` drain). Append-only; first event `seq` == stored next-seq after any repair; rejects non-JSON-serializable data naming the offending type. |
|
||||
| `load(id): Promise<{ meta; events }>` | Reload meta + log up to the last complete `turn/end`; events contiguous (`events[i].seq === i`); rejects a mid-log gap/parse error or unknown `version`. |
|
||||
| `list(): Promise<SessionMeta[]>` | Lightweight listing from metadata, no full-log parse. |
|
||||
| `has(id)` / `delete(id)` | Existence / removal. A zero-event lazily-materialized session is absent from `has`/`list`. |
|
||||
| `update(id, summary): Promise<void>` | Update mutable `SessionSummary` fields without touching the append-only log. |
|
||||
|
||||
## Invariants every backend must honor
|
||||
|
||||
- **Append-only.** Committed events (at or below a flushed `turn/end`) are never rewritten. The only exception is the one-time truncation-repair of a never-committed crash tail on the first `append` after a `load`.
|
||||
- **Contiguous seq.** `load` rejects a `seq` gap/parse error in the MIDDLE of the log; `append`'s first `seq` must equal the stored next-seq.
|
||||
- **JSON-serializable data.** `append` rejects non-serializable `event.data`; backends snapshot each event when buffering (the live `session.events` object is mutable).
|
||||
- **Durability.** `append` returns only once the batch is durable.
|
||||
|
||||
## Testing backends
|
||||
|
||||
Import `runPersistenceContract` from `tests/contract.ts` and call it with a factory that yields a fresh, empty backend plus a teardown. Every backend is held to the same append-only / contiguous-seq / lazy-materialization / serializability semantics; a backend's own spec adds implementation-specific tests (crash repair, path sanitization) on top.
|
||||
|
||||
> **TODO (validate the abstraction with a second backend):** `dsh-session-persistence-jsonl` is currently the only implementation, so the interface and `runPersistenceContract` are only proven against one storage model. A second backend — a SQLite implementation (`dsh-session-persistence-sqlite`), where each `SessionEvent` maps 1:1 onto a row `(session_id, seq, type, time, data)` — would run the SAME `runPersistenceContract` suite and so prove the seam is genuinely backend-agnostic (lazy materialization, crash-tail-on-load, contiguous-seq all expressed against a transactional store rather than an append-only file).
|
||||
|
||||
## Metadata types
|
||||
|
||||
Re-exported from `dsh-session`: `SessionHeader` (immutable: `version`, `id`, `createdAt`, `cwd?`, `parentSession?`), `SessionSummary` (mutable: `updatedAt`, `title?`, `firstPrompt?`), `SessionMeta` (their intersection).
|
||||
30
packages/session-persistence/package.json
Normal file
30
packages/session-persistence/package.json
Normal file
@@ -0,0 +1,30 @@
|
||||
{
|
||||
"name": "@deepseek-ai/dsh-session-persistence",
|
||||
"description": "Abstract durable session persistence seam (ctx.sessionPersistence) for the DeepSeek Harness",
|
||||
"version": "0.0.1",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"main": "lib/index.js",
|
||||
"types": "lib/index.d.ts",
|
||||
"exports": {
|
||||
".": {
|
||||
"types": "./lib/index.d.ts",
|
||||
"default": "./lib/index.js"
|
||||
},
|
||||
"./src/*": "./src/*",
|
||||
"./package.json": "./package.json"
|
||||
},
|
||||
"files": [
|
||||
"lib",
|
||||
"src"
|
||||
],
|
||||
"license": "BSD-3-Clause",
|
||||
"peerDependencies": {
|
||||
"@deepseek-ai/dsh-session": "^0.0.1",
|
||||
"cordis": "^4.0.0-rc.6"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@deepseek-ai/dsh-session": "^0.0.1",
|
||||
"cordis": "^4.0.0-rc.6"
|
||||
}
|
||||
}
|
||||
114
packages/session-persistence/src/index.ts
Normal file
114
packages/session-persistence/src/index.ts
Normal file
@@ -0,0 +1,114 @@
|
||||
/**
|
||||
* The durable session-persistence seam (`ctx.sessionPersistence`): an abstract
|
||||
* service defining WHAT a persistence backend does — durably store, reload,
|
||||
* list, and update sessions — without saying HOW. Implementations subclass
|
||||
* {@link SessionPersistence} and register themselves as the
|
||||
* `sessionPersistence` service; `@deepseek-ai/dsh-session-persistence-jsonl`
|
||||
* (an append-only JSONL log per session) is the first. Future backends swap in
|
||||
* SQLite/WAL, an object store, or a remote service without touching the
|
||||
* consumers (the write-path plugin, the agent-loop resume seam).
|
||||
*
|
||||
* The persisted unit IS the existing {@link SessionEvent} — there is no
|
||||
* parallel "persisted message" type the log must be converted to and from
|
||||
* (faithful to the event-sourced model: the log is the single source of
|
||||
* truth). Metadata that is NOT replayable conversation state (format version,
|
||||
* cwd, lineage) travels separately as {@link SessionMeta}, which is owned by
|
||||
* `dsh-session` and re-exported here.
|
||||
*
|
||||
* @module @deepseek-ai/dsh-session-persistence
|
||||
*/
|
||||
|
||||
import { Context, Service } from 'cordis'
|
||||
import type { SessionEvent, SessionId, SessionMeta, SessionSummary } from '@deepseek-ai/dsh-session'
|
||||
|
||||
// Re-export the metadata vocabulary so consumers import it from the seam.
|
||||
export type { SessionHeader, SessionSummary, SessionMeta } from '@deepseek-ai/dsh-session'
|
||||
|
||||
declare module 'cordis' {
|
||||
interface Context {
|
||||
sessionPersistence: SessionPersistence
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Abstract durable session-persistence service. Subclass, implement the
|
||||
* abstract methods, and load the subclass as a plugin — it registers as
|
||||
* `ctx.sessionPersistence` (one implementation per context; loading a second
|
||||
* throws, cordis' standard duplicate-service behavior).
|
||||
*
|
||||
* Contracts every implementation MUST honor (a DB backend asserts them inside
|
||||
* a transaction; a file backend appends at EOF):
|
||||
*
|
||||
* - **Append-only.** Committed events — those at or below a flushed `turn/end`
|
||||
* — are never rewritten. The ONLY exception is the one-time truncation-repair
|
||||
* of a never-committed crash tail on the first {@link append} after a
|
||||
* {@link load} (see {@link load}).
|
||||
* - **Contiguous seq.** A persisted log is contiguous: `events[i].seq === i`.
|
||||
* {@link load} rejects a parse error or a `seq` gap in the MIDDLE of the log
|
||||
* (unloadable); {@link append}'s first event `seq` MUST equal the backend's
|
||||
* stored next-seq after any repair.
|
||||
* - **JSON-serializable data.** `SessionEventMap` is merge-extensible and
|
||||
* `event.data` is typed only as `SessionEventMap[K]`, so {@link append}
|
||||
* REJECTS non-JSON-serializable data with an error naming the offending
|
||||
* event type. A backend snapshots (serializes/clones) each event when it
|
||||
* buffers, since `session.events` hands out the live mutable object.
|
||||
* - **Durability.** {@link append} returns only once the batch is durable
|
||||
* (the file backend fsyncs; a DB commits). {@link create} MAY defer the
|
||||
* physical write until the first {@link append} (lazy materialization).
|
||||
*/
|
||||
export abstract class SessionPersistence extends Service {
|
||||
constructor(ctx: Context) {
|
||||
super(ctx, 'sessionPersistence')
|
||||
}
|
||||
|
||||
/**
|
||||
* Register a new session's metadata. A backend MAY defer the physical write
|
||||
* until the first {@link append} (lazy materialization), in which case a
|
||||
* created-but-never-appended session is absent from {@link has}/{@link list}
|
||||
* — abandoned sessions leave nothing behind.
|
||||
*/
|
||||
abstract create(meta: SessionMeta): Promise<void>
|
||||
|
||||
/**
|
||||
* Durably persist a batch of events (called from the write-behind drain at
|
||||
* the `session/flush` checkpoint). Honors the append-only and contiguous-seq
|
||||
* contracts: the first event's `seq` MUST equal the stored next-seq after
|
||||
* any truncation-repair of a crash tail. Rejects non-JSON-serializable
|
||||
* `event.data` with an error naming the offending event type.
|
||||
*/
|
||||
abstract append(id: SessionId, events: readonly SessionEvent[]): Promise<void>
|
||||
|
||||
/**
|
||||
* Reload a session: its {@link SessionMeta} plus the event log up to the last
|
||||
* durable checkpoint. Returns `meta` AND `events` so the live session is
|
||||
* reconstructed with its `cwd`/lineage, not just its log.
|
||||
*
|
||||
* The loop only flushes at `turn/end`, so a crash can leave a half-written
|
||||
* final turn below the last committed checkpoint. `load` returns events only
|
||||
* up to the **last complete `turn/end`**; a subsequent {@link append} runs
|
||||
* the one-time truncation-repair that physically discards the orphaned tail
|
||||
* before writing. Returned events are contiguous (`events[i].seq === i`); a
|
||||
* parse error or a `seq` gap in the MIDDLE of the log makes the session
|
||||
* unloadable (reject). Rejects an unknown format `version`.
|
||||
*/
|
||||
abstract load(id: SessionId): Promise<{ meta: SessionMeta; events: SessionEvent[] }>
|
||||
|
||||
/** Lightweight listing from metadata, without a full-log parse. */
|
||||
abstract list(): Promise<SessionMeta[]>
|
||||
|
||||
/** Whether a session is durably present (materialized). */
|
||||
abstract has(id: SessionId): Promise<boolean>
|
||||
|
||||
/** Remove a session and all its persisted artifacts. */
|
||||
abstract delete(id: SessionId): Promise<void>
|
||||
|
||||
/**
|
||||
* Update mutable metadata ({@link SessionSummary}: `updatedAt`, `title`,
|
||||
* `firstPrompt`) WITHOUT touching the append-only event log. A backend
|
||||
* stores the summary beside the log (a sidecar file, a header row) and
|
||||
* rewrites only it.
|
||||
*/
|
||||
abstract update(id: SessionId, summary: Partial<SessionSummary>): Promise<void>
|
||||
}
|
||||
|
||||
export default SessionPersistence
|
||||
167
packages/session-persistence/tests/contract.ts
Normal file
167
packages/session-persistence/tests/contract.ts
Normal file
@@ -0,0 +1,167 @@
|
||||
/**
|
||||
* Reusable contract test for any {@link SessionPersistence} backend. A backend
|
||||
* package imports {@link runPersistenceContract} and calls it with a factory
|
||||
* that yields a fresh, empty backend (and a teardown), so every backend is held
|
||||
* to the same append-only / contiguous-seq / lazy-materialization / crash
|
||||
* semantics. The JSONL backend's own spec adds file-specific tests on top.
|
||||
*
|
||||
* @module @deepseek-ai/dsh-session-persistence/tests/contract
|
||||
*/
|
||||
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { SessionId } from '@deepseek-ai/dsh-session'
|
||||
import type { SessionEvent, SessionMeta } from '@deepseek-ai/dsh-session'
|
||||
import type { SessionPersistence } from '../src/index.ts'
|
||||
|
||||
/** A backend under test plus its teardown. */
|
||||
export interface ContractBackend {
|
||||
persistence: SessionPersistence
|
||||
dispose: () => Promise<void>
|
||||
}
|
||||
|
||||
/** Build a minimal {@link SessionMeta} for a session id. */
|
||||
export function meta(id: string, cwd?: string): SessionMeta {
|
||||
return {
|
||||
version: 1,
|
||||
id: SessionId(id),
|
||||
createdAt: 1000,
|
||||
updatedAt: 1000,
|
||||
...cwd !== undefined ? { cwd } : {},
|
||||
}
|
||||
}
|
||||
|
||||
/** A well-formed one-turn event log (contiguous seqs from 0). */
|
||||
export function oneTurnLog(): SessionEvent[] {
|
||||
return [
|
||||
{ type: 'turn/start', seq: 0, time: 1, data: { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } } },
|
||||
{ type: 'user/message', seq: 1, time: 2, data: { content: [{ type: 'text', text: 'hi' }], source: { kind: 'user' } } },
|
||||
{ type: 'step/start', seq: 2, time: 3, data: { turn: 1, step: 1 } },
|
||||
{ type: 'assistant/message', seq: 3, time: 4, data: { turn: 1, step: 1, content: [{ type: 'text', text: 'hello' }] } },
|
||||
{ type: 'step/end', seq: 4, time: 5, data: { turn: 1, step: 1 } },
|
||||
{ type: 'turn/end', seq: 5, time: 6, data: { turn: 1, reason: { kind: 'completed' } } },
|
||||
]
|
||||
}
|
||||
|
||||
/**
|
||||
* Run the backend-agnostic contract suite. `make()` MUST return a fresh, empty
|
||||
* backend each call.
|
||||
*/
|
||||
export function runPersistenceContract(name: string, make: () => Promise<ContractBackend>): void {
|
||||
describe(`SessionPersistence contract: ${name}`, () => {
|
||||
it('round-trips a session: create + append → load returns identical meta and byte-identical events', async () => {
|
||||
const { persistence, dispose } = await make()
|
||||
try {
|
||||
const m = meta('s1', '/work')
|
||||
const log = oneTurnLog()
|
||||
await persistence.create(m)
|
||||
await persistence.append(m.id, log)
|
||||
|
||||
const loaded = await persistence.load(m.id)
|
||||
expect(loaded.meta).toMatchObject({ version: 1, id: m.id, cwd: '/work' })
|
||||
expect(loaded.events).toEqual(log)
|
||||
} finally {
|
||||
await dispose()
|
||||
}
|
||||
})
|
||||
|
||||
it('has()/list() exclude a created-but-never-appended (zero-event) session', async () => {
|
||||
const { persistence, dispose } = await make()
|
||||
try {
|
||||
await persistence.create(meta('empty'))
|
||||
expect(await persistence.has(SessionId('empty'))).toBe(false)
|
||||
expect((await persistence.list()).map(m => m.id)).not.toContain(SessionId('empty'))
|
||||
} finally {
|
||||
await dispose()
|
||||
}
|
||||
})
|
||||
|
||||
it('has()/list() include a session once it has events', async () => {
|
||||
const { persistence, dispose } = await make()
|
||||
try {
|
||||
const m = meta('s2')
|
||||
await persistence.create(m)
|
||||
await persistence.append(m.id, oneTurnLog())
|
||||
expect(await persistence.has(m.id)).toBe(true)
|
||||
expect((await persistence.list()).map(x => x.id)).toContain(m.id)
|
||||
} finally {
|
||||
await dispose()
|
||||
}
|
||||
})
|
||||
|
||||
it('append rejects a batch whose first seq does not match the stored next-seq', async () => {
|
||||
const { persistence, dispose } = await make()
|
||||
try {
|
||||
const m = meta('s3')
|
||||
await persistence.create(m)
|
||||
await persistence.append(m.id, oneTurnLog()) // seqs 0..5, next-seq = 6
|
||||
// A re-append of an already-stored seq must be rejected, not duplicated.
|
||||
const restated = oneTurnLog()
|
||||
await expect(persistence.append(m.id, restated)).rejects.toThrow()
|
||||
} finally {
|
||||
await dispose()
|
||||
}
|
||||
})
|
||||
|
||||
it('append rejects a mid-batch seq gap', async () => {
|
||||
const { persistence, dispose } = await make()
|
||||
try {
|
||||
const m = meta('s4')
|
||||
await persistence.create(m)
|
||||
const gapped: SessionEvent[] = [
|
||||
{ type: 'turn/start', seq: 0, time: 1, data: { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } } },
|
||||
{ type: 'step/start', seq: 2, time: 2, data: { turn: 1, step: 1 } }, // gap: missing seq 1
|
||||
]
|
||||
await expect(persistence.append(m.id, gapped)).rejects.toThrow()
|
||||
} finally {
|
||||
await dispose()
|
||||
}
|
||||
})
|
||||
|
||||
it('append rejects non-JSON-serializable event data, naming the event type', async () => {
|
||||
const { persistence, dispose } = await make()
|
||||
try {
|
||||
const m = meta('s5')
|
||||
await persistence.create(m)
|
||||
// A plugin-added event carrying a BigInt (not JSON-serializable).
|
||||
const bad = [
|
||||
{ type: 'user/message', seq: 0, time: 1, data: { content: [{ type: 'text', text: 'x' }], source: { kind: 'user' }, extra: 1n } },
|
||||
] as unknown as SessionEvent[]
|
||||
await expect(persistence.append(m.id, bad)).rejects.toThrow(/user\/message/)
|
||||
} finally {
|
||||
await dispose()
|
||||
}
|
||||
})
|
||||
|
||||
it('delete removes a session', async () => {
|
||||
const { persistence, dispose } = await make()
|
||||
try {
|
||||
const m = meta('s6')
|
||||
await persistence.create(m)
|
||||
await persistence.append(m.id, oneTurnLog())
|
||||
expect(await persistence.has(m.id)).toBe(true)
|
||||
await persistence.delete(m.id)
|
||||
expect(await persistence.has(m.id)).toBe(false)
|
||||
} finally {
|
||||
await dispose()
|
||||
}
|
||||
})
|
||||
|
||||
it('update mutates summary fields without touching the event log', async () => {
|
||||
const { persistence, dispose } = await make()
|
||||
try {
|
||||
const m = meta('s7')
|
||||
const log = oneTurnLog()
|
||||
await persistence.create(m)
|
||||
await persistence.append(m.id, log)
|
||||
await persistence.update(m.id, { title: 'My session', firstPrompt: 'hi' })
|
||||
|
||||
const loaded = await persistence.load(m.id)
|
||||
expect(loaded.meta.title).toBe('My session')
|
||||
expect(loaded.meta.firstPrompt).toBe('hi')
|
||||
expect(loaded.events).toEqual(log) // log untouched
|
||||
} finally {
|
||||
await dispose()
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
110
packages/session-persistence/tests/persistence.spec.ts
Normal file
110
packages/session-persistence/tests/persistence.spec.ts
Normal file
@@ -0,0 +1,110 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { Context } from 'cordis'
|
||||
import { SessionId } from '@deepseek-ai/dsh-session'
|
||||
import type { SessionEvent, SessionMeta, SessionSummary } from '@deepseek-ai/dsh-session'
|
||||
import { SessionPersistence } from '../src/index.ts'
|
||||
import { runPersistenceContract, meta, oneTurnLog } from './contract.ts'
|
||||
|
||||
/**
|
||||
* A minimal in-memory {@link SessionPersistence} used to (a) cover the abstract
|
||||
* base's constructor + service registration and (b) validate the reusable
|
||||
* contract suite itself. The real durable backend is
|
||||
* `@deepseek-ai/dsh-session-persistence-jsonl`.
|
||||
*/
|
||||
class MemoryPersistence extends SessionPersistence {
|
||||
private store = new Map<string, { meta: SessionMeta; events: SessionEvent[] }>()
|
||||
private pending = new Map<string, SessionMeta>()
|
||||
|
||||
async create(m: SessionMeta): Promise<void> {
|
||||
// Lazy: record the intended meta, but stay absent from has/list until the
|
||||
// first append materializes the session.
|
||||
this.pending.set(m.id, m)
|
||||
}
|
||||
|
||||
async append(id: SessionId, events: readonly SessionEvent[]): Promise<void> {
|
||||
const existing = this.store.get(id)
|
||||
const nextSeq = existing ? existing.events.length : 0
|
||||
if (events.length > 0 && events[0]!.seq !== nextSeq) {
|
||||
throw new Error(`append seq mismatch for "${id}": expected ${nextSeq}, got ${events[0]!.seq}`)
|
||||
}
|
||||
for (let i = 0; i < events.length; i++) {
|
||||
const e = events[i]!
|
||||
if (e.seq !== nextSeq + i) throw new Error(`non-contiguous seq in batch for "${id}" at index ${i}`)
|
||||
if (containsNonSerializable(e.data)) {
|
||||
throw new Error(`event "${e.type}" carries non-JSON-serializable data`)
|
||||
}
|
||||
}
|
||||
if (!existing) {
|
||||
const m = this.pending.get(id)
|
||||
if (!m) throw new Error(`append before create for "${id}"`)
|
||||
this.store.set(id, { meta: m, events: structuredClone(events) as SessionEvent[] })
|
||||
} else {
|
||||
existing.events.push(...structuredClone(events) as SessionEvent[])
|
||||
}
|
||||
}
|
||||
|
||||
async load(id: SessionId): Promise<{ meta: SessionMeta; events: SessionEvent[] }> {
|
||||
const entry = this.store.get(id)
|
||||
if (!entry) throw new Error(`session "${id}" not found`)
|
||||
return { meta: structuredClone(entry.meta), events: structuredClone(entry.events) }
|
||||
}
|
||||
|
||||
async list(): Promise<SessionMeta[]> {
|
||||
return [...this.store.values()].map(e => structuredClone(e.meta))
|
||||
}
|
||||
|
||||
async has(id: SessionId): Promise<boolean> {
|
||||
return this.store.has(id)
|
||||
}
|
||||
|
||||
async delete(id: SessionId): Promise<void> {
|
||||
this.store.delete(id)
|
||||
this.pending.delete(id)
|
||||
}
|
||||
|
||||
async update(id: SessionId, summary: Partial<SessionSummary>): Promise<void> {
|
||||
const entry = this.store.get(id)
|
||||
if (entry) Object.assign(entry.meta, summary)
|
||||
}
|
||||
}
|
||||
|
||||
/** Detect BigInt (and other JSON-hostile values) in event data. */
|
||||
function containsNonSerializable(value: unknown): boolean {
|
||||
if (typeof value === 'bigint' || typeof value === 'function' || typeof value === 'symbol') return true
|
||||
if (value && typeof value === 'object') {
|
||||
return Object.values(value).some(containsNonSerializable)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Run the shared contract against the in-memory backend.
|
||||
runPersistenceContract('memory', async () => {
|
||||
const ctx = new Context()
|
||||
const fiber = await ctx.plugin(MemoryPersistence)
|
||||
return {
|
||||
persistence: ctx.sessionPersistence,
|
||||
dispose: async () => { await fiber.dispose() },
|
||||
}
|
||||
})
|
||||
|
||||
describe('SessionPersistence service registration', () => {
|
||||
it('registers as ctx.sessionPersistence and is removed on fiber dispose (HMR safety)', async () => {
|
||||
const ctx = new Context()
|
||||
const fiber = await ctx.plugin(MemoryPersistence)
|
||||
expect(ctx.sessionPersistence).toBeInstanceOf(SessionPersistence)
|
||||
|
||||
await fiber.dispose()
|
||||
expect(ctx.sessionPersistence).toBeUndefined()
|
||||
})
|
||||
|
||||
it('round-trips through the registered service instance', async () => {
|
||||
const ctx = new Context()
|
||||
const fiber = await ctx.plugin(MemoryPersistence)
|
||||
const m = meta('reg')
|
||||
await ctx.sessionPersistence.create(m)
|
||||
await ctx.sessionPersistence.append(m.id, oneTurnLog())
|
||||
const loaded = await ctx.sessionPersistence.load(m.id)
|
||||
expect(loaded.events).toHaveLength(6)
|
||||
await fiber.dispose()
|
||||
})
|
||||
})
|
||||
13
packages/session-persistence/tsconfig.json
Normal file
13
packages/session-persistence/tsconfig.json
Normal file
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"extends": "../../tsconfig.base.json",
|
||||
"compilerOptions": {
|
||||
"rootDir": "src",
|
||||
"outDir": "lib"
|
||||
},
|
||||
"include": ["src"],
|
||||
"references": [
|
||||
{ "path": "../../vendor/cosmokit" },
|
||||
{ "path": "../../vendor/cordis" },
|
||||
{ "path": "../session" }
|
||||
]
|
||||
}
|
||||
Reference in New Issue
Block a user