Merge remote-tracking branch 'origin/master' into session-query-search

# Conflicts:
#	.agents/notes/implemented/feature/2026-07-10-session-query-service.md
#	.agents/notes/implemented/feature/2026-07-10-sqlite-session-query-provider.md
#	.agents/notes/proposed/feature/2026-07-10-sqlite-session-query-provider.md
#	docs/architecture.md
#	docs/capability-seams.md
#	docs/config-catalog.md
#	docs/cordis-catalog/services.md
#	docs/core-data-structures/core.md
#	docs/core-data-structures/persistence.md
#	docs/core-data-structures/session-query.md
#	docs/module-graph.md
#	docs/rfc/INDEX.md
#	packages/README.md
#	packages/cordis/tool-cordis/src/api-catalog.ts
#	packages/hooks/hooks-claude/tests/coverage.spec.ts
#	packages/session-persistence/session-persistence-jsonl/src/index.ts
#	packages/session-persistence/session-persistence-jsonl/tests/jsonl.spec.ts
#	packages/session-persistence/session-persistence-sqlite/README.md
#	packages/session-persistence/session-persistence-sqlite/src/index.ts
#	packages/session-persistence/session-persistence-sqlite/src/schema.ts
#	packages/session-persistence/session-persistence-sqlite/tests/sqlite.spec.ts
#	packages/session-persistence/session-persistence/README.md
#	packages/session-persistence/session-persistence/package.json
#	packages/session-query/README.md
#	packages/session-query/session-query/README.md
#	packages/session-query/session-query/package.json
#	packages/session-query/session-query/src/config.ts
#	packages/session-query/session-query/src/index.ts
#	packages/session-query/session-query/src/types.ts
#	pnpm-lock.yaml
#	scripts/gen-doc-graphs.ts
#	scripts/type-equiv.manifest.json
#	tsconfig.host.json
#	tsconfig.json
This commit is contained in:
Hypatia May
2026-07-23 13:56:56 +08:00
2630 changed files with 198302 additions and 30288 deletions

View File

@@ -10,10 +10,23 @@
import { createHash } from 'node:crypto'
import { join } from 'node:path'
import type { SessionEvent, SessionHeader, SessionId } from '@deepseek-ai/dsh-session'
import { decodeStorageRecord, packChunkRuns } from '@deepseek-ai/dsh-session'
import type { SessionEvent, SessionHeader, SessionId, StorageRecord } from '@deepseek-ai/dsh-session'
/** Physical encoding selected for JSONL session artifacts. */
export type JsonlCompression = 'zstd' | 'none'
/**
* The first line of a session's `.jsonl` file: the immutable
* Return the artifact suffix for one physical encoding.
* @param compression - configured JSONL artifact encoding.
* @returns `.jsonl.zstd` for Zstandard or `.jsonl` for plaintext.
*/
export function logSuffix(compression: JsonlCompression): '.jsonl.zstd' | '.jsonl' {
return compression === 'zstd' ? '.jsonl.zstd' : '.jsonl'
}
/**
* The first JSONL record of a session artifact: the immutable
* {@link SessionHeader} tagged as a `session` record so a reader can tell it
* apart from an event line.
*/
@@ -25,6 +38,7 @@ export interface HeaderLine {
cwd?: string
parentSession?: SessionId
seedLength?: number
delegationDepth: number
}
/**
@@ -41,6 +55,7 @@ export function toHeaderLine(header: SessionHeader): HeaderLine {
...header.cwd !== undefined ? { cwd: header.cwd } : {},
...header.parentSession !== undefined ? { parentSession: header.parentSession } : {},
...header.seedLength !== undefined ? { seedLength: header.seedLength } : {},
delegationDepth: header.delegationDepth ?? 0,
}
}
@@ -57,6 +72,7 @@ export function fromHeaderLine(line: HeaderLine): SessionHeader {
...line.cwd !== undefined ? { cwd: line.cwd } : {},
...line.parentSession !== undefined ? { parentSession: line.parentSession } : {},
...line.seedLength !== undefined ? { seedLength: line.seedLength } : {},
delegationDepth: line.delegationDepth,
}
}
@@ -68,6 +84,10 @@ function isHeaderLine(value: unknown): value is HeaderLine {
&& typeof (value as { version?: unknown }).version === 'number'
&& typeof (value as { id?: unknown }).id === 'string'
&& typeof (value as { createdAt?: unknown }).createdAt === 'number'
&& typeof (value as { delegationDepth?: unknown }).delegationDepth === 'number'
&& Number.isSafeInteger((value as { delegationDepth: number }).delegationDepth)
&& (value as { delegationDepth: number }).delegationDepth >= 0
&& !Object.is((value as { delegationDepth: number }).delegationDepth, -0)
)
}
@@ -119,24 +139,39 @@ export function sessionDir(root: string, cwd: string | undefined): string {
* @param root - the backend's session root directory.
* @param cwd - the session's project directory (picks the per-cwd bucket; `undefined` → `_no-cwd`).
* @param id - the session id, path-encoded via {@link encodeSegment} before filesystem use.
* @returns the session's `.jsonl` log file path.
* @param compression - physical artifact encoding and filename suffix.
* @returns the session's configured JSONL artifact path.
*/
export function logPath(root: string, cwd: string | undefined, id: SessionId): string {
return join(sessionDir(root, cwd), `${encodeSegment(id)}.jsonl`)
export function logPath(
root: string,
cwd: string | undefined,
id: SessionId,
compression: JsonlCompression,
): string {
return join(sessionDir(root, cwd), `${encodeSegment(id)}${logSuffix(compression)}`)
}
/**
* Serialize one event as a JSONL line (no trailing newline).
* @param event - the event to serialize verbatim.
* @returns the event's single-line JSON text; the writer adds the newline.
* Serialize an event batch as JSONL lines (no trailing newline). With
* `packChunks` on, delta-chunk runs pack into `text-chunks` /
* `reasoning-chunks` / `tool-call-chunks` storage rows; off writes one event
* per line, byte-identical to the pre-packing layout. Reading is layout-blind
* either way ({@link scanLog} always decodes rows), so the switch only shapes
* NEW bytes.
* @param events - the batch to serialize, in log order.
* @param packChunks - whether to pack delta runs into storage rows.
* @returns the batch's JSONL text; the writer adds the final newline.
*/
export function eventLine(event: SessionEvent): string {
return JSON.stringify(event)
export function eventLines(events: readonly SessionEvent[], packChunks: boolean): string {
const records: readonly StorageRecord[] = packChunks ? packChunkRuns(events) : events
return records.map(record => JSON.stringify(record)).join('\n')
}
/**
* Parse a JSONL log buffer into its preserved event prefix (the header is line
* 0). Fully written events in an interrupted final turn remain part of the
* 0). Event lines pass through verbatim; packed chunk rows expand back into
* their events, so callers see one contiguous event list regardless of layout.
* Fully written events in an interrupted final turn remain part of the
* prefix. The first unparsable record or seq gap after the last `turn/end`
* marks a tolerated torn tail; the same hole in the committed region rejects.
*
@@ -175,46 +210,60 @@ export function scanLog(buffer: Buffer): { meta: SessionHeader; events: SessionE
}
const headerLine = parsedHeader
// Parse every complete record first so the last valid `turn/end` determines
// whether an earlier hole is committed corruption or an uncommitted tail.
interface Parsed { ok: boolean; event?: SessionEvent; endByte: number }
// Parse and decode every complete line first so the last valid `turn/end`
// determines whether an earlier hole is committed corruption or an
// uncommitted tail. One line yields one event, or a whole run for a packed
// chunk row; a row-tagged line that fails row validation is a hole, exactly
// like unparsable JSON.
interface Parsed { ok: boolean; events?: SessionEvent[]; endByte: number }
const parsed: Parsed[] = eventEntries.map((entry) => {
try {
return { ok: true, event: JSON.parse(entry.text) as SessionEvent, endByte: entry.endByte }
return { ok: true, events: decodeStorageRecord(JSON.parse(entry.text)), endByte: entry.endByte }
} catch {
return { ok: false, endByte: entry.endByte }
}
})
// The last index (into eventEntries) that is a valid `turn/end` — the last
// fully-committed boundary (the loop flushes only at turn/end).
// The last index (into eventEntries) that ends in a valid `turn/end` — the
// last fully-committed boundary (the loop flushes only at turn/end). A packed
// row never stores a turn/end, so only single-event lines can match.
let lastTurnEnd = -1
for (let i = parsed.length - 1; i >= 0; i--) {
const p = parsed[i]
if (p?.ok && p.event?.type === 'turn/end') { lastTurnEnd = i; break }
if (p?.ok && p.events?.some(e => e.type === 'turn/end')) { lastTurnEnd = i; break }
}
// Preserve the contiguous prefix, including a complete interrupted turn;
// holes through the last committed boundary throw, while later holes stop.
// Contiguity is a cursor over seqs (not the line index): a packed row
// advances the cursor by its whole run.
const preserved: SessionEvent[] = []
for (let i = 0; i < parsed.length; i++) {
let lastPreservedLine = -1
scan: for (let i = 0; i < parsed.length; i++) {
const p = parsed[i]
if (!p?.ok || p.event === undefined) {
if (!p?.ok || p.events === undefined) {
if (i <= lastTurnEnd) throw new Error(`corrupt session log: unparsable committed event at line ${i + 1}`)
break // torn tail fragment after the last turn/end — stop, tolerate
}
if (p.event.seq !== i) {
if (i <= lastTurnEnd) throw new Error(`corrupt session log: seq gap in committed region at line ${i + 1} (expected ${i}, got ${p.event.seq})`)
break // gap after the last turn/end — torn tail, stop
for (const event of p.events) {
if (event.seq !== preserved.length) {
if (i <= lastTurnEnd) {
throw new Error(`corrupt session log: seq gap in committed region at line ${i + 1} (expected ${preserved.length}, got ${event.seq})`)
}
break scan // gap after the last turn/end — torn tail, stop
}
preserved.push(event)
}
preserved.push(p.event)
lastPreservedLine = i
}
// committedBytes = end of the last PRESERVED line (header if none): the next
// append truncates any torn bytes past this point before writing the
// synthetic closers + new events.
const lastPreserved = parsed[preserved.length - 1]
const committedBytes = preserved.length > 0 && lastPreserved ? lastPreserved.endByte : headerEntry.endByte
// committedBytes = end of the last FULLY preserved line (header if none): the
// next append truncates any torn bytes past this point before writing the
// synthetic closers + new events. A line is preserved whole or not at all —
// a mid-row seq gap discards the whole row, keeping the truncation offset on
// a line boundary.
const lastPreserved = parsed[lastPreservedLine]
const committedBytes = lastPreserved !== undefined ? lastPreserved.endByte : headerEntry.endByte
return { meta: fromHeaderLine(headerLine), events: preserved, committedBytes }
}

View File

@@ -1,7 +1,8 @@
/**
* JSONL durable session-persistence backend. It stores a header and contiguous
* events in one append-only file per session, and delegates orchestration to
* {@link PersistenceCoordinator}.
* {@link PersistenceCoordinator}. Its side-effect-free locator returns the
* absolute per-session log target before materialization.
* @module @deepseek-ai/dsh-session-persistence-jsonl
*/
@@ -12,14 +13,28 @@ import { dirname, resolve } from 'node:path'
import { randomBytes } from 'node:crypto'
import {
SessionPersistence, SessionPersistenceRevision, PersistenceCoordinator,
type PersistenceBackend, type SessionPersistenceSnapshot, type StoredPrefix,
type PersistenceBackend, type SessionLocation, type SessionPersistenceSnapshot,
type StoredPrefix,
} from '@deepseek-ai/dsh-session-persistence'
import type { SessionEvent, SessionId, SessionHeader } from '@deepseek-ai/dsh-session'
import {
encodeSegment, eventLine, logPath, parseHeaderMeta, scanLog, sessionDir, toHeaderLine,
encodeSegment, eventLines, logPath, logSuffix, parseHeaderMeta, scanLog, sessionDir, toHeaderLine,
type JsonlCompression,
} from './format.ts'
import { compressZstdFrame, decompressZstdFrame, scanZstdFrames } from './zstd.ts'
import { ensureDurableDirectoryWin32, publishNewFileWin32 } from './win32.ts'
/** Plugin config: where the JSONL backend keeps its session logs (`root` is required — no default). */
export type { JsonlCompression } from './format.ts'
const DEFAULT_COMPRESSION: JsonlCompression = 'zstd'
/** Loader schema for the JSONL artifact's physical encoding. */
export const JsonlCompressionSchema: z<JsonlCompression> = z.union([
z.const('zstd'),
z.const('none'),
]).default(DEFAULT_COMPRESSION)
/** Plugin config: where the JSONL backend keeps its session logs, and the packed-row write switch. */
export interface Config {
/**
* Root directory for all session files. Required (no default): a default of
@@ -27,6 +42,23 @@ export interface Config {
* (bash calls, subprocesses). Sessions group under per-cwd subdirectories.
*/
root: string
/**
* Write runs of consecutive `assistant/chunk` delta events as packed
* `text-chunks`/`reasoning-chunks`/`tool-call-chunks` rows (lossless,
* ~60% smaller logs measured on a real session). Off by default while
* snapshot fixtures stay in the one-event-per-line layout: recording with
* packing on rewrites every golden `session.jsonl`. READING packed rows is
* unconditional — a log's layout never depends on this switch.
*/
packChunks?: boolean
/** Physical encoding; defaults to checksummed Zstandard frames. */
compression?: JsonlCompression
}
/** Opaque coordinator token for replacing bytes recovered from a torn frame. */
interface JsonlTornMarker {
truncateTo: number
recoveredEvents: SessionEvent[]
}
/** Whether a filesystem error means absence; every non-ENOENT failure must surface. */
@@ -37,13 +69,16 @@ function isENOENT(error: unknown): boolean {
/**
* The JSONL persistence backend. Load as a plugin; it registers as
* `ctx.sessionPersistence` and (via the coordinator) installs the write-path
* listeners. Its torn-tail marker is the byte offset to truncate the log to.
* listeners. Its torn-tail marker carries the byte offset and any events
* recovered from an incomplete final Zstandard frame.
*/
export class SessionPersistenceJsonl extends SessionPersistence implements PersistenceBackend<number> {
export class SessionPersistenceJsonl extends SessionPersistence implements PersistenceBackend<JsonlTornMarker> {
static inject = ['sessions']
static Config: z<Config> = z.object({
root: z.string().required(),
packChunks: z.boolean().default(false),
compression: JsonlCompressionSchema,
})
/**
@@ -54,13 +89,20 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi
override readonly name = 'session-persistence-jsonl'
private root: string
private coordinator: PersistenceCoordinator<number>
private packChunks: boolean
private compression: JsonlCompression
private coordinator: PersistenceCoordinator<JsonlTornMarker>
private rootEncodingCheck: Promise<void> | undefined
constructor(ctx: Context, public config: Config) {
super(ctx)
// Resolve once so later process.cwd() changes cannot split one backend across roots.
this.root = resolve(config.root)
this.coordinator = new PersistenceCoordinator<number>(this.ctx, this)
// schemastery (static Config) applied the default before construction;
// the cast records that runtime fact for exactOptionalPropertyTypes.
this.packChunks = (config as Required<Config>).packChunks
this.compression = config.compression ?? DEFAULT_COMPRESSION
this.coordinator = new PersistenceCoordinator<JsonlTornMarker>(this.ctx, this)
}
// Each backend keeps the typed service surface beside its storage hooks;
@@ -68,6 +110,11 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi
/* jscpd:ignore-start */
// --- SessionPersistence service surface (delegated to the coordinator) ---
/** Resolve the absolute target path without touching the filesystem. */
locate(meta: SessionHeader): SessionLocation {
return { kind: 'jsonl', path: logPath(this.root, meta.cwd, meta.id, this.compression) }
}
create(meta: SessionHeader): Promise<void> {
return this.coordinator.create(meta)
}
@@ -87,7 +134,8 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi
// --- PersistenceBackend hooks (the file-bytes storage primitives) ---
/** Read a stored prefix by id across all cwd buckets when cwd is unknown. */
async loadStored(id: SessionId): Promise<StoredPrefix<number> | undefined> {
async loadStored(id: SessionId): Promise<StoredPrefix<JsonlTornMarker> | undefined> {
await this.ensureRootEncoding()
const file = await this.findLog(id)
if (file === undefined) return undefined
return this.readPrefix(file.path)
@@ -97,28 +145,85 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi
* Read a stored prefix within one cwd for HMR adoption. `undefined` names the
* no-cwd bucket rather than an unknown cwd, so this never scans other buckets.
*/
async loadLive(id: SessionId, cwd: string | undefined): Promise<StoredPrefix<number> | undefined> {
const path = logPath(this.root, cwd, id)
if (!await this.exists(path)) return undefined
async loadLive(id: SessionId, cwd: string | undefined): Promise<StoredPrefix<JsonlTornMarker> | undefined> {
await this.ensureRootEncoding()
const path = logPath(this.root, cwd, id, this.compression)
if (!await this.exists(path)) {
await this.rejectOppositeArtifact(cwd, id)
return undefined
}
return this.readPrefix(path)
}
/**
* Read a stored prefix and convert torn-tail state to the byte offset the
* coordinator can round-trip without knowing the file format.
* Read a stored prefix and convert torn-tail state to the opaque marker the
* coordinator can round-trip without knowing the physical encoding.
*/
private async readPrefix(path: string): Promise<StoredPrefix<number>> {
private async readPrefix(path: string): Promise<StoredPrefix<JsonlTornMarker>> {
const buffer = await readFile(path)
if (this.compression === 'zstd') return this.readZstdPrefix(buffer)
const { meta, events, committedBytes } = scanLog(buffer)
return {
meta,
events,
...committedBytes < buffer.byteLength ? { tornMarker: committedBytes } : {},
...committedBytes < buffer.byteLength
? { tornMarker: { truncateTo: committedBytes, recoveredEvents: [] } }
: {},
}
}
/** Decode complete frames and retain complete JSONL records from a torn final frame. */
private async readZstdPrefix(buffer: Buffer): Promise<StoredPrefix<JsonlTornMarker>> {
const { frames, tornStart } = scanZstdFrames(buffer)
if (frames.length === 0) throw new Error('empty or header-less Zstandard session log')
const plaintextFrames: Buffer[] = []
for (const frame of frames) {
try {
plaintextFrames.push(await decompressZstdFrame(buffer.subarray(frame.start, frame.end)))
} catch (error) {
throw new Error(`corrupt Zstandard session log: frame at byte ${frame.start} failed validation`, { cause: error })
}
}
const headerFrame = plaintextFrames[0]
if (headerFrame === undefined || headerFrame.length === 0 || headerFrame.indexOf(0x0A) !== headerFrame.length - 1) {
throw new Error('corrupt Zstandard session log: first frame is not exactly one header line')
}
const completePlaintext = Buffer.concat(plaintextFrames)
const completePrefix = scanLog(completePlaintext)
if (completePrefix.committedBytes !== completePlaintext.length) {
throw new Error('corrupt Zstandard session log: complete frame contains a torn JSONL record')
}
if (tornStart === undefined) {
return { meta: completePrefix.meta, events: completePrefix.events }
}
let recoveredPlaintext: Buffer = Buffer.alloc(0)
try {
recoveredPlaintext = await decompressZstdFrame(buffer.subarray(tornStart))
} catch {
// A structurally incomplete final frame may end before Node's decoder can
// emit any plaintext; the complete prior frames remain recoverable.
}
const recoveredPrefix = scanLog(Buffer.concat([completePlaintext, recoveredPlaintext]))
/* v8 ignore next 3 -- appending plaintext cannot shorten the already-scanned complete prefix */
if (recoveredPrefix.events.length < completePrefix.events.length) {
throw new Error('corrupt Zstandard session log: recovered prefix does not extend complete frames')
}
return {
meta: recoveredPrefix.meta,
events: recoveredPrefix.events,
tornMarker: {
truncateTo: tornStart,
recoveredEvents: recoveredPrefix.events.slice(completePrefix.events.length),
},
}
}
/** Durably append a batch, lazily materializing the file when not yet present. */
async appendBatch(meta: SessionHeader, events: readonly SessionEvent[], isMaterialized: boolean): Promise<void> {
await this.ensureRootEncoding()
if (isMaterialized) {
await this.appendLines(meta, events)
} else {
@@ -127,13 +232,18 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi
}
/**
* Make a crash repair durable: truncate the torn tail to `tornMarker` bytes (if
* any), then append the synthetic `closers` (if any). Two fsync'd steps — the
* seam does not require this to be atomic.
* Make a crash repair durable: truncate a torn tail, restore complete events
* decoded from it, then append synthetic closers. Two fsync'd steps — the seam
* does not require this to be atomic.
*/
async commitRepair(meta: SessionHeader, tornMarker: number | undefined, closers: readonly SessionEvent[]): Promise<void> {
if (tornMarker !== undefined) await this.repair(meta, tornMarker)
if (closers.length > 0) await this.appendLines(meta, closers)
async commitRepair(
meta: SessionHeader,
tornMarker: JsonlTornMarker | undefined,
closers: readonly SessionEvent[],
): Promise<void> {
if (tornMarker !== undefined) await this.repair(meta, tornMarker.truncateTo)
const repairedEvents = [...(tornMarker?.recoveredEvents ?? []), ...closers]
if (repairedEvents.length > 0) await this.appendLines(meta, repairedEvents)
}
/** List all stored sessions' metadata (header line only — no full-log parse). */
@@ -165,12 +275,15 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi
}
private async listArtifacts(): Promise<Array<{ header: SessionHeader; path: string }>> {
await this.ensureRootEncoding()
const artifacts: Array<{ header: SessionHeader; path: string }> = []
for (const dir of await this.listCwdDirs()) {
for (const name of await this.listJsonl(dir)) {
for (const name of await this.listArtifactNames(dir)) {
const path = `${dir}/${name}`
// Read only headers so listing scales with session count, not log size.
const first = await this.readFirstLine(path)
const first = this.compression === 'zstd'
? await this.readFirstZstdLine(path)
: await this.readFirstLine(path)
if (first === undefined) continue // empty/half-written file
const meta = parseHeaderMeta(first)
if (meta === undefined) continue // not a session header
@@ -182,33 +295,36 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi
// --- materialization / append / repair (file mechanics) ---
/** Atomically write the header line + first batch (temp-write, fsync, collision-safe hard-link publish). */
/** Atomically write the header line + first batch (temp-write, fsync, publish). */
private async materialize(meta: SessionHeader, events: readonly SessionEvent[]): Promise<void> {
const dir = sessionDir(this.root, meta.cwd)
await mkdir(this.root, { recursive: true, mode: 0o700 })
await this.syncDir(dirname(this.root))
await mkdir(dir, { recursive: true, mode: 0o700 })
await this.syncDir(this.root)
const finalPath = logPath(this.root, meta.cwd, meta.id)
// Materialization is the first write; an existing log is an id collision.
/* v8 ignore next 3 -- createCore guards collisions before materialize; this is a TOCTOU backstop */
if (await this.exists(finalPath)) {
throw new Error(`refusing to materialize "${meta.id}": a log already exists on disk (load/resume it instead)`)
const finalPath = logPath(this.root, meta.cwd, meta.id, this.compression)
await this.rejectOppositeArtifact(meta.cwd, meta.id)
const content = await this.encodeMaterialization(meta, events)
/* v8 ignore next -- native Windows coverage exercises this platform dispatch; Linux covers the POSIX peer */
if (process.platform === 'win32') {
await this.materializeWin32(dir, finalPath, meta.id, content)
} else {
await this.materializePosix(dir, finalPath, meta.id, content)
}
const header = JSON.stringify(toHeaderLine(meta))
const body = events.map(eventLine).join('\n')
const content = header + '\n' + body + '\n'
}
const tmp = `${finalPath}.${randomBytes(6).toString('hex')}.tmp`
const handle = await open(tmp, 'wx', 0o600)
try {
await handle.writeFile(content)
await handle.sync()
} finally {
await handle.close()
}
// Publish with link()+unlink(): unlike rename(), link fails if another
// process materialized the same id first.
/* v8 ignore start -- Windows uses the Win32 durable-publish path; POSIX coverage exercises this peer. */
private async materializePosix(
dir: string,
finalPath: string,
id: SessionId,
content: Buffer | string,
): Promise<void> {
await mkdir(this.root, { recursive: true, mode: 0o700 })
await this.syncDirPosix(dirname(this.root))
await mkdir(dir, { recursive: true, mode: 0o700 })
await this.syncDirPosix(this.root)
await this.rejectExistingLog(finalPath, id)
const tmp = await this.writeSyncedTempFile(finalPath, content)
// Publish via link()+unlink(), NOT rename(): link fails with EEXIST if the
// final path already exists, so two processes materializing the same id
// concurrently cannot clobber each other. rename() would silently overwrite.
let linked = false
try {
await link(tmp, finalPath)
@@ -219,19 +335,84 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi
/* v8 ignore next -- link failure is the TOCTOU/IO race guarded above; not reachable in test */
if (!linked) await rm(tmp, { force: true })
}
// The published link becomes crash-durable only after its directory fsync.
await this.syncDir(dir)
// Best-effort temp cleanup: the log is already published and durable, so a failure to
// remove the (now-redundant) temp hard link must not reject the append.
// link() succeeded — the log is published. fsync the directory so the new
// entry survives a power loss: the new link is not crash-durable until the
// parent directory's metadata is synced.
await this.syncDirPosix(dir)
// Best-effort temp cleanup: the log is already published and durable, so a
// failure to remove the (now-redundant) temp hard link must NOT reject the
// append. Swallow only the rm failure; nothing else of consequence runs here.
try {
await rm(tmp, { force: true })
} catch {
/* v8 ignore next -- redundant temp link; publish already durable, rm failure is an unreachable IO edge */
}
}
/* v8 ignore stop */
/** fsync a directory so a just-created or published entry inside it is crash-durable. */
private async syncDir(dir: string): Promise<void> {
/* v8 ignore start -- native Windows coverage exercises this integration path */
private async materializeWin32(
dir: string,
finalPath: string,
id: SessionId,
content: Buffer | string,
): Promise<void> {
await ensureDurableDirectoryWin32(this.root)
await ensureDurableDirectoryWin32(dir)
await this.rejectExistingLog(finalPath, id)
const tmp = await this.writeSyncedTempFile(finalPath, content)
try {
await publishNewFileWin32(tmp, finalPath)
} catch (error) {
await rm(tmp, { force: true })
throw error
}
}
/* v8 ignore stop */
private async rejectExistingLog(finalPath: string, id: SessionId): Promise<void> {
// Never publish over an existing committed log: materialize is the first
// write of a session the backend believes is new. A file here means a
// different session shares this id on disk — reject loudly. (createCore
// already guards the create path, so this is unreachable-in-practice TOCTOU
// defense.)
/* v8 ignore next 3 -- createCore guards collisions before materialize; this is a TOCTOU backstop */
if (await this.exists(finalPath)) {
throw new Error(`refusing to materialize "${id}": a log already exists on disk (load/resume it instead)`)
}
}
private async writeSyncedTempFile(finalPath: string, content: Buffer | string): Promise<string> {
const tmp = `${finalPath}.${randomBytes(6).toString('hex')}.tmp`
const handle = await open(tmp, 'wx', 0o600)
try {
await handle.writeFile(content)
await handle.sync()
} finally {
await handle.close()
}
return tmp
}
/** Encode the header and first batch without combining their frame boundaries. */
private async encodeMaterialization(meta: SessionHeader, events: readonly SessionEvent[]): Promise<Buffer | string> {
const header = JSON.stringify(toHeaderLine(meta)) + '\n'
const body = eventLines(events, this.packChunks) + '\n'
if (this.compression === 'none') return header + body
const headerFrame = await compressZstdFrame(header)
const eventFrame = await compressZstdFrame(body)
return Buffer.concat([headerFrame, eventFrame])
}
/** Encode one durable append batch in the configured physical representation. */
private async encodeEventBatch(events: readonly SessionEvent[]): Promise<Buffer | string> {
const body = eventLines(events, this.packChunks) + '\n'
return this.compression === 'zstd' ? compressZstdFrame(body) : body
}
/** fsync a POSIX directory so a just-created/renamed entry is crash-durable. */
/* v8 ignore start -- Windows uses write-through namespace operations; POSIX coverage exercises directory fsync. */
private async syncDirPosix(dir: string): Promise<void> {
const handle = await open(dir, 'r')
try {
await handle.sync()
@@ -239,6 +420,7 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi
await handle.close()
}
}
/* v8 ignore stop */
/**
* Append and fsync event lines. On a partial write or sync failure, restore the
@@ -246,19 +428,40 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi
* batch; leaving partial bytes would create duplicate sequence numbers.
*/
private async appendLines(meta: SessionHeader, events: readonly SessionEvent[]): Promise<void> {
const path = logPath(this.root, meta.cwd, meta.id)
const content = await this.encodeEventBatch(events)
const path = logPath(this.root, meta.cwd, meta.id, this.compression)
const handle = await open(path, 'a')
let closed = false
const closeAppendHandle = async (): Promise<void> => {
if (closed) return
closed = true
await handle.close()
}
try {
const { size: before } = await handle.stat()
try {
await handle.writeFile(events.map(eventLine).join('\n') + '\n')
await handle.writeFile(content)
await handle.sync()
} catch (error) {
// Roll back whatever bytes landed so a retry starts from a clean EOF.
await handle.truncate(before)
await handle.sync()
try {
await closeAppendHandle()
await this.rollbackAppend(path, before)
} catch (rollbackError) {
throw new AggregateError([error, rollbackError], `failed to roll back append to "${path}"`)
}
throw error
}
} finally {
await closeAppendHandle()
}
}
private async rollbackAppend(path: string, size: number): Promise<void> {
const handle = await open(path, 'r+')
try {
await handle.truncate(size)
await handle.sync()
} finally {
await handle.close()
}
@@ -266,7 +469,7 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi
/** Truncate the log file to `offset` bytes and fsync (discard the crash tail). */
private async repair(meta: SessionHeader, offset: number): Promise<void> {
const path = logPath(this.root, meta.cwd, meta.id)
const path = logPath(this.root, meta.cwd, meta.id, this.compression)
await truncate(path, offset)
const handle = await open(path, 'r+')
try {
@@ -304,17 +507,47 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi
}
}
/** Read and validate only the independently compressed header frame. */
private async readFirstZstdLine(path: string): Promise<string | undefined> {
const handle = await open(path, 'r')
try {
let content = Buffer.alloc(0)
const chunk = Buffer.alloc(8192)
for (;;) {
const { bytesRead } = await handle.read(chunk, 0, chunk.length, null)
if (bytesRead === 0) return undefined
content = Buffer.concat([content, chunk.subarray(0, bytesRead)])
const first = scanZstdFrames(content, 1).frames[0]
if (first === undefined) continue
let plaintext: Buffer
try {
plaintext = await decompressZstdFrame(content.subarray(first.start, first.end))
} catch (error) {
throw new Error('corrupt Zstandard session log: header frame failed validation', { cause: error })
}
if (plaintext.length === 0 || plaintext.indexOf(0x0A) !== plaintext.length - 1) {
throw new Error('corrupt Zstandard session log: first frame is not exactly one header line')
}
return plaintext.subarray(0, -1).toString('utf8')
}
} finally {
await handle.close()
}
}
/**
* Find a session by id across cwd buckets for resume. Cwd-scoped HMR adoption
* bypasses this scan so a no-cwd session cannot claim another bucket.
*/
private async findLog(id: SessionId): Promise<{ path: string; cwd: string | undefined } | undefined> {
const target = encodeSegment(id) + '.jsonl'
const target = encodeSegment(id) + logSuffix(this.compression)
for (const dir of await this.listCwdDirs()) {
const path = `${dir}/${target}`
const opposite = `${dir}/${encodeSegment(id)}${logSuffix(this.oppositeCompression())}`
if (await this.exists(opposite)) throw this.encodingMismatch(opposite)
if (await this.exists(path)) {
// Recover the cwd from the header so the caller has the session's bucket.
const { meta } = scanLog(await readFile(path))
const { meta } = await this.readPrefix(path)
return { path, cwd: meta.cwd }
}
}
@@ -333,9 +566,45 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi
}
}
private async listJsonl(dir: string): Promise<string[]> {
private async listArtifactNames(dir: string): Promise<string[]> {
const entries = await readdir(dir)
return entries.filter(n => n.endsWith('.jsonl'))
const oppositeSuffix = logSuffix(this.oppositeCompression())
const incompatible = entries.find(name => name.endsWith(oppositeSuffix))
if (incompatible !== undefined) throw this.encodingMismatch(`${dir}/${incompatible}`)
const suffix = logSuffix(this.compression)
return entries.filter(name => name.endsWith(suffix))
}
/** Reject a root that already belongs to the other physical encoding. */
private ensureRootEncoding(): Promise<void> {
this.rootEncodingCheck ??= this.checkRootEncoding()
return this.rootEncodingCheck
}
private async checkRootEncoding(): Promise<void> {
const oppositeSuffix = logSuffix(this.oppositeCompression())
for (const dir of await this.listCwdDirs()) {
const entries = await readdir(dir)
const incompatible = entries.find(name => name.endsWith(oppositeSuffix))
if (incompatible !== undefined) throw this.encodingMismatch(`${dir}/${incompatible}`)
}
}
private async rejectOppositeArtifact(cwd: string | undefined, id: SessionId): Promise<void> {
const path = logPath(this.root, cwd, id, this.oppositeCompression())
if (await this.exists(path)) throw this.encodingMismatch(path)
}
private oppositeCompression(): JsonlCompression {
return this.compression === 'zstd' ? 'none' : 'zstd'
}
private encodingMismatch(path: string): Error {
return new Error(
`session artifact ${JSON.stringify(path)} uses ${logSuffix(this.oppositeCompression())}, `
+ `but this backend is configured for compression ${JSON.stringify(this.compression)}; `
+ 'use a separate root or select the matching compression mode',
)
}
private async exists(path: string): Promise<boolean> {
@@ -344,13 +613,36 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi
await handle.close()
return true
} catch (error) {
// Only ENOENT means absent. A permission/I/O error must surface, not be
// collapsed to `false` — otherwise load() reports "not found" and collision
// checks proceed under a false absence assumption.
if (isENOENT(error)) return false
// Only ENOENT means absent. A permission/I/O error must surface rather
// than letting load or collision checks proceed under false absence.
// Windows reports ENOENT, not ENOTDIR, for `regular-file/child`; verify
// the immediate parent so a blocked cwd bucket remains a storage fault.
/* v8 ignore else -- Windows reports file-valued parents as ENOENT; POSIX covers direct ENOTDIR. */
if (isENOENT(error)) {
await this.assertLogParentAllowsAbsence(path)
return false
}
/* v8 ignore next -- Windows repairs ENOTDIR from ENOENT above; POSIX covers direct ENOTDIR. */
throw error
}
}
/* v8 ignore start -- native Windows coverage exercises this repair; POSIX open reports ENOTDIR before this point. */
private async assertLogParentAllowsAbsence(path: string): Promise<void> {
try {
const parent = dirname(path)
const info = await stat(parent)
if (info.isDirectory()) return
const error = new Error(`ENOTDIR: parent path exists but is not a directory: ${parent}`) as NodeJS.ErrnoException
error.code = 'ENOTDIR'
error.path = parent
throw error
} catch (error) {
if (isENOENT(error)) return
throw error
}
}
/* v8 ignore stop */
}
export default SessionPersistenceJsonl

View File

@@ -0,0 +1,30 @@
/**
* Package-owned invariant companion for `@deepseek-ai/dsh-session-persistence-jsonl`.
* @module @deepseek-ai/dsh-session-persistence-jsonl/invariant
*/
/* jscpd:ignore-start */
import type { Context } from 'cordis'
import type { InvariantInstaller } from '@deepseek-ai/dsh-invariants'
const PACKAGE_NAME = '@deepseek-ai/dsh-session-persistence-jsonl'
/** Cordis companion plugin name. */
export const name = 'session-persistence-jsonl-invariant'
/** Service required before the companion can reserve package ownership. */
export const inject = ['invariants']
/**
* No runtime invariant: persistence correctness requires backend round-trip and crash-tail tests;
* this package exposes no continuously observable in-process relation.
*/
const install: InvariantInstaller = () => {}
/**
* Register this package's invariant companion.
* @param ctx - Cordis context carrying the invariant service.
* @returns the installed registration's disposer after setup succeeds.
*/
export const apply = (ctx: Context): Promise<() => void> =>
Promise.resolve(ctx.invariants.register(PACKAGE_NAME, install))
/* jscpd:ignore-end */

View File

@@ -0,0 +1,150 @@
/**
* Windows durable namespace helpers for the JSONL backend.
*
* POSIX publishes a newly-created log by creating a directory entry and then
* fsyncing the parent directory. Windows does not expose that parent-directory
* fsync contract through Node, so the Windows path uses the native durable
* namespace primitive instead: create a staging object in the target directory
* and publish it with `MoveFileExW(..., MOVEFILE_WRITE_THROUGH)` without
* replacement or cross-volume copy fallback.
*
* @module dsh-session-persistence-jsonl/win32
*/
import { mkdtemp, rm, stat } from 'node:fs/promises'
import { basename, join, parse, resolve, toNamespacedPath } from 'node:path'
type MoveFileExW = (existing: string, replacement: string, flags: number) => number
type GetLastError = () => number
interface Win32Bindings {
moveFileExW: MoveFileExW
getLastError: GetLastError
}
interface Win32ErrnoException extends NodeJS.ErrnoException {
win32Code: number
dest: string
}
const MOVEFILE_WRITE_THROUGH = 0x00000008
const ERROR_FILE_NOT_FOUND = 2
const ERROR_PATH_NOT_FOUND = 3
const ERROR_ACCESS_DENIED = 5
const ERROR_NOT_SAME_DEVICE = 17
const ERROR_FILE_EXISTS = 80
const ERROR_INVALID_NAME = 123
const ERROR_ALREADY_EXISTS = 183
let bindings: Win32Bindings | undefined
/** Load the small Win32 surface lazily so non-Windows processes never load Koffi. */
async function win32(): Promise<Win32Bindings> {
if (bindings !== undefined) return bindings
const koffi = (await import('koffi')).default
const kernel32 = koffi.load('kernel32.dll')
bindings = {
moveFileExW: kernel32.func('__stdcall', 'MoveFileExW', 'int', ['str16', 'str16', 'uint']) as MoveFileExW,
getLastError: kernel32.func('__stdcall', 'GetLastError', 'uint', []) as GetLastError,
}
return bindings
}
function errnoCode(win32Code: number): string {
switch (win32Code) {
case ERROR_FILE_NOT_FOUND:
case ERROR_PATH_NOT_FOUND:
return 'ENOENT'
case ERROR_ACCESS_DENIED:
return 'EACCES'
case ERROR_NOT_SAME_DEVICE:
return 'EXDEV'
case ERROR_FILE_EXISTS:
case ERROR_ALREADY_EXISTS:
return 'EEXIST'
case ERROR_INVALID_NAME:
return 'EINVAL'
default:
return 'EIO'
}
}
function win32Error(syscall: string, win32Code: number, path: string, dest: string): Win32ErrnoException {
const code = errnoCode(win32Code)
const error = new Error(`${syscall} ${code} (Win32 ${win32Code}): ${path} -> ${dest}`) as Win32ErrnoException
error.code = code
error.errno = win32Code
error.syscall = syscall
error.path = path
error.dest = dest
error.win32Code = win32Code
return error
}
function isENOENT(error: unknown): boolean {
return (error as NodeJS.ErrnoException | null)?.code === 'ENOENT'
}
function isEEXIST(error: unknown): boolean {
return (error as NodeJS.ErrnoException | null)?.code === 'EEXIST'
}
async function assertDirectory(path: string): Promise<boolean> {
try {
const info = await stat(path)
if (info.isDirectory()) return true
const error = new Error(`path exists but is not a directory: ${path}`) as NodeJS.ErrnoException
error.code = 'ENOTDIR'
error.path = path
throw error
} catch (error) {
if (isENOENT(error)) return false
throw error
}
}
/**
* Publish `existing` at `replacement` with Windows write-through rename
* semantics. The destination must not already exist; the move must stay within
* the volume (no copy fallback flag is set).
* @param existing - the synced staging path to move.
* @param replacement - the final path, which must not already exist.
*/
export async function publishNewFileWin32(existing: string, replacement: string): Promise<void> {
const api = await win32()
const ok = api.moveFileExW(toNamespacedPath(existing), toNamespacedPath(replacement), MOVEFILE_WRITE_THROUGH)
if (ok === 0) throw win32Error('MoveFileExW', api.getLastError(), existing, replacement)
}
/**
* Create `target` and its missing ancestors with durable Windows namespace
* publication. Each missing directory is first created as a random staging
* sibling, then moved to its final name with `MOVEFILE_WRITE_THROUGH`; races
* with another creator are accepted only after verifying the winner is a
* directory.
* @param target - the absolute directory path to create durably when absent.
*/
export async function ensureDurableDirectoryWin32(target: string): Promise<void> {
const absolute = resolve(target)
const root = parse(absolute).root
await assertDirectory(root)
const segments = absolute.slice(root.length).split(/[\\/]+/).filter(part => part.length > 0)
let current = root
for (const segment of segments) {
const next = join(current, segment)
if (!await assertDirectory(next)) await createLeafDirectoryWin32(current, next)
current = next
}
}
async function createLeafDirectoryWin32(parent: string, target: string): Promise<void> {
const staging = await mkdtemp(join(parent, `.dsh-mkdir-${basename(target)}-`))
try {
await publishNewFileWin32(staging, target)
} catch (error) {
await rm(staging, { recursive: true, force: true })
if (isEEXIST(error) && await assertDirectory(target)) return
throw error
}
}

View File

@@ -0,0 +1,116 @@
/**
* Zstandard frame primitives for the JSONL persistence backend. The backend
* owns a concatenated-frame container so it can append and recover batches
* without exposing compression mechanics through the persistence seam.
* @module dsh-session-persistence-jsonl/zstd
*/
import { constants, zstdCompress, zstdDecompress, type ZstdOptions } from 'node:zlib'
import { promisify } from 'node:util'
const ZSTD_MAGIC = 0xFD2FB528
const zstdCompressAsync = promisify(zstdCompress)
const zstdDecompressAsync = promisify(zstdDecompress)
const CHECKSUM_OPTIONS: ZstdOptions = {
params: { [constants.ZSTD_c_checksumFlag]: 1 },
}
/** Byte range occupied by one structurally complete Zstandard frame. */
export interface ZstdFrameRange {
/** Inclusive frame start. */
start: number
/** Exclusive frame end. */
end: number
}
/** Structural scan result for a concatenated Zstandard stream. */
export interface ZstdFrameScan {
/** Complete frames in file order. */
frames: ZstdFrameRange[]
/** Start of an incomplete final frame, when EOF interrupts one. */
tornStart?: number
}
/**
* Locate complete frames without decompressing their blocks. Invalid complete
* structure rejects; EOF inside the final frame returns its start for repair.
* @param buffer - complete bytes currently present in the session artifact.
* @param maxFrames - optional complete-frame limit for metadata-only readers.
* @returns complete frame ranges and an optional incomplete-final-frame start.
*/
export function scanZstdFrames(buffer: Buffer, maxFrames = Number.POSITIVE_INFINITY): ZstdFrameScan {
const frames: ZstdFrameRange[] = []
let offset = 0
while (offset < buffer.length) {
const start = offset
if (buffer.length - offset < 4) return { frames, tornStart: start }
if (buffer.readUInt32LE(offset) !== ZSTD_MAGIC) {
throw new Error(`corrupt Zstandard session log: invalid frame magic at byte ${offset}`)
}
offset += 4
if (offset === buffer.length) return { frames, tornStart: start }
const descriptor = buffer.readUInt8(offset)
offset += 1
if ((descriptor & 0x18) !== 0) {
throw new Error(`corrupt Zstandard session log: reserved frame-header bit at byte ${offset - 1}`)
}
const contentSizeFlag = descriptor >>> 6
const singleSegment = (descriptor & 0x20) !== 0
const checksum = (descriptor & 0x04) !== 0
const dictionaryFlag = descriptor & 0x03
const dictionaryBytes = dictionaryFlag === 3 ? 4 : dictionaryFlag
const contentSizeBytes = contentSizeFlag === 0
? (singleSegment ? 1 : 0)
: 1 << contentSizeFlag
const remainingHeaderBytes = (singleSegment ? 0 : 1) + dictionaryBytes + contentSizeBytes
if (buffer.length - offset < remainingHeaderBytes) return { frames, tornStart: start }
offset += remainingHeaderBytes
for (;;) {
if (buffer.length - offset < 3) return { frames, tornStart: start }
const blockHeader = buffer.readUIntLE(offset, 3)
offset += 3
const lastBlock = (blockHeader & 1) !== 0
const blockType = (blockHeader >>> 1) & 0x03
const blockSize = blockHeader >>> 3
if (blockType === 0x03) {
throw new Error(`corrupt Zstandard session log: reserved block type at byte ${offset - 3}`)
}
const payloadBytes = blockType === 0x01 ? 1 : blockSize
if (buffer.length - offset < payloadBytes) return { frames, tornStart: start }
offset += payloadBytes
if (lastBlock) break
}
if (checksum) {
if (buffer.length - offset < 4) return { frames, tornStart: start }
offset += 4
}
frames.push({ start, end: offset })
if (frames.length === maxFrames) return { frames }
}
return { frames }
}
/**
* Compress one independently decodable, checksummed Zstandard frame.
* @param input - JSONL bytes for a header or durable event batch.
* @returns the complete encoded frame.
*/
export async function compressZstdFrame(input: Buffer | string): Promise<Buffer> {
return zstdCompressAsync(input, CHECKSUM_OPTIONS)
}
/**
* Decompress one complete frame or the available prefix of a torn final frame.
* Complete-frame checksums are validated by Node's decoder.
* @param input - bytes beginning at a Zstandard frame boundary.
* @returns plaintext produced from the available input.
*/
export async function decompressZstdFrame(input: Buffer): Promise<Buffer> {
return zstdDecompressAsync(input)
}