Merge commit 'refs/codex-unblock/20260723/pr570-master-initial' into worktree/pr570-merge-master-20260723

# Conflicts:
#	.agents/notes/implemented/architecture/2026-06-18-shared-persistence-write-coordinator.md
#	docs/core-data-structures/persistence.md
#	packages/session-persistence/session-persistence/README.md
This commit is contained in:
Tianyi Cui
2026-07-23 23:21:08 +08:00
366 changed files with 11617 additions and 1027 deletions

View File

@@ -37,7 +37,9 @@ A root belongs to one encoding. Startup discovery and targeted lookup reject the
- **Lazy materialization.** `create(meta)` writes nothing; on the first `append`, the backend writes and `fsync`s the encoded header and first batch in a temporary file. POSIX publishes it without overwrite via a hard link and `fsync`s the parent directory. Windows publishes it without overwrite via `MoveFileExW(..., MOVEFILE_WRITE_THROUGH)` and creates missing directories through the same write-through pattern. A created-but-never-appended session leaves nothing on disk and is absent from `list`.
- **Append-only.** Flushed events are never rewritten. Subsequent raw batches append lines; compressed batches append one frame. Both paths `fsync`, and a caught write or sync failure rolls the file back to its prior byte length.
- **Crash recovery — preserve valid tail work.** `load` validates every complete compressed frame and scans their decompressed JSONL. If the last frame is structurally incomplete, the reader keeps its complete decoded records, truncates from that frame's start, and re-encodes those records with the synthetic tool, step, and turn closers required by the shared [persistence contract](../../../.agents/notes/implemented/architecture/2026-06-14-session-persistence.md). Raw mode truncates from its first incomplete line. A checksum/decompression failure in a complete frame, or a defect at or before the last committed `turn/end`, is corruption and rejects.
- **Non-mutating inspection.** `inspect()` returns the detached valid prefix without truncating an incomplete tail or closing an interrupted turn, and leaves the lightweight revision unchanged.
- **Contiguous-seq.** `append` rejects a batch whose first `seq` does not continue the stored log, and rejects non-JSON-serializable `event.data` naming the offending event type.
- **Lightweight revisions.** `listSnapshots()` identifies a log by its device, inode, size, and nanosecond timestamps, avoiding a full-log parse while changing after append, repair, replacement, or store changes.
## Write path

View File

@@ -9,12 +9,13 @@
import { Context } from 'cordis'
import z from 'schemastery'
import { readdirSync } from 'node:fs'
import { open, mkdir, readFile, readdir, link, rm, stat as fsStat, truncate } from 'node:fs/promises'
import { open, mkdir, readFile, readdir, link, rm, stat, truncate } from 'node:fs/promises'
import { dirname, join, resolve } from 'node:path'
import { randomBytes } from 'node:crypto'
import {
SessionPersistence, PersistenceCoordinator,
type PersistenceBackend, type SessionLocation, type StoredPrefix,
SessionPersistence, SessionPersistenceRevision, PersistenceCoordinator,
type PersistenceBackend, type SessionLocation, type SessionPersistenceSnapshot,
type StoredPrefix,
} from '@deepseek-ai/dsh-session-persistence'
import type { SessionEvent, SessionId, SessionHeader } from '@deepseek-ai/dsh-session'
import {
@@ -130,6 +131,10 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi
return this.coordinator.load(id)
}
inspect(id: SessionId): Promise<{ meta: SessionHeader; events: SessionEvent[] }> {
return this.coordinator.inspect(id)
}
// One method serves both public `list` and the backend hook; delegating it to
// the coordinator would call this hook recursively.
@@ -243,11 +248,38 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi
/** List valid unique stored sessions' metadata (header line only — no full-log parse). */
async list(): Promise<SessionHeader[]> {
return (await this.listArtifacts()).map(artifact => artifact.header)
}
/** List metadata plus a stat-derived identity for each append-only log. */
async listSnapshots(): Promise<SessionPersistenceSnapshot[]> {
const snapshots: SessionPersistenceSnapshot[] = []
for (const artifact of await this.listArtifacts()) {
try {
const identity = await stat(artifact.path, { bigint: true })
snapshots.push({
header: artifact.header,
revision: SessionPersistenceRevision([
identity.dev,
identity.ino,
identity.size,
identity.mtimeNs,
identity.ctimeNs,
].join(':')),
})
} catch (error: unknown) {
if (!isENOENT(error)) throw error
}
}
return snapshots
}
private async listArtifacts(): Promise<Array<{ header: SessionHeader; path: string }>> {
await this.ensureRootEncoding()
const metas: SessionHeader[] = []
const artifacts: Array<{ header: SessionHeader; path: string }> = []
const ids = new Set<SessionId>()
for (const dir of await this.listCwdDirs()) {
for (const name of await this.listArtifacts(dir)) {
for (const name of await this.listArtifactNames(dir)) {
const path = join(dir, name)
// Read only headers so listing scales with session count, not log size.
const first = this.compression === 'zstd'
@@ -261,10 +293,10 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi
throw new Error(`duplicate JSONL session id "${meta.id}" appears in multiple cwd buckets`)
}
ids.add(meta.id)
metas.push(meta)
artifacts.push({ header: meta, path })
}
}
return metas
return artifacts
}
// --- materialization / append / repair (file mechanics) ---
@@ -564,7 +596,7 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi
}
}
private async listArtifacts(dir: string): Promise<string[]> {
private async listArtifactNames(dir: string): Promise<string[]> {
const entries = await readdir(dir)
const oppositeSuffix = logSuffix(this.oppositeCompression())
const incompatible = entries.find(name => name.endsWith(oppositeSuffix))
@@ -629,7 +661,7 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi
private async assertLogParentAllowsAbsence(path: string): Promise<void> {
try {
const parent = dirname(path)
const info = await fsStat(parent)
const info = await stat(parent)
if (info.isDirectory()) return
const error = new Error(`ENOTDIR: parent path exists but is not a directory: ${parent}`) as NodeJS.ErrnoException
error.code = 'ENOTDIR'

View File

@@ -209,6 +209,62 @@ describe('SessionPersistenceJsonl: durability and crash semantics', () => {
expect(loaded.events).toEqual(log) // chunks preserved, contiguous seqs
})
it('source-qualifies revisions across roots while preserving same-log reopen identity', async () => {
const m = meta('revision-source')
await ctx.sessionPersistence.create(m)
await ctx.sessionPersistence.append(m.id, oneTurnLog())
const revision = (await ctx.sessionPersistence.listSnapshots())[0]?.revision
const reopenedCtx = new Context()
await reopenedCtx.plugin(SessionStore)
await reopenedCtx.plugin(SessionPersistenceJsonl, { root, compression: 'none' })
expect((await reopenedCtx.sessionPersistence.listSnapshots())[0]?.revision).toBe(revision)
const otherRoot = await freshRoot()
const otherCtx = new Context()
await otherCtx.plugin(SessionStore)
await otherCtx.plugin(SessionPersistenceJsonl, { root: otherRoot, compression: 'none' })
await otherCtx.sessionPersistence.create(m)
await otherCtx.sessionPersistence.append(m.id, oneTurnLog())
expect((await otherCtx.sessionPersistence.listSnapshots())[0]?.revision).not.toBe(revision)
await reopenedCtx.fiber.dispose()
await otherCtx.fiber.dispose()
})
it('omits a snapshot artifact removed after discovery', async () => {
const m = meta('vanishing-snapshot')
await ctx.sessionPersistence.create(m)
await ctx.sessionPersistence.append(m.id, oneTurnLog())
const persistence = ctx.sessionPersistence as unknown as {
listArtifacts(): Promise<Array<{ header: SessionHeader; path: string }>>
}
const listArtifacts = persistence.listArtifacts.bind(persistence)
const discovery = vi.spyOn(persistence, 'listArtifacts').mockImplementation(async () => {
const artifacts = await listArtifacts()
await rm(artifacts[0]!.path)
return artifacts
})
await expect(ctx.sessionPersistence.listSnapshots()).resolves.toEqual([])
discovery.mockRestore()
})
it('surfaces non-ENOENT snapshot stat failures after discovery', async () => {
const blocker = join(root, 'snapshot-not-a-directory')
await writeFile(blocker, 'x')
const persistence = ctx.sessionPersistence as unknown as {
listArtifacts(): Promise<Array<{ header: SessionHeader; path: string }>>
}
const discovery = vi.spyOn(persistence, 'listArtifacts').mockResolvedValue([{
header: meta('snapshot-stat-failure'),
path: join(blocker, 'session.jsonl'),
}])
await expect(ctx.sessionPersistence.listSnapshots()).rejects.toThrow(/ENOTDIR/)
discovery.mockRestore()
})
it('rejects a stored v0 log containing a legacy request/header-delta event', async () => {
const m = meta('legacy-header-delta', '/legacy')
const path = rawLogPath(root, m.cwd, m.id)