docs: trim generated prose

This commit is contained in:
Tianyi Cui
2026-07-12 03:36:43 +08:00
parent 3dca90261c
commit 75838e10b5
323 changed files with 2857 additions and 11833 deletions

View File

@@ -72,19 +72,10 @@ function isHeaderLine(value: unknown): value is HeaderLine {
}
/**
* Encode an arbitrary string as a single safe path segment, injectively over
* ALL JS (UTF-16) strings — including lone surrogates. A {@link SessionId} is
* an unvalidated branded string, so this neutralizes `../`, absolute paths,
* NUL, and separators before any filesystem use.
* Encode an arbitrary string as a single safe path segment, injectively over ALL JS (UTF-16)
* strings — including lone surrogates. A {@link SessionId} is an unvalidated branded string,
* so this neutralizes `../`, absolute paths, NUL, and separators before any filesystem use.
*
* Each UTF-16 code unit is either kept literal (the safe set `[A-Za-z0-9_-]`)
* or escaped as `~XXXX` (its 4-hex-digit code unit). `~` is itself escaped, so
* the mapping is injective and reversible: distinct inputs never collide. We
* iterate code UNITS (`charCodeAt`), not code points, so a lone surrogate
* escapes to a distinct `~XXXX` instead of being normalized to U+FFFD (which
* `Buffer.from(…, 'utf8')` would do, breaking injectivity). `.` is in the safe
* set for readability but the whole-segment tokens `.`/`..` are escaped so they
* can never traverse.
* @param raw - the string to encode; must be non-empty (throws on `''`).
* @returns the escaped single path segment, decodable back to `raw`.
*/
@@ -141,39 +132,19 @@ export function eventLine(event: SessionEvent): string {
}
/**
* Parse a JSONL log buffer into its preserved event prefix (the header is line
* 0). Returns the longest prefix of complete, seq-contiguous events plus the
* byte offset of the end of the last preserved line (`committedBytes`).
* Parse a JSONL log buffer into its preserved event prefix (the header is line 0). Returns the
* longest prefix of complete, seq-contiguous events plus the byte offset of the end of the
* last preserved line (`committedBytes`).
*
* A crash can leave a durable log whose final turn never closed: real,
* fully-written events sit after the last `turn/end`. Those are PRESERVED (a
* single turn can be huge in a long-horizon task — truncating it would destroy
* real work); the backend closes the orphaned open turn with a synthetic
* `turn/end {kind:'interrupted'}` on reload (the session-persistence RFC). Only a TORN trailing
* fragment — a final line never fully flushed (no newline, unparseable, or a
* seq gap) — is excluded; it bounds the preserved region. A parse error or seq
* gap AT OR BEFORE the last committed `turn/end` is committed-data corruption
* and makes the session unloadable (throws).
*
* This relies on the session-log invariant that every event lives inside a turn
* (`Session.append` enforces it): only the final turn can be open, so the
* preserved tail is at most one unclosed turn.
* @param buffer - the raw bytes of the log file (header line first).
* @returns the header, the preserved event prefix, and `committedBytes` — the
* byte offset the next append truncates any torn tail to.
*/
export function scanLog(buffer: Buffer): { meta: SessionHeader; events: SessionEvent[]; committedBytes: number } {
const text = buffer.toString('utf8')
// Split into complete (newline-terminated) lines, tracking the byte offset of
// each line's end so the truncation point is exact (multi-byte chars make the
// char offset differ from the byte offset). A trailing line with no newline is
// an uncommitted crash fragment and is ignored — it is below the last
// turn/end by construction (the loop only flushes whole lines).
//
// Track the byte offset with a RUNNING accumulator (`endByte`), adding each
// line's byte length as we go. Recomputing `Buffer.byteLength(text.slice(0, i))`
// per newline would rescan the whole prefix every time — O(n²) over a long
// log (one assistant/chunk line per token makes that pathological).
// Split into complete (newline-terminated) lines, tracking the byte offset of each line's end
// so the truncation point is exact (multi-byte chars make the char offset differ from the
// byte offset).
const lines: { text: string; endByte: number }[] = []
let start = 0
let byteOffset = 0
@@ -201,14 +172,8 @@ export function scanLog(buffer: Buffer): { meta: SessionHeader; events: SessionE
}
const headerLine = parsedHeader
// Find the committed region: the prefix up to and including the LAST complete
// `turn/end` in the WHOLE log. Two passes so a crash tail after the last
// turn/end is tolerated, but corruption/gaps AT OR BEFORE the last committed
// turn/end make the log unloadable (committed data must never be silently
// dropped).
//
// Pass 1: parse every line that parses, recording (parsedOk, seq, isTurnEnd,
// endByte) per line index. Lines that fail to parse are holes.
// Find the committed region: the prefix up to and including the LAST complete `turn/end` in
// the WHOLE log.
interface Parsed { ok: boolean; event?: SessionEvent; endByte: number }
const parsed: Parsed[] = eventEntries.map((entry) => {
try {
@@ -226,18 +191,8 @@ export function scanLog(buffer: Buffer): { meta: SessionHeader; events: SessionE
if (p?.ok && p.event?.type === 'turn/end') { lastTurnEnd = i; break }
}
// Walk the longest PREFIX of complete, seq-contiguous, parseable event lines
// (line i is a parsed event with seq === i). This is the preservable region:
// it includes any fully-written events of an interrupted final turn AFTER the
// last turn/end — those are real, durably-written work and must NOT be
// truncated (a single turn can be huge in a long-horizon task; the orphaned
// open turn is closed with a synthetic turn/end on reload, not discarded —
// the session-persistence RFC). The walk stops at the first hole (unparseable line or seq gap):
// - if that hole is AT OR BEFORE the last committed turn/end, committed data
// was damaged → the session is unloadable (throw);
// - if it is AFTER (or there is no committed turn/end yet), it is the
// tolerated crash boundary — a torn final line never fully flushed — and
// it simply bounds the preserved tail.
// Walk the longest PREFIX of complete, seq-contiguous, parseable event lines (line i is a
// parsed event with seq === i).
const preserved: SessionEvent[] = []
for (let i = 0; i < parsed.length; i++) {
const p = parsed[i]

View File

@@ -1,19 +1,5 @@
/**
* JSONL durable session-persistence backend (`@deepseek-ai/dsh-session-persistence-jsonl`).
*
* One append-only `.jsonl` event log per session (a header line then one
* `SessionEvent` per line, verbatim including `assistant/chunk` so `seq` stays
* contiguous), with lazy materialization (no file until the first `append`),
* atomic first write, and load-time repair of a never-committed crash tail.
*
* The backend supplies ONLY the file-bytes storage primitives (the
* {@link PersistenceBackend} hooks below); all the write-path orchestration
* (the `session/event` → buffer → `session/flush` drain, per-session
* serialization, write cursors, fork-seed persistence, HMR live-adoption,
* crash-repair sequencing, dispose quiescence) lives in the backend-agnostic
* {@link PersistenceCoordinator} this class composes. The four public
* {@link SessionPersistence} methods delegate to the coordinator.
*
* @module @deepseek-ai/dsh-session-persistence-jsonl
*/
@@ -80,10 +66,7 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi
constructor(ctx: Context, public config: Config) {
super(ctx)
// Resolve the configured root to an ABSOLUTE path ONCE, here. A relative root
// would otherwise re-resolve against `process.cwd()` at every later
// readdir/open — so if any plugin or test changed cwd between create, append,
// and load, one session's files could split across directories.
// Resolve the configured root to an ABSOLUTE path ONCE, here.
this.root = resolve(config.root)
this.coordinator = new PersistenceCoordinator<number>(this.ctx, this)
}
@@ -102,11 +85,8 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi
return this.coordinator.load(id)
}
// `list` is BOTH the public service method and the PersistenceBackend hook —
// one method, the bucket walk below. The coordinator adds no orchestration for
// listing (no per-id serialization, no cursor), so it would just call back into
// this same method; routing it through the coordinator would recurse. Defined
// once, in the "PersistenceBackend hooks" section.
// `list` is BOTH the public service method and the PersistenceBackend hook — one method, the
// bucket walk below.
/**
* The per-session init promises, exposed for white-box tests that await a
@@ -201,10 +181,8 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi
await mkdir(dir, { recursive: true, mode: 0o700 })
await this.syncDir(this.root)
const finalPath = logPath(this.root, meta.cwd, meta.id)
// Never rename over an existing committed log: materialize is the FIRST write
// of a session the backend believes is new. A file here means a different
// session shares this id on disk — reject loudly. (createCore already guards
// the create path, so this is unreachable-in-practice TOCTOU defense.)
// Never rename over an existing committed log: materialize is the FIRST write of a session
// the backend believes is new.
/* v8 ignore next 3 -- createCore guards collisions before materialize; this is a TOCTOU backstop */
if (await this.exists(finalPath)) {
throw new Error(`refusing to materialize "${meta.id}": a log already exists on disk (load/resume it instead)`)
@@ -229,10 +207,8 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi
await link(tmp, finalPath)
linked = true
} finally {
// If link FAILED, the temp is the only reference and must be removed before
// the original error propagates. If it SUCCEEDED, defer temp cleanup to
// AFTER the publish is durable (below) so a temp-rm failure can never reject
// a session whose log already published.
// If link failed, the temp is the only reference and must be removed before the original
// error propagates.
/* v8 ignore next -- link failure is the TOCTOU/IO race guarded above; not reachable in test */
if (!linked) await rm(tmp, { force: true })
}
@@ -240,9 +216,8 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi
// entry survives a power loss: the new link is not crash-durable until the
// parent directory's metadata is synced.
await this.syncDir(dir)
// Best-effort temp cleanup: the log is already published and durable, so a
// failure to remove the (now-redundant) temp hard link must NOT reject the
// append. Swallow only the rm failure; nothing else of consequence runs here.
// Best-effort temp cleanup: the log is already published and durable, so a failure to
// remove the (now-redundant) temp hard link must not reject the append.
try {
await rm(tmp, { force: true })
} catch {
@@ -351,9 +326,7 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi
const entries = await readdir(this.root, { withFileTypes: true })
return entries.filter(e => e.isDirectory()).map(e => `${this.root}/${e.name}`)
} catch (error) {
// ENOENT = the root has not been created yet → genuinely no sessions. Any
// other error (EACCES, ENOTDIR, transient I/O) must NOT be reported as "no
// sessions" — a durable backend cannot silently pretend state is absent.
// ENOENT = the root has not been created yet → genuinely no sessions.
if (isENOENT(error)) return []
throw error
}

View File

@@ -48,11 +48,6 @@ runPersistenceContract('jsonl', async () => {
})
// Run the shared coordinator orchestration suite against the real JSONL backend.
// One temp root is the shared storage scope (two mounted instances over the same
// root = HMR/reload). `corruptTail` appends a partial, newline-less fragment to
// the session's .jsonl past the committed region — a never-committed torn tail
// that drives the coordinator's commitRepair-with-tornMarker branch over real
// file bytes.
runCoordinatorContract('jsonl', async (): Promise<CoordinatorFixture> => {
const dir = await mkdtemp(join(tmpdir(), 'dsh-jsonl-coord-'))
return {
@@ -340,10 +335,9 @@ describe('SessionPersistenceJsonl: scanLog unit', () => {
JSON.stringify({ type: 'turn/start', seq: 0, time: 1, data: { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } } }),
JSON.stringify({ type: 'step/start', seq: 2, time: 2, data: { turn: 1, step: 1 } }), // gap: missing seq 1
].join('\n') + '\n'
// No committed turn/end, so the gap is a tolerated crash boundary: scanLog
// PRESERVES the contiguous prefix (turn/start seq 0) — real interrupted-turn
// work, not discarded — and stops at the gap. The orphaned open turn is
// closed by loadCore's synthetic turn/end, not here.
// No committed turn/end, so the gap is a tolerated crash boundary: scanLog PRESERVES the
// contiguous prefix (turn/start seq 0) — real interrupted-turn work, not discarded — and
// stops at the gap.
expect(scanLog(Buffer.from(log)).events.map(e => e.seq)).toEqual([0])
})
@@ -463,9 +457,7 @@ describe('SessionPersistenceJsonl: edge cases', () => {
})
it('list reads a header line longer than the 8KB read chunk', async () => {
// readFirstLine accumulates across reads when the first line exceeds its
// buffer. Plant a valid header whose line is > 8192 bytes (a long extra
// field is tolerated by the header type guard) and confirm list() reads it.
// readFirstLine accumulates across reads when the first line exceeds its buffer.
const bucket = join(root, '_no-cwd')
await mkdir(bucket, { recursive: true })
const bigHeader = JSON.stringify({ type: 'session', version: 0, id: 'big', createdAt: 1, pad: 'x'.repeat(9000) })
@@ -485,10 +477,7 @@ describe('SessionPersistenceJsonl: edge cases', () => {
for (const s of ctx.sessions.list()) await ctx.parallel('session/flush', s)
await sessFiberA.dispose()
// A NEW live Session object reuses id "reuse". The init cache is keyed by
// the Session OBJECT, so this gets its OWN onCreated (not A's stale promise)
// — which detects the on-disk collision and rejects, rather than silently
// appending the new session's events onto A's log under a stale cursor.
// A NEW live Session object reuses id "reuse".
const backend = ctx.sessionPersistence as unknown as { inits: Map<Session, Promise<void>> }
let b!: Session
await ctx.plugin(Object.assign((inner: Context) => {
@@ -506,13 +495,7 @@ describe('SessionPersistenceJsonl: edge cases', () => {
await ctx.sessionPersistence.append(SessionId('x'), oneTurnLog())
await ctx.fiber.dispose()
// Backend 2 over the SAME root. A live no-cwd session reuses id "x". Because
// loadLive(id, undefined) is the DEFINITE no-cwd bucket (NOT an all-buckets
// scan), case-2 adoption does NOT match the "/w" log — so it would NOT
// silently graft the no-cwd events onto the "/w" log with a mismatched cwd
// (the bug a non-scope-exact loadLive caused). It falls through to the
// new-session path, where createCore's any-cwd collision probe (loadStored)
// catches the duplicate id and REJECTS — the id is taken in another bucket.
// Backend 2 over the same root.
const ctx2 = new Context()
await ctx2.plugin(SessionStore)
await ctx2.plugin(SessionPersistenceJsonl, { root })
@@ -580,9 +563,7 @@ describe('SessionPersistenceJsonl: edge cases', () => {
})
it('list surfaces a non-ENOENT root error (ENOTDIR) instead of reporting no sessions', async () => {
// A durable backend must NOT collapse a storage fault to "no sessions". Point
// the root at a regular FILE: readdir then fails with ENOTDIR, which must
// propagate rather than be swallowed as an empty listing.
// A durable backend must not collapse a storage fault to "no sessions".
const filePath = join(root, 'not-a-dir')
await writeFile(filePath, 'x')
const ctx2 = new Context()
@@ -593,11 +574,8 @@ describe('SessionPersistenceJsonl: edge cases', () => {
})
it('loadLive surfaces a non-ENOENT lookup error (ENOTDIR) instead of reporting absent', async () => {
// A non-ENOENT error from the per-id open() must surface, not be collapsed to
// "not found" (which would let live-adoption proceed under a false absence
// assumption). A live session's onCreated reaches loadLive(id, cwd) →
// exists(logPath). Make that cwd's bucket DIRECTORY a regular file: open()ing
// `bucket/<id>.jsonl` under it then fails ENOTDIR.
// A non-ENOENT error from the per-id open() must surface, not be collapsed to "not found"
// (which would let live-adoption proceed under a false absence assumption).
const cwd = '/x'
const ctx2 = new Context()
await ctx2.plugin(SessionStore)
@@ -716,10 +694,9 @@ describe('SessionPersistenceJsonl: edge cases', () => {
it('Session.append rejects a non-serializable event at the source (never enters the log)', () => {
const session = ctx.sessions.create(SessionId('reject-bad'))
// Serializability is enforced at the source: Session.append throws on a
// BigInt-bearing event BEFORE it enters session.events, so the durable log
// can never diverge from the live log. The throw surfaces at the caller's
// append site, not asynchronously in a backend flush.
// Serializability is enforced at the source: Session.append throws on a BigInt-bearing
// event before it enters session.events, so the durable log can never diverge from the live
// log.
expect(() => {
session.append('user/message', { content: [{ type: 'text', text: 'bad' }], source: { kind: 'user' }, bad: 1n } as never, { surfaceOp: 'append' })
}).toThrow(/non-JSON-serializable/)