docs: trim generated prose
This commit is contained in:
@@ -72,19 +72,10 @@ function isHeaderLine(value: unknown): value is HeaderLine {
|
||||
}
|
||||
|
||||
/**
|
||||
* Encode an arbitrary string as a single safe path segment, injectively over
|
||||
* ALL JS (UTF-16) strings — including lone surrogates. A {@link SessionId} is
|
||||
* an unvalidated branded string, so this neutralizes `../`, absolute paths,
|
||||
* NUL, and separators before any filesystem use.
|
||||
* Encode an arbitrary string as a single safe path segment, injectively over ALL JS (UTF-16)
|
||||
* strings — including lone surrogates. A {@link SessionId} is an unvalidated branded string,
|
||||
* so this neutralizes `../`, absolute paths, NUL, and separators before any filesystem use.
|
||||
*
|
||||
* Each UTF-16 code unit is either kept literal (the safe set `[A-Za-z0-9_-]`)
|
||||
* or escaped as `~XXXX` (its 4-hex-digit code unit). `~` is itself escaped, so
|
||||
* the mapping is injective and reversible: distinct inputs never collide. We
|
||||
* iterate code UNITS (`charCodeAt`), not code points, so a lone surrogate
|
||||
* escapes to a distinct `~XXXX` instead of being normalized to U+FFFD (which
|
||||
* `Buffer.from(…, 'utf8')` would do, breaking injectivity). `.` is in the safe
|
||||
* set for readability but the whole-segment tokens `.`/`..` are escaped so they
|
||||
* can never traverse.
|
||||
* @param raw - the string to encode; must be non-empty (throws on `''`).
|
||||
* @returns the escaped single path segment, decodable back to `raw`.
|
||||
*/
|
||||
@@ -141,39 +132,19 @@ export function eventLine(event: SessionEvent): string {
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a JSONL log buffer into its preserved event prefix (the header is line
|
||||
* 0). Returns the longest prefix of complete, seq-contiguous events plus the
|
||||
* byte offset of the end of the last preserved line (`committedBytes`).
|
||||
* Parse a JSONL log buffer into its preserved event prefix (the header is line 0). Returns the
|
||||
* longest prefix of complete, seq-contiguous events plus the byte offset of the end of the
|
||||
* last preserved line (`committedBytes`).
|
||||
*
|
||||
* A crash can leave a durable log whose final turn never closed: real,
|
||||
* fully-written events sit after the last `turn/end`. Those are PRESERVED (a
|
||||
* single turn can be huge in a long-horizon task — truncating it would destroy
|
||||
* real work); the backend closes the orphaned open turn with a synthetic
|
||||
* `turn/end {kind:'interrupted'}` on reload (the session-persistence RFC). Only a TORN trailing
|
||||
* fragment — a final line never fully flushed (no newline, unparseable, or a
|
||||
* seq gap) — is excluded; it bounds the preserved region. A parse error or seq
|
||||
* gap AT OR BEFORE the last committed `turn/end` is committed-data corruption
|
||||
* and makes the session unloadable (throws).
|
||||
*
|
||||
* This relies on the session-log invariant that every event lives inside a turn
|
||||
* (`Session.append` enforces it): only the final turn can be open, so the
|
||||
* preserved tail is at most one unclosed turn.
|
||||
* @param buffer - the raw bytes of the log file (header line first).
|
||||
* @returns the header, the preserved event prefix, and `committedBytes` — the
|
||||
* byte offset the next append truncates any torn tail to.
|
||||
*/
|
||||
export function scanLog(buffer: Buffer): { meta: SessionHeader; events: SessionEvent[]; committedBytes: number } {
|
||||
const text = buffer.toString('utf8')
|
||||
// Split into complete (newline-terminated) lines, tracking the byte offset of
|
||||
// each line's end so the truncation point is exact (multi-byte chars make the
|
||||
// char offset differ from the byte offset). A trailing line with no newline is
|
||||
// an uncommitted crash fragment and is ignored — it is below the last
|
||||
// turn/end by construction (the loop only flushes whole lines).
|
||||
//
|
||||
// Track the byte offset with a RUNNING accumulator (`endByte`), adding each
|
||||
// line's byte length as we go. Recomputing `Buffer.byteLength(text.slice(0, i))`
|
||||
// per newline would rescan the whole prefix every time — O(n²) over a long
|
||||
// log (one assistant/chunk line per token makes that pathological).
|
||||
// Split into complete (newline-terminated) lines, tracking the byte offset of each line's end
|
||||
// so the truncation point is exact (multi-byte chars make the char offset differ from the
|
||||
// byte offset).
|
||||
const lines: { text: string; endByte: number }[] = []
|
||||
let start = 0
|
||||
let byteOffset = 0
|
||||
@@ -201,14 +172,8 @@ export function scanLog(buffer: Buffer): { meta: SessionHeader; events: SessionE
|
||||
}
|
||||
const headerLine = parsedHeader
|
||||
|
||||
// Find the committed region: the prefix up to and including the LAST complete
|
||||
// `turn/end` in the WHOLE log. Two passes so a crash tail after the last
|
||||
// turn/end is tolerated, but corruption/gaps AT OR BEFORE the last committed
|
||||
// turn/end make the log unloadable (committed data must never be silently
|
||||
// dropped).
|
||||
//
|
||||
// Pass 1: parse every line that parses, recording (parsedOk, seq, isTurnEnd,
|
||||
// endByte) per line index. Lines that fail to parse are holes.
|
||||
// Find the committed region: the prefix up to and including the LAST complete `turn/end` in
|
||||
// the WHOLE log.
|
||||
interface Parsed { ok: boolean; event?: SessionEvent; endByte: number }
|
||||
const parsed: Parsed[] = eventEntries.map((entry) => {
|
||||
try {
|
||||
@@ -226,18 +191,8 @@ export function scanLog(buffer: Buffer): { meta: SessionHeader; events: SessionE
|
||||
if (p?.ok && p.event?.type === 'turn/end') { lastTurnEnd = i; break }
|
||||
}
|
||||
|
||||
// Walk the longest PREFIX of complete, seq-contiguous, parseable event lines
|
||||
// (line i is a parsed event with seq === i). This is the preservable region:
|
||||
// it includes any fully-written events of an interrupted final turn AFTER the
|
||||
// last turn/end — those are real, durably-written work and must NOT be
|
||||
// truncated (a single turn can be huge in a long-horizon task; the orphaned
|
||||
// open turn is closed with a synthetic turn/end on reload, not discarded —
|
||||
// the session-persistence RFC). The walk stops at the first hole (unparseable line or seq gap):
|
||||
// - if that hole is AT OR BEFORE the last committed turn/end, committed data
|
||||
// was damaged → the session is unloadable (throw);
|
||||
// - if it is AFTER (or there is no committed turn/end yet), it is the
|
||||
// tolerated crash boundary — a torn final line never fully flushed — and
|
||||
// it simply bounds the preserved tail.
|
||||
// Walk the longest PREFIX of complete, seq-contiguous, parseable event lines (line i is a
|
||||
// parsed event with seq === i).
|
||||
const preserved: SessionEvent[] = []
|
||||
for (let i = 0; i < parsed.length; i++) {
|
||||
const p = parsed[i]
|
||||
|
||||
@@ -1,19 +1,5 @@
|
||||
/**
|
||||
* JSONL durable session-persistence backend (`@deepseek-ai/dsh-session-persistence-jsonl`).
|
||||
*
|
||||
* One append-only `.jsonl` event log per session (a header line then one
|
||||
* `SessionEvent` per line, verbatim including `assistant/chunk` so `seq` stays
|
||||
* contiguous), with lazy materialization (no file until the first `append`),
|
||||
* atomic first write, and load-time repair of a never-committed crash tail.
|
||||
*
|
||||
* The backend supplies ONLY the file-bytes storage primitives (the
|
||||
* {@link PersistenceBackend} hooks below); all the write-path orchestration
|
||||
* (the `session/event` → buffer → `session/flush` drain, per-session
|
||||
* serialization, write cursors, fork-seed persistence, HMR live-adoption,
|
||||
* crash-repair sequencing, dispose quiescence) lives in the backend-agnostic
|
||||
* {@link PersistenceCoordinator} this class composes. The four public
|
||||
* {@link SessionPersistence} methods delegate to the coordinator.
|
||||
*
|
||||
* @module @deepseek-ai/dsh-session-persistence-jsonl
|
||||
*/
|
||||
|
||||
@@ -80,10 +66,7 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi
|
||||
|
||||
constructor(ctx: Context, public config: Config) {
|
||||
super(ctx)
|
||||
// Resolve the configured root to an ABSOLUTE path ONCE, here. A relative root
|
||||
// would otherwise re-resolve against `process.cwd()` at every later
|
||||
// readdir/open — so if any plugin or test changed cwd between create, append,
|
||||
// and load, one session's files could split across directories.
|
||||
// Resolve the configured root to an ABSOLUTE path ONCE, here.
|
||||
this.root = resolve(config.root)
|
||||
this.coordinator = new PersistenceCoordinator<number>(this.ctx, this)
|
||||
}
|
||||
@@ -102,11 +85,8 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi
|
||||
return this.coordinator.load(id)
|
||||
}
|
||||
|
||||
// `list` is BOTH the public service method and the PersistenceBackend hook —
|
||||
// one method, the bucket walk below. The coordinator adds no orchestration for
|
||||
// listing (no per-id serialization, no cursor), so it would just call back into
|
||||
// this same method; routing it through the coordinator would recurse. Defined
|
||||
// once, in the "PersistenceBackend hooks" section.
|
||||
// `list` is BOTH the public service method and the PersistenceBackend hook — one method, the
|
||||
// bucket walk below.
|
||||
|
||||
/**
|
||||
* The per-session init promises, exposed for white-box tests that await a
|
||||
@@ -201,10 +181,8 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi
|
||||
await mkdir(dir, { recursive: true, mode: 0o700 })
|
||||
await this.syncDir(this.root)
|
||||
const finalPath = logPath(this.root, meta.cwd, meta.id)
|
||||
// Never rename over an existing committed log: materialize is the FIRST write
|
||||
// of a session the backend believes is new. A file here means a different
|
||||
// session shares this id on disk — reject loudly. (createCore already guards
|
||||
// the create path, so this is unreachable-in-practice TOCTOU defense.)
|
||||
// Never rename over an existing committed log: materialize is the FIRST write of a session
|
||||
// the backend believes is new.
|
||||
/* v8 ignore next 3 -- createCore guards collisions before materialize; this is a TOCTOU backstop */
|
||||
if (await this.exists(finalPath)) {
|
||||
throw new Error(`refusing to materialize "${meta.id}": a log already exists on disk (load/resume it instead)`)
|
||||
@@ -229,10 +207,8 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi
|
||||
await link(tmp, finalPath)
|
||||
linked = true
|
||||
} finally {
|
||||
// If link FAILED, the temp is the only reference and must be removed before
|
||||
// the original error propagates. If it SUCCEEDED, defer temp cleanup to
|
||||
// AFTER the publish is durable (below) so a temp-rm failure can never reject
|
||||
// a session whose log already published.
|
||||
// If link failed, the temp is the only reference and must be removed before the original
|
||||
// error propagates.
|
||||
/* v8 ignore next -- link failure is the TOCTOU/IO race guarded above; not reachable in test */
|
||||
if (!linked) await rm(tmp, { force: true })
|
||||
}
|
||||
@@ -240,9 +216,8 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi
|
||||
// entry survives a power loss: the new link is not crash-durable until the
|
||||
// parent directory's metadata is synced.
|
||||
await this.syncDir(dir)
|
||||
// Best-effort temp cleanup: the log is already published and durable, so a
|
||||
// failure to remove the (now-redundant) temp hard link must NOT reject the
|
||||
// append. Swallow only the rm failure; nothing else of consequence runs here.
|
||||
// Best-effort temp cleanup: the log is already published and durable, so a failure to
|
||||
// remove the (now-redundant) temp hard link must not reject the append.
|
||||
try {
|
||||
await rm(tmp, { force: true })
|
||||
} catch {
|
||||
@@ -351,9 +326,7 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi
|
||||
const entries = await readdir(this.root, { withFileTypes: true })
|
||||
return entries.filter(e => e.isDirectory()).map(e => `${this.root}/${e.name}`)
|
||||
} catch (error) {
|
||||
// ENOENT = the root has not been created yet → genuinely no sessions. Any
|
||||
// other error (EACCES, ENOTDIR, transient I/O) must NOT be reported as "no
|
||||
// sessions" — a durable backend cannot silently pretend state is absent.
|
||||
// ENOENT = the root has not been created yet → genuinely no sessions.
|
||||
if (isENOENT(error)) return []
|
||||
throw error
|
||||
}
|
||||
|
||||
@@ -48,11 +48,6 @@ runPersistenceContract('jsonl', async () => {
|
||||
})
|
||||
|
||||
// Run the shared coordinator orchestration suite against the real JSONL backend.
|
||||
// One temp root is the shared storage scope (two mounted instances over the same
|
||||
// root = HMR/reload). `corruptTail` appends a partial, newline-less fragment to
|
||||
// the session's .jsonl past the committed region — a never-committed torn tail
|
||||
// that drives the coordinator's commitRepair-with-tornMarker branch over real
|
||||
// file bytes.
|
||||
runCoordinatorContract('jsonl', async (): Promise<CoordinatorFixture> => {
|
||||
const dir = await mkdtemp(join(tmpdir(), 'dsh-jsonl-coord-'))
|
||||
return {
|
||||
@@ -340,10 +335,9 @@ describe('SessionPersistenceJsonl: scanLog unit', () => {
|
||||
JSON.stringify({ type: 'turn/start', seq: 0, time: 1, data: { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } } }),
|
||||
JSON.stringify({ type: 'step/start', seq: 2, time: 2, data: { turn: 1, step: 1 } }), // gap: missing seq 1
|
||||
].join('\n') + '\n'
|
||||
// No committed turn/end, so the gap is a tolerated crash boundary: scanLog
|
||||
// PRESERVES the contiguous prefix (turn/start seq 0) — real interrupted-turn
|
||||
// work, not discarded — and stops at the gap. The orphaned open turn is
|
||||
// closed by loadCore's synthetic turn/end, not here.
|
||||
// No committed turn/end, so the gap is a tolerated crash boundary: scanLog PRESERVES the
|
||||
// contiguous prefix (turn/start seq 0) — real interrupted-turn work, not discarded — and
|
||||
// stops at the gap.
|
||||
expect(scanLog(Buffer.from(log)).events.map(e => e.seq)).toEqual([0])
|
||||
})
|
||||
|
||||
@@ -463,9 +457,7 @@ describe('SessionPersistenceJsonl: edge cases', () => {
|
||||
})
|
||||
|
||||
it('list reads a header line longer than the 8KB read chunk', async () => {
|
||||
// readFirstLine accumulates across reads when the first line exceeds its
|
||||
// buffer. Plant a valid header whose line is > 8192 bytes (a long extra
|
||||
// field is tolerated by the header type guard) and confirm list() reads it.
|
||||
// readFirstLine accumulates across reads when the first line exceeds its buffer.
|
||||
const bucket = join(root, '_no-cwd')
|
||||
await mkdir(bucket, { recursive: true })
|
||||
const bigHeader = JSON.stringify({ type: 'session', version: 0, id: 'big', createdAt: 1, pad: 'x'.repeat(9000) })
|
||||
@@ -485,10 +477,7 @@ describe('SessionPersistenceJsonl: edge cases', () => {
|
||||
for (const s of ctx.sessions.list()) await ctx.parallel('session/flush', s)
|
||||
await sessFiberA.dispose()
|
||||
|
||||
// A NEW live Session object reuses id "reuse". The init cache is keyed by
|
||||
// the Session OBJECT, so this gets its OWN onCreated (not A's stale promise)
|
||||
// — which detects the on-disk collision and rejects, rather than silently
|
||||
// appending the new session's events onto A's log under a stale cursor.
|
||||
// A NEW live Session object reuses id "reuse".
|
||||
const backend = ctx.sessionPersistence as unknown as { inits: Map<Session, Promise<void>> }
|
||||
let b!: Session
|
||||
await ctx.plugin(Object.assign((inner: Context) => {
|
||||
@@ -506,13 +495,7 @@ describe('SessionPersistenceJsonl: edge cases', () => {
|
||||
await ctx.sessionPersistence.append(SessionId('x'), oneTurnLog())
|
||||
await ctx.fiber.dispose()
|
||||
|
||||
// Backend 2 over the SAME root. A live no-cwd session reuses id "x". Because
|
||||
// loadLive(id, undefined) is the DEFINITE no-cwd bucket (NOT an all-buckets
|
||||
// scan), case-2 adoption does NOT match the "/w" log — so it would NOT
|
||||
// silently graft the no-cwd events onto the "/w" log with a mismatched cwd
|
||||
// (the bug a non-scope-exact loadLive caused). It falls through to the
|
||||
// new-session path, where createCore's any-cwd collision probe (loadStored)
|
||||
// catches the duplicate id and REJECTS — the id is taken in another bucket.
|
||||
// Backend 2 over the same root.
|
||||
const ctx2 = new Context()
|
||||
await ctx2.plugin(SessionStore)
|
||||
await ctx2.plugin(SessionPersistenceJsonl, { root })
|
||||
@@ -580,9 +563,7 @@ describe('SessionPersistenceJsonl: edge cases', () => {
|
||||
})
|
||||
|
||||
it('list surfaces a non-ENOENT root error (ENOTDIR) instead of reporting no sessions', async () => {
|
||||
// A durable backend must NOT collapse a storage fault to "no sessions". Point
|
||||
// the root at a regular FILE: readdir then fails with ENOTDIR, which must
|
||||
// propagate rather than be swallowed as an empty listing.
|
||||
// A durable backend must not collapse a storage fault to "no sessions".
|
||||
const filePath = join(root, 'not-a-dir')
|
||||
await writeFile(filePath, 'x')
|
||||
const ctx2 = new Context()
|
||||
@@ -593,11 +574,8 @@ describe('SessionPersistenceJsonl: edge cases', () => {
|
||||
})
|
||||
|
||||
it('loadLive surfaces a non-ENOENT lookup error (ENOTDIR) instead of reporting absent', async () => {
|
||||
// A non-ENOENT error from the per-id open() must surface, not be collapsed to
|
||||
// "not found" (which would let live-adoption proceed under a false absence
|
||||
// assumption). A live session's onCreated reaches loadLive(id, cwd) →
|
||||
// exists(logPath). Make that cwd's bucket DIRECTORY a regular file: open()ing
|
||||
// `bucket/<id>.jsonl` under it then fails ENOTDIR.
|
||||
// A non-ENOENT error from the per-id open() must surface, not be collapsed to "not found"
|
||||
// (which would let live-adoption proceed under a false absence assumption).
|
||||
const cwd = '/x'
|
||||
const ctx2 = new Context()
|
||||
await ctx2.plugin(SessionStore)
|
||||
@@ -716,10 +694,9 @@ describe('SessionPersistenceJsonl: edge cases', () => {
|
||||
|
||||
it('Session.append rejects a non-serializable event at the source (never enters the log)', () => {
|
||||
const session = ctx.sessions.create(SessionId('reject-bad'))
|
||||
// Serializability is enforced at the source: Session.append throws on a
|
||||
// BigInt-bearing event BEFORE it enters session.events, so the durable log
|
||||
// can never diverge from the live log. The throw surfaces at the caller's
|
||||
// append site, not asynchronously in a backend flush.
|
||||
// Serializability is enforced at the source: Session.append throws on a BigInt-bearing
|
||||
// event before it enters session.events, so the durable log can never diverge from the live
|
||||
// log.
|
||||
expect(() => {
|
||||
session.append('user/message', { content: [{ type: 'text', text: 'bad' }], source: { kind: 'user' }, bad: 1n } as never, { surfaceOp: 'append' })
|
||||
}).toThrow(/non-JSON-serializable/)
|
||||
|
||||
Reference in New Issue
Block a user