Coder/deepseek-harness
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: batch cancellable title reads

Browse Source
This commit is contained in:
Hypatia May
2026-07-24 18:13:11 +08:00
parent f5fc7ac04a
commit fec4ce52cc
23 changed files with 1152 additions and 150 deletions
Download Patch File Download Diff File Expand all files Collapse all files

100
packages/session-persistence/session-persistence-jsonl/src/index.ts
View File

@@ -131,8 +131,8 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi
return this.coordinator.load(id)
}
inspect(id: SessionId): Promise<{ meta: SessionHeader; events: SessionEvent[] }> {
return this.coordinator.inspect(id)
inspect(id: SessionId, signal?: AbortSignal): Promise<{ meta: SessionHeader; events: SessionEvent[] }> {
return this.coordinator.inspect(id, signal)
}
// One method serves both public `list` and the backend hook; delegating it to
@@ -142,24 +142,33 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi
// --- PersistenceBackend hooks (the file-bytes storage primitives) ---
/** Read a stored prefix by id across all cwd buckets when cwd is unknown. */
async loadStored(id: SessionId): Promise<StoredPrefix<JsonlTornMarker> | undefined> {
async loadStored(id: SessionId, signal?: AbortSignal): Promise<StoredPrefix<JsonlTornMarker> | undefined> {
signal?.throwIfAborted()
await this.ensureRootEncoding()
const path = await this.findLog(id)
signal?.throwIfAborted()
const path = await this.findLog(id, signal)
if (path === undefined) return undefined
return this.readPrefix(path, id)
return this.readPrefix(path, id, signal)
}
/**
* Read a stored prefix and convert torn-tail state to the opaque marker the
* coordinator can round-trip without knowing the physical encoding.
*/
private async readPrefix(path: string, expectedId?: SessionId): Promise<StoredPrefix<JsonlTornMarker>> {
const buffer = await readFile(path)
private async readPrefix(
path: string,
expectedId?: SessionId,
signal?: AbortSignal,
): Promise<StoredPrefix<JsonlTornMarker>> {
const buffer = await readFile(path, { signal })
signal?.throwIfAborted()
let prefix: StoredPrefix<JsonlTornMarker>
if (this.compression === 'zstd') {
prefix = await this.readZstdPrefix(buffer)
prefix = await this.readZstdPrefix(buffer, signal)
} else {
signal?.throwIfAborted()
const { meta, events, committedBytes } = scanLog(buffer)
signal?.throwIfAborted()
prefix = {
meta,
events,
@@ -168,30 +177,45 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi
: {},
}
}
signal?.throwIfAborted()
this.assertStoredIdentity(path, prefix.meta, expectedId)
return prefix
}
/** Decode complete frames and retain complete JSONL records from a torn final frame. */
private async readZstdPrefix(buffer: Buffer): Promise<StoredPrefix<JsonlTornMarker>> {
private async readZstdPrefix(
buffer: Buffer,
signal?: AbortSignal,
): Promise<StoredPrefix<JsonlTornMarker>> {
signal?.throwIfAborted()
const { frames, tornStart } = scanZstdFrames(buffer)
signal?.throwIfAborted()
if (frames.length === 0) throw new Error('empty or header-less Zstandard session log')
const plaintextFrames: Buffer[] = []
for (const frame of frames) {
let plaintext: Buffer
try {
plaintextFrames.push(await decompressZstdFrame(buffer.subarray(frame.start, frame.end)))
signal?.throwIfAborted()
plaintext = await decompressZstdFrame(buffer.subarray(frame.start, frame.end))
} catch (error) {
/* v8 ignore next -- decoder failure plus concurrent abort is timing-dependent */
if (signal?.aborted) signal.throwIfAborted()
throw new Error(`corrupt Zstandard session log: frame at byte ${frame.start} failed validation`, { cause: error })
}
signal?.throwIfAborted()
plaintextFrames.push(plaintext)
}
const headerFrame = plaintextFrames[0]
if (headerFrame === undefined || headerFrame.length === 0 || headerFrame.indexOf(0x0A) !== headerFrame.length - 1) {
throw new Error('corrupt Zstandard session log: first frame is not exactly one header line')
}
signal?.throwIfAborted()
const completePlaintext = Buffer.concat(plaintextFrames)
signal?.throwIfAborted()
const completePrefix = scanLog(completePlaintext)
signal?.throwIfAborted()
if (completePrefix.committedBytes !== completePlaintext.length) {
throw new Error('corrupt Zstandard session log: complete frame contains a torn JSONL record')
}
@@ -201,12 +225,17 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi
let recoveredPlaintext: Buffer = Buffer.alloc(0)
try {
signal?.throwIfAborted()
recoveredPlaintext = await decompressZstdFrame(buffer.subarray(tornStart))
} catch {
/* v8 ignore next -- decoder failure plus concurrent abort is timing-dependent */
if (signal?.aborted) signal.throwIfAborted()
// A structurally incomplete final frame may end before Node's decoder can
// emit any plaintext; the complete prior frames remain recoverable.
}
signal?.throwIfAborted()
const recoveredPrefix = scanLog(Buffer.concat([completePlaintext, recoveredPlaintext]))
signal?.throwIfAborted()
/* v8 ignore next 3 -- appending plaintext cannot shorten the already-scanned complete prefix */
if (recoveredPrefix.events.length < completePrefix.events.length) {
throw new Error('corrupt Zstandard session log: recovered prefix does not extend complete frames')
@@ -247,8 +276,8 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi
}
/** List valid unique stored sessions' metadata (header line only — no full-log parse). */
async list(): Promise<SessionHeader[]> {
return (await this.listArtifacts()).map(artifact => artifact.header)
async list(signal?: AbortSignal): Promise<SessionHeader[]> {
return (await this.listArtifacts(signal)).map(artifact => artifact.header)
}
/** List metadata plus a stat-derived identity for each append-only log. */
@@ -274,17 +303,21 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi
return snapshots
}
private async listArtifacts(): Promise<Array<{ header: SessionHeader; path: string }>> {
private async listArtifacts(signal?: AbortSignal): Promise<Array<{ header: SessionHeader; path: string }>> {
signal?.throwIfAborted()
await this.ensureRootEncoding()
signal?.throwIfAborted()
const artifacts: Array<{ header: SessionHeader; path: string }> = []
const ids = new Set<SessionId>()
for (const dir of await this.listCwdDirs()) {
for (const name of await this.listArtifactNames(dir)) {
for (const dir of await this.listCwdDirs(signal)) {
for (const name of await this.listArtifactNames(dir, signal)) {
signal?.throwIfAborted()
const path = join(dir, name)
// Read only headers so listing scales with session count, not log size.
const first = this.compression === 'zstd'
? await this.readFirstZstdLine(path)
: await this.readFirstLine(path)
? await this.readFirstZstdLine(path, signal)
: await this.readFirstLine(path, signal)
signal?.throwIfAborted()
if (first === undefined) continue // empty/half-written file
const meta = parseHeaderMeta(first)
if (meta === undefined) continue // not a session header
@@ -492,18 +525,23 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi
* file. Returns undefined if the file is empty or has no complete first line.
* Reads in bounded chunks so a huge log costs only the header read.
*/
private async readFirstLine(path: string): Promise<string | undefined> {
private async readFirstLine(path: string, signal?: AbortSignal): Promise<string | undefined> {
signal?.throwIfAborted()
const handle = await open(path, 'r')
try {
signal?.throwIfAborted()
const chunks: Buffer[] = []
const buf = Buffer.alloc(8192)
for (;;) {
signal?.throwIfAborted()
const { bytesRead } = await handle.read(buf, 0, buf.length, null)
signal?.throwIfAborted()
if (bytesRead === 0) return undefined // EOF with no newline → no complete line
const slice = buf.subarray(0, bytesRead)
const nl = slice.indexOf(0x0a)
if (nl !== -1) {
chunks.push(slice.subarray(0, nl))
signal?.throwIfAborted()
return Buffer.concat(chunks).toString('utf8')
}
chunks.push(Buffer.from(slice))
@@ -514,23 +552,34 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi
}
/** Read and validate only the independently compressed header frame. */
private async readFirstZstdLine(path: string): Promise<string | undefined> {
private async readFirstZstdLine(path: string, signal?: AbortSignal): Promise<string | undefined> {
signal?.throwIfAborted()
const handle = await open(path, 'r')
try {
signal?.throwIfAborted()
let content = Buffer.alloc(0)
const chunk = Buffer.alloc(8192)
for (;;) {
signal?.throwIfAborted()
const { bytesRead } = await handle.read(chunk, 0, chunk.length, null)
signal?.throwIfAborted()
if (bytesRead === 0) return undefined
signal?.throwIfAborted()
content = Buffer.concat([content, chunk.subarray(0, bytesRead)])
signal?.throwIfAborted()
const first = scanZstdFrames(content, 1).frames[0]
signal?.throwIfAborted()
if (first === undefined) continue
let plaintext: Buffer
try {
signal?.throwIfAborted()
plaintext = await decompressZstdFrame(content.subarray(first.start, first.end))
} catch (error) {
/* v8 ignore next -- decoder failure plus concurrent abort is timing-dependent */
if (signal?.aborted) signal.throwIfAborted()
throw new Error('corrupt Zstandard session log: header frame failed validation', { cause: error })
}
signal?.throwIfAborted()
if (plaintext.length === 0 || plaintext.indexOf(0x0A) !== plaintext.length - 1) {
throw new Error('corrupt Zstandard session log: first frame is not exactly one header line')
}
@@ -542,11 +591,12 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi
}
/** Find the unique physical log for an id across every cwd bucket. */
private async findLog(id: SessionId): Promise<string | undefined> {
private async findLog(id: SessionId, signal?: AbortSignal): Promise<string | undefined> {
const target = encodeSegment(id) + logSuffix(this.compression)
const oppositeTarget = encodeSegment(id) + logSuffix(this.oppositeCompression())
const matches: string[] = []
for (const dir of await this.listCwdDirs()) {
for (const dir of await this.listCwdDirs(signal)) {
signal?.throwIfAborted()
const path = join(dir, target)
const opposite = join(dir, oppositeTarget)
if (await this.exists(opposite)) throw this.encodingMismatch(opposite)
@@ -585,9 +635,11 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi
}
/** The cwd-bucket directories under the root (absolute paths). */
private async listCwdDirs(): Promise<string[]> {
private async listCwdDirs(signal?: AbortSignal): Promise<string[]> {
try {
signal?.throwIfAborted()
const entries = await readdir(this.root, { withFileTypes: true })
signal?.throwIfAborted()
return entries.filter(e => e.isDirectory()).map(e => join(this.root, e.name))
} catch (error) {
// Only an absent root means no sessions; rethrow every other I/O failure.
@@ -596,8 +648,10 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi
}
}
private async listArtifactNames(dir: string): Promise<string[]> {
private async listArtifactNames(dir: string, signal?: AbortSignal): Promise<string[]> {
signal?.throwIfAborted()
const entries = await readdir(dir)
signal?.throwIfAborted()
const oppositeSuffix = logSuffix(this.oppositeCompression())
const incompatible = entries.find(name => name.endsWith(oppositeSuffix))
if (incompatible !== undefined) throw this.encodingMismatch(`${dir}/${incompatible}`)

71
packages/session-persistence/session-persistence-jsonl/tests/zstd.spec.ts
View File

@@ -16,6 +16,18 @@ const MAGIC = Buffer.from([0x28, 0xB5, 0x2F, 0xFD])
const roots: string[] = []
const contexts: Context[] = []
interface ZstdReaderInternals {
readZstdPrefix(buffer: Buffer, signal?: AbortSignal): Promise<unknown>
}
type HeaderRead = (
this: FileHandle,
buffer: Buffer,
offset: number,
length: number,
position: number | null,
) => Promise<{ bytesRead: number; buffer: Buffer }>
async function freshRoot(prefix = 'dsh-jsonl-zstd-'): Promise<string> {
const root = await mkdtemp(join(tmpdir(), prefix))
roots.push(root)
@@ -275,6 +287,65 @@ describe('SessionPersistenceJsonl: default Zstandard encoding', () => {
await expect(ctx.sessionPersistence.load(header.id)).rejects.toThrow(/frame at byte .* failed validation/)
})
it('stops multi-frame inspection after cancellation interrupts the active decode', async () => {
const root = await freshRoot()
const ctx = await mount(root)
const header = meta('cancel-zstd-frames')
const headerFrame = await compressZstdFrame(`${JSON.stringify(toHeaderLine(header))}\n`)
const eventFrame = await compressZstdFrame(`${JSON.stringify(oneTurnLog()[0])}\n`)
const laterFrame = await compressZstdFrame(`${JSON.stringify(oneTurnLog()[1])}\n`)
const stream = Buffer.concat([headerFrame, eventFrame, laterFrame])
expect(scanZstdFrames(stream).frames).toHaveLength(3)
const controller = new AbortController()
const reason = new Error('cancel after Zstandard decode starts')
const reader = ctx.sessionPersistence as unknown as ZstdReaderInternals
const zstdModule = await import('../src/zstd.ts')
const decode = vi.spyOn(zstdModule, 'decompressZstdFrame')
// readZstdPrefix reaches its first asynchronous decompression before it
// returns this promise. The microtask abort therefore occurs after decode
// starts and must prevent every later frame from reaching the decoder.
const pending = reader.readZstdPrefix(stream, controller.signal)
queueMicrotask(() => { controller.abort(reason) })
await expect(pending).rejects.toBe(reason)
expect(decode).toHaveBeenCalledTimes(1)
expect(decode).toHaveBeenCalledWith(headerFrame)
})
it.each(['none', 'zstd'] as const)(
'observes cancellation after each async %s header read during listing',
async (compression) => {
const root = await freshRoot()
const ctx = await mount(root, compression)
const header = meta(`cancel-${compression}-header-read`, '/work')
await ctx.sessionPersistence.create(header)
await ctx.sessionPersistence.append(header.id, oneTurnLog())
await ctx.sessionPersistence.list()
const path = logPath(root, header.cwd, header.id, compression)
const probe = await open(path, 'r')
const prototype = Object.getPrototypeOf(probe) as { read: HeaderRead }
const originalRead = prototype.read
await probe.close()
const controller = new AbortController()
const reason = new Error(`cancel ${compression} header read`)
const read = vi.spyOn(prototype, 'read').mockImplementation(async function (
this: FileHandle,
buffer: Buffer,
offset: number,
length: number,
position: number | null,
) {
const result = await originalRead.call(this, buffer, offset, length, position)
controller.abort(reason)
return result
})
await expect(ctx.sessionPersistence.list(controller.signal)).rejects.toBe(reason)
expect(read).toHaveBeenCalledTimes(1)
},
)
it('preserves complete records from a torn frame and re-encodes them with crash closers', async () => {
const root = await freshRoot()
const ctx = await mount(root)