feat(persistence): compress JSONL logs with Zstandard

This commit is contained in:
Tianyi Cui
2026-07-20 00:06:21 +08:00
parent 13433025f2
commit d139948ed2
45 changed files with 1113 additions and 135 deletions

View File

@@ -87,7 +87,7 @@ pnpm run hygiene
out=$(printf 'echo ci smoke\n' | pnpm run demo:echo 2>&1) out=$(printf 'echo ci smoke\n' | pnpm run demo:echo 2>&1)
printf '%s\n' "$out" | grep -q '\[tool call\] echo({"text":"ci smoke"})' printf '%s\n' "$out" | grep -q '\[tool call\] echo({"text":"ci smoke"})'
printf '%s\n' "$out" | grep -q '\[tool result\] ECHO: CI SMOKE' printf '%s\n' "$out" | grep -q '\[tool result\] ECHO: CI SMOKE'
test -n "$(find .sessions -path '.sessions/cwd-*/main-session-*.jsonl' -type f -print -quit)" test -n "$(find .sessions -path '.sessions/cwd-*/main-session-*.jsonl.zstd' -type f -print -quit)"
rm -rf .sessions rm -rf .sessions
pnpm exec vitest run --config vitest.e2e.config.ts packages/examples/stdio-demo/tests/built-bin.e2e.ts packages/examples/cli-demo/tests/built-bin.e2e.ts packages/examples/acp-demo/tests/built-bin.e2e.ts packages/ui/jsonrpc/tests/built-scope-carrier.e2e.ts packages/workflow/workflow-workerthread/tests/built-worker.e2e.ts packages/code-runtime/code-runtime-worker/tests/built-lib.e2e.ts pnpm exec vitest run --config vitest.e2e.config.ts packages/examples/stdio-demo/tests/built-bin.e2e.ts packages/examples/cli-demo/tests/built-bin.e2e.ts packages/examples/acp-demo/tests/built-bin.e2e.ts packages/ui/jsonrpc/tests/built-scope-carrier.e2e.ts packages/workflow/workflow-workerthread/tests/built-worker.e2e.ts packages/code-runtime/code-runtime-worker/tests/built-lib.e2e.ts
``` ```

View File

@@ -58,6 +58,8 @@ export interface Config {
dshHome?: string dshHome?: string
/** Directory the JSONL session backend writes under. Defaults to `./.sessions`. */ /** Directory the JSONL session backend writes under. Defaults to `./.sessions`. */
persistenceRoot?: string persistenceRoot?: string
/** JSONL artifact encoding; defaults to checksummed Zstandard frames. */
persistenceCompression?: JsonlCompression
/** Controls automatic AGENTS.md/CLAUDE.md loading; configure a byte budget or set `false`. */ /** Controls automatic AGENTS.md/CLAUDE.md loading; configure a byte budget or set `false`. */
workspaceContext: agentCore.Config['workspaceContext'] workspaceContext: agentCore.Config['workspaceContext']
/** Skill registry, local-provider, and model-facing consumer config forwarded to agent-spine-demo. */ /** Skill registry, local-provider, and model-facing consumer config forwarded to agent-spine-demo. */
@@ -69,9 +71,9 @@ export interface Config {
} }
``` ```
Depends on: [`agentCore`](../packages/examples/agent-spine-demo/src/index.ts) · [`ToolsConfig`](#deepseek-aidsh-tools) Depends on: [`agentCore`](../packages/examples/agent-spine-demo/src/index.ts) · [`JsonlCompression`](../packages/session-persistence/session-persistence-jsonl/src/index.ts) · [`ToolsConfig`](#deepseek-aidsh-tools)
Source: [`packages/examples/acp-demo/src/index.ts:33`](../packages/examples/acp-demo/src/index.ts) Source: [`packages/examples/acp-demo/src/index.ts:36`](../packages/examples/acp-demo/src/index.ts)
## `@deepseek-ai/dsh-agent-loop` ## `@deepseek-ai/dsh-agent-loop`
@@ -231,6 +233,8 @@ export interface Config {
dshHome?: string dshHome?: string
/** Directory the JSONL session backend writes under. Defaults to `./.sessions`. */ /** Directory the JSONL session backend writes under. Defaults to `./.sessions`. */
persistenceRoot?: string persistenceRoot?: string
/** JSONL artifact encoding; defaults to checksummed Zstandard frames. */
persistenceCompression?: JsonlCompression
/** Skill registry, local-provider, and model-facing consumer config. */ /** Skill registry, local-provider, and model-facing consumer config. */
skills?: agentCore.SkillConfig skills?: agentCore.SkillConfig
/** Model-facing bash tool config forwarded through agent-spine-demo. */ /** Model-facing bash tool config forwarded through agent-spine-demo. */
@@ -242,9 +246,9 @@ export interface Config {
} }
``` ```
Depends on: [`agentCore`](../packages/examples/agent-spine-demo/src/index.ts) · [`ToolsConfig`](#deepseek-aidsh-tools) Depends on: [`agentCore`](../packages/examples/agent-spine-demo/src/index.ts) · [`JsonlCompression`](../packages/session-persistence/session-persistence-jsonl/src/index.ts) · [`ToolsConfig`](#deepseek-aidsh-tools)
Source: [`packages/examples/cli-demo/src/index.ts:22`](../packages/examples/cli-demo/src/index.ts) Source: [`packages/examples/cli-demo/src/index.ts:25`](../packages/examples/cli-demo/src/index.ts)
## `@deepseek-ai/dsh-code-runtime-worker` ## `@deepseek-ai/dsh-code-runtime-worker`
@@ -686,10 +690,15 @@ export interface Config {
* (bash calls, subprocesses). Sessions group under per-cwd subdirectories. * (bash calls, subprocesses). Sessions group under per-cwd subdirectories.
*/ */
root: string root: string
/** Physical encoding; defaults to checksummed Zstandard frames. */
compression?: JsonlCompression
} }
/** Physical encoding selected for JSONL session artifacts. */
export type JsonlCompression = 'zstd' | 'none'
``` ```
Source: [`packages/session-persistence/session-persistence-jsonl/src/index.ts:24`](../packages/session-persistence/session-persistence-jsonl/src/index.ts) Source: [`packages/session-persistence/session-persistence-jsonl/src/index.ts:36`](../packages/session-persistence/session-persistence-jsonl/src/index.ts)
## `@deepseek-ai/dsh-session-persistence-sqlite` ## `@deepseek-ai/dsh-session-persistence-sqlite`
@@ -854,6 +863,8 @@ export interface Config {
dshHome?: string dshHome?: string
/** Directory the JSONL session backend writes under. Defaults to `./.sessions`. */ /** Directory the JSONL session backend writes under. Defaults to `./.sessions`. */
persistenceRoot?: string persistenceRoot?: string
/** JSONL artifact encoding; defaults to checksummed Zstandard frames. */
persistenceCompression?: JsonlCompression
/** stdin-chat banner printed once on start. Defaults to `'ready.'`. */ /** stdin-chat banner printed once on start. Defaults to `'ready.'`. */
welcome?: string welcome?: string
/** Terminal front-door selection and pi-tui presentation settings. */ /** Terminal front-door selection and pi-tui presentation settings. */
@@ -886,9 +897,9 @@ export interface UiConfig {
export type TerminalMode = 'auto' | 'readline' | 'tui' export type TerminalMode = 'auto' | 'readline' | 'tui'
``` ```
Depends on: [`agentCore`](../packages/examples/agent-spine-demo/src/index.ts) · [`ToolsConfig`](#deepseek-aidsh-tools) · [`uiTui`](../packages/ui/tui/src/index.ts) Depends on: [`agentCore`](../packages/examples/agent-spine-demo/src/index.ts) · [`JsonlCompression`](../packages/session-persistence/session-persistence-jsonl/src/index.ts) · [`ToolsConfig`](#deepseek-aidsh-tools) · [`uiTui`](../packages/ui/tui/src/index.ts)
Source: [`packages/examples/stdio-demo/src/index.ts:75`](../packages/examples/stdio-demo/src/index.ts) Source: [`packages/examples/stdio-demo/src/index.ts:78`](../packages/examples/stdio-demo/src/index.ts)
## `@deepseek-ai/dsh-subagent-acp` ## `@deepseek-ai/dsh-subagent-acp`

View File

@@ -13,6 +13,7 @@
provider: deepseek provider: deepseek
model: deepseek-v4-flash model: deepseek-v4-flash
persistenceRoot: !!js process.env.DSH_SNAPSHOT_SESSIONS_ROOT ?? './.sessions' persistenceRoot: !!js process.env.DSH_SNAPSHOT_SESSIONS_ROOT ?? './.sessions'
persistenceCompression: 'none'
workspaceContext: workspaceContext:
maxBytes: 65536 maxBytes: 65536
tools: tools:

View File

@@ -11,6 +11,7 @@
provider: deepseek provider: deepseek
model: deepseek-v4-flash model: deepseek-v4-flash
persistenceRoot: !!js process.env.DSH_SNAPSHOT_SESSIONS_ROOT ?? './.sessions' persistenceRoot: !!js process.env.DSH_SNAPSHOT_SESSIONS_ROOT ?? './.sessions'
persistenceCompression: !!js "process.env.DSH_SNAPSHOT === undefined ? 'zstd' : 'none'"
workspaceContext: workspaceContext:
maxBytes: 65536 maxBytes: 65536
tools: tools:

View File

@@ -15,6 +15,7 @@
provider: deepseek provider: deepseek
model: deepseek-v4-flash model: deepseek-v4-flash
persistenceRoot: !!js process.env.DSH_SNAPSHOT_SESSIONS_ROOT ?? './.sessions' persistenceRoot: !!js process.env.DSH_SNAPSHOT_SESSIONS_ROOT ?? './.sessions'
persistenceCompression: 'none'
workspaceContext: workspaceContext:
maxBytes: 65536 maxBytes: 65536
tools: tools:

View File

@@ -13,6 +13,7 @@
provider: deepseek provider: deepseek
model: deepseek-v4-flash model: deepseek-v4-flash
persistenceRoot: !!js process.env.DSH_SNAPSHOT_SESSIONS_ROOT ?? './.sessions' persistenceRoot: !!js process.env.DSH_SNAPSHOT_SESSIONS_ROOT ?? './.sessions'
persistenceCompression: !!js "process.env.DSH_SNAPSHOT === undefined ? 'zstd' : 'none'"
workspaceContext: workspaceContext:
maxBytes: 65536 maxBytes: 65536
tools: tools:

View File

@@ -14,6 +14,7 @@
provider: deepseek provider: deepseek
model: deepseek-v4-flash model: deepseek-v4-flash
persistenceRoot: !!js process.env.DSH_SNAPSHOT_SESSIONS_ROOT ?? './.sessions' persistenceRoot: !!js process.env.DSH_SNAPSHOT_SESSIONS_ROOT ?? './.sessions'
persistenceCompression: 'none'
workspaceContext: workspaceContext:
maxBytes: 65536 maxBytes: 65536
tools: tools:

View File

@@ -11,6 +11,7 @@
provider: deepseek provider: deepseek
model: deepseek-v4-flash model: deepseek-v4-flash
persistenceRoot: !!js process.env.DSH_SNAPSHOT_SESSIONS_ROOT ?? './.sessions' persistenceRoot: !!js process.env.DSH_SNAPSHOT_SESSIONS_ROOT ?? './.sessions'
persistenceCompression: !!js "process.env.DSH_SNAPSHOT === undefined ? 'zstd' : 'none'"
workspaceContext: workspaceContext:
maxBytes: 65536 maxBytes: 65536
tools: tools:

View File

@@ -15,6 +15,7 @@
provider: deepseek provider: deepseek
model: deepseek-v4-flash model: deepseek-v4-flash
persistenceRoot: !!js process.env.DSH_SNAPSHOT_SESSIONS_ROOT ?? './.sessions' persistenceRoot: !!js process.env.DSH_SNAPSHOT_SESSIONS_ROOT ?? './.sessions'
persistenceCompression: 'none'
workspaceContext: workspaceContext:
maxBytes: 65536 maxBytes: 65536
tools: tools:

View File

@@ -14,6 +14,7 @@
provider: deepseek provider: deepseek
model: deepseek-v4-flash model: deepseek-v4-flash
persistenceRoot: !!js process.env.DSH_SNAPSHOT_SESSIONS_ROOT ?? './.sessions' persistenceRoot: !!js process.env.DSH_SNAPSHOT_SESSIONS_ROOT ?? './.sessions'
persistenceCompression: !!js "process.env.DSH_SNAPSHOT === undefined ? 'zstd' : 'none'"
workspaceContext: workspaceContext:
maxBytes: 65536 maxBytes: 65536
tools: tools:

View File

@@ -33,12 +33,14 @@
# The ACP server app: the agent-spine-demo spine + JSONL persistence + the ACP bridge. # The ACP server app: the agent-spine-demo spine + JSONL persistence + the ACP bridge.
# Persistence root: $DSH_SNAPSHOT_SESSIONS_ROOT when the snapshot harness sets it # Persistence root: $DSH_SNAPSHOT_SESSIONS_ROOT when the snapshot harness sets it
# (so it can harvest / isolate the log), else ./.sessions for the demo. # (so it can harvest / isolate the log), else ./.sessions for the demo.
# Snapshot modes use raw JSONL fixtures; ordinary runs keep the compressed default.
- id: acp-agent - id: acp-agent
name: '@deepseek-ai/dsh-acp-demo' name: '@deepseek-ai/dsh-acp-demo'
config: config:
provider: deepseek provider: deepseek
model: deepseek-v4-flash model: deepseek-v4-flash
persistenceRoot: !!js process.env.DSH_SNAPSHOT_SESSIONS_ROOT ?? './.sessions' persistenceRoot: !!js process.env.DSH_SNAPSHOT_SESSIONS_ROOT ?? './.sessions'
persistenceCompression: !!js "process.env.DSH_SNAPSHOT === undefined ? 'zstd' : 'none'"
workspaceContext: workspaceContext:
maxBytes: 65536 maxBytes: 65536
# Keep the persona to identity and behavior; tool plugins own tool guidance. # Keep the persona to identity and behavior; tool plugins own tool guidance.

View File

@@ -15,6 +15,7 @@
provider: deepseek provider: deepseek
model: deepseek-v4-flash model: deepseek-v4-flash
persistenceRoot: !!js process.env.DSH_SNAPSHOT_SESSIONS_ROOT ?? './.sessions' persistenceRoot: !!js process.env.DSH_SNAPSHOT_SESSIONS_ROOT ?? './.sessions'
persistenceCompression: 'none'
workspaceContext: workspaceContext:
maxBytes: 65536 maxBytes: 65536
dshHome: !!js process.cwd() + '/.dsh' dshHome: !!js process.cwd() + '/.dsh'

View File

@@ -12,6 +12,7 @@
provider: deepseek provider: deepseek
model: deepseek-v4-flash model: deepseek-v4-flash
persistenceRoot: !!js process.env.DSH_SNAPSHOT_SESSIONS_ROOT ?? './.sessions' persistenceRoot: !!js process.env.DSH_SNAPSHOT_SESSIONS_ROOT ?? './.sessions'
persistenceCompression: !!js "process.env.DSH_SNAPSHOT === undefined ? 'zstd' : 'none'"
workspaceContext: workspaceContext:
maxBytes: 65536 maxBytes: 65536
dshHome: !!js process.cwd() + '/.dsh' dshHome: !!js process.cwd() + '/.dsh'

View File

@@ -31,4 +31,4 @@ node --expose-internals --import tsx packages/examples/stdio-demo/src/bin.ts exa
Type a message and press Enter. "echo <text>" triggers a tool call round-trip (the mock model requests the `echo` tool, which echoes the text uppercased, and the next model step acknowledges it). Type a message and press Enter. "echo <text>" triggers a tool call round-trip (the mock model requests the `echo` tool, which echoes the text uppercased, and the next model step acknowledges it).
The session is persisted under `.sessions/` relative to the directory you launch the demo from. `pnpm run demo:echo` runs from the repo root, so the logs land in `<repo-root>/.sessions/cwd-<hash>/` (one `.jsonl` log per session). Clean up with: `rm -rf .sessions` The session is persisted under `.sessions/` relative to the directory you launch the demo from. `pnpm run demo:echo` runs from the repo root, so the logs land in `<repo-root>/.sessions/cwd-<hash>/` (one `.jsonl.zstd` log per session). Clean up with: `rm -rf .sessions`

View File

@@ -16,4 +16,5 @@
persona: 'Test the time-context plugin.' persona: 'Test the time-context plugin.'
welcome: 'time-context e2e ready.' welcome: 'time-context e2e ready.'
persistenceRoot: './.sessions' persistenceRoot: './.sessions'
persistenceCompression: 'none'
workspaceContext: false workspaceContext: false

View File

@@ -10,6 +10,7 @@
provider: deepseek provider: deepseek
model: deepseek-v4-flash model: deepseek-v4-flash
persistenceRoot: './.sessions' persistenceRoot: './.sessions'
persistenceCompression: !!js "process.env.DSH_SNAPSHOT === undefined ? 'zstd' : 'none'"
workspaceContext: workspaceContext:
maxBytes: 65536 maxBytes: 65536
tools: tools:

View File

@@ -25,6 +25,7 @@
provider: deepseek provider: deepseek
model: deepseek-v4-flash model: deepseek-v4-flash
persistenceRoot: './.sessions' persistenceRoot: './.sessions'
persistenceCompression: !!js "process.env.DSH_SNAPSHOT === undefined ? 'zstd' : 'none'"
workspaceContext: workspaceContext:
maxBytes: 65536 maxBytes: 65536
persona: | persona: |

View File

@@ -1,4 +1,7 @@
import { readdir } from 'node:fs/promises' import { readFile, readdir } from 'node:fs/promises'
import { zstdDecompress } from 'node:zlib'
import { promisify } from 'node:util'
import { join } from 'node:path'
import { fileURLToPath } from 'node:url' import { fileURLToPath } from 'node:url'
import { describe, expect, it } from 'vitest' import { describe, expect, it } from 'vitest'
import { LOADER_SMOKE_TEST_TIMEOUT_MS, runLoaderSmoke } from '@deepseek-ai/dsh-loader-smoke' import { LOADER_SMOKE_TEST_TIMEOUT_MS, runLoaderSmoke } from '@deepseek-ai/dsh-loader-smoke'
@@ -7,10 +10,11 @@ import type { SessionEvent } from '@deepseek-ai/dsh-session'
const binScript = fileURLToPath(new URL('../../../packages/examples/cli-demo/src/bin.ts', import.meta.url)) const binScript = fileURLToPath(new URL('../../../packages/examples/cli-demo/src/bin.ts', import.meta.url))
const configPath = fileURLToPath(new URL('./fixtures/cli.cordis.yml', import.meta.url)) const configPath = fileURLToPath(new URL('./fixtures/cli.cordis.yml', import.meta.url))
const tsconfigPath = fileURLToPath(new URL('../../../tsconfig.json', import.meta.url)) const tsconfigPath = fileURLToPath(new URL('../../../tsconfig.json', import.meta.url))
const decompress = promisify(zstdDecompress)
describe('headless-agent keyless smoke', () => { describe('headless-agent keyless smoke', () => {
it('boots the real Loader tree, runs a real bash tool round trip, and persists the turn', async () => { it('boots the real Loader tree, runs a real bash tool round trip, and persists the turn', async () => {
let persisted = false let persistedHeader: Record<string, unknown> | undefined
const { stdout, stderr } = await runLoaderSmoke({ const { stdout, stderr } = await runLoaderSmoke({
label: 'headless-agent', label: 'headless-agent',
tempDirPrefix: 'headless-agent-smoke-', tempDirPrefix: 'headless-agent-smoke-',
@@ -20,7 +24,11 @@ describe('headless-agent keyless smoke', () => {
tsconfigPath, tsconfigPath,
inspect: async (cwd) => { inspect: async (cwd) => {
const files = await readdir(cwd, { recursive: true }) const files = await readdir(cwd, { recursive: true })
persisted = files.some(file => file.endsWith('.jsonl')) const relativePath = files.find(file => file.endsWith('.jsonl.zstd'))
if (relativePath === undefined) return
const compressed = await readFile(join(cwd, relativePath))
expect(compressed.subarray(0, 4).toString('hex')).toBe('28b52ffd')
persistedHeader = JSON.parse((await decompress(compressed)).toString()) as Record<string, unknown>
}, },
}) })
const lines = stdout.trimEnd().split('\n').map(line => JSON.parse(line) as Record<string, unknown>) const lines = stdout.trimEnd().split('\n').map(line => JSON.parse(line) as Record<string, unknown>)
@@ -38,6 +46,6 @@ describe('headless-agent keyless smoke', () => {
usage: { inputTokens: 18, outputTokens: 8, cacheReadTokens: 2, reasoningTokens: 1 }, usage: { inputTokens: 18, outputTokens: 8, cacheReadTokens: 2, reasoningTokens: 1 },
}) })
expect(String(result?.['result'])).toContain('CLI_TOOL_ROUND_TRIP') expect(String(result?.['result'])).toContain('CLI_TOOL_ROUND_TRIP')
expect(persisted).toBe(true) expect(persistedHeader).toMatchObject({ type: 'session' })
}, LOADER_SMOKE_TEST_TIMEOUT_MS) }, LOADER_SMOKE_TEST_TIMEOUT_MS)
}) })

View File

@@ -1,14 +1,17 @@
import { spawn } from 'node:child_process' import { spawn } from 'node:child_process'
import { createServer } from 'node:http' import { createServer } from 'node:http'
import { mkdtemp, rm } from 'node:fs/promises' import { mkdtemp, readFile, readdir, rm } from 'node:fs/promises'
import { tmpdir } from 'node:os' import { tmpdir } from 'node:os'
import { join } from 'node:path' import { join } from 'node:path'
import { fileURLToPath } from 'node:url' import { fileURLToPath } from 'node:url'
import { promisify } from 'node:util'
import { zstdDecompress } from 'node:zlib'
import { describe, expect, it } from 'vitest' import { describe, expect, it } from 'vitest'
const binScript = fileURLToPath(new URL('../../../packages/examples/jsonrpc-demo/src/bin.ts', import.meta.url)) const binScript = fileURLToPath(new URL('../../../packages/examples/jsonrpc-demo/src/bin.ts', import.meta.url))
const configPath = fileURLToPath(new URL('../cordis.yml', import.meta.url)) const configPath = fileURLToPath(new URL('../cordis.yml', import.meta.url))
const repoRoot = fileURLToPath(new URL('../../..', import.meta.url)) const repoRoot = fileURLToPath(new URL('../../..', import.meta.url))
const decompress = promisify(zstdDecompress)
function waitForLine( function waitForLine(
lines: string[], lines: string[],
@@ -152,6 +155,13 @@ describe('jsonrpc-agent keyless smoke', () => {
} else { } else {
expect(child.exitCode, stderr).toBe(0) expect(child.exitCode, stderr).toBe(0)
} }
const sessionsRoot = join(root, '.sessions')
const files = await readdir(sessionsRoot, { recursive: true })
const log = files.find(file => file.endsWith('.jsonl.zstd'))
expect(log).toBeDefined()
const compressed = await readFile(join(sessionsRoot, log!))
expect(compressed.subarray(0, 4).toString('hex')).toBe('28b52ffd')
expect(JSON.parse((await decompress(compressed)).toString())).toMatchObject({ type: 'session', id: 'main' })
} finally { } finally {
if (child.exitCode === null) child.kill('SIGKILL') if (child.exitCode === null) child.kill('SIGKILL')
await new Promise<void>(resolve => modelServer.close(() => { resolve() })) await new Promise<void>(resolve => modelServer.close(() => { resolve() }))

View File

@@ -23,7 +23,9 @@ import { MockAdapter, textResponse, toolCallResponse } from '../../../core/agent
async function harness(adapter: MockAdapter, sessionRoot?: string, dshHome?: string) { async function harness(adapter: MockAdapter, sessionRoot?: string, dshHome?: string) {
const ctx = new Context() const ctx = new Context()
await mountAgentLoopTestDependencies(ctx) await mountAgentLoopTestDependencies(ctx)
if (sessionRoot !== undefined) await ctx.plugin(SessionPersistenceJsonl, { root: sessionRoot }) if (sessionRoot !== undefined) {
await ctx.plugin(SessionPersistenceJsonl, { root: sessionRoot, compression: 'none' })
}
await ctx.plugin(AgentLoop, { agents: [] }) await ctx.plugin(AgentLoop, { agents: [] })
await ctx.plugin(TaskService) await ctx.plugin(TaskService)
await ctx.plugin(ToolTasks) await ctx.plugin(ToolTasks)

View File

@@ -36,6 +36,7 @@ Because the package wires no logger entry, an ACP leaf has **nothing to get wron
| `toolBash` | owner defaults | model-facing bash config routed through `dsh-agent-spine-demo`, including bash's producer-local `enableRunInBackground` | | `toolBash` | owner defaults | model-facing bash config routed through `dsh-agent-spine-demo`, including bash's producer-local `enableRunInBackground` |
| `toolTasks` | owner defaults | generic `task_output` wait bounds routed through `dsh-agent-spine-demo` | | `toolTasks` | owner defaults | generic `task_output` wait bounds routed through `dsh-agent-spine-demo` |
| `persistenceRoot` | `./.sessions` | the JSONL backend's root directory | | `persistenceRoot` | `./.sessions` | the JSONL backend's root directory |
| `persistenceCompression` | `'zstd'` | JSONL artifact encoding (`'zstd'` or raw `'none'`) |
The leaf supplies the swappable backends: an LLM adapter (`llm-deepseek` for the real model, `llm-replay` for keyless snapshot replay) and a bash executor. The leaf supplies the swappable backends: an LLM adapter (`llm-deepseek` for the real model, `llm-replay` for keyless snapshot replay) and a bash executor.

View File

@@ -15,7 +15,10 @@ import * as acp from '@deepseek-ai/dsh-acp'
import * as agentCore from '@deepseek-ai/dsh-agent-spine-demo' import * as agentCore from '@deepseek-ai/dsh-agent-spine-demo'
import * as workspaceContext from '@deepseek-ai/dsh-workspace-context' import * as workspaceContext from '@deepseek-ai/dsh-workspace-context'
import ToolRegistry, { type Config as ToolsConfig } from '@deepseek-ai/dsh-tools' import ToolRegistry, { type Config as ToolsConfig } from '@deepseek-ai/dsh-tools'
import SessionPersistenceJsonl from '@deepseek-ai/dsh-session-persistence-jsonl' import SessionPersistenceJsonl, {
JsonlCompressionSchema,
type JsonlCompression,
} from '@deepseek-ai/dsh-session-persistence-jsonl'
import UserInteractionService from '@deepseek-ai/dsh-user-interaction' import UserInteractionService from '@deepseek-ai/dsh-user-interaction'
export const name = 'acp-demo' export const name = 'acp-demo'
@@ -47,6 +50,8 @@ export interface Config {
dshHome?: string dshHome?: string
/** Directory the JSONL session backend writes under. Defaults to `./.sessions`. */ /** Directory the JSONL session backend writes under. Defaults to `./.sessions`. */
persistenceRoot?: string persistenceRoot?: string
/** JSONL artifact encoding; defaults to checksummed Zstandard frames. */
persistenceCompression?: JsonlCompression
/** Controls automatic AGENTS.md/CLAUDE.md loading; configure a byte budget or set `false`. */ /** Controls automatic AGENTS.md/CLAUDE.md loading; configure a byte budget or set `false`. */
workspaceContext: agentCore.Config['workspaceContext'] workspaceContext: agentCore.Config['workspaceContext']
/** Skill registry, local-provider, and model-facing consumer config forwarded to agent-spine-demo. */ /** Skill registry, local-provider, and model-facing consumer config forwarded to agent-spine-demo. */
@@ -72,6 +77,7 @@ export const Config: z<Config> = z.object({
tools: ToolRegistry.Config, tools: ToolRegistry.Config,
dshHome: z.string(), dshHome: z.string(),
persistenceRoot: z.string().default(DEFAULT_PERSISTENCE_ROOT), persistenceRoot: z.string().default(DEFAULT_PERSISTENCE_ROOT),
persistenceCompression: JsonlCompressionSchema,
workspaceContext: z.union([z.const(false), workspaceContext.Config]).required(), workspaceContext: z.union([z.const(false), workspaceContext.Config]).required(),
skills: agentCore.SkillConfigSchema, skills: agentCore.SkillConfigSchema,
toolBash: agentCore.ToolBashConfigSchema, toolBash: agentCore.ToolBashConfigSchema,
@@ -89,6 +95,9 @@ export const Config: z<Config> = z.object({
export function apply(ctx: Context, config: Config): void { export function apply(ctx: Context, config: Config): void {
ctx.plugin(agentCore, agentCore.pickSpineConfig(config)) ctx.plugin(agentCore, agentCore.pickSpineConfig(config))
ctx.plugin(UserInteractionService) ctx.plugin(UserInteractionService)
ctx.plugin(SessionPersistenceJsonl, { root: config.persistenceRoot ?? DEFAULT_PERSISTENCE_ROOT }) ctx.plugin(SessionPersistenceJsonl, {
root: config.persistenceRoot ?? DEFAULT_PERSISTENCE_ROOT,
...(config.persistenceCompression === undefined ? {} : { compression: config.persistenceCompression }),
})
ctx.plugin(acp, { provider: config.provider, model: config.model }) ctx.plugin(acp, { provider: config.provider, model: config.model })
} }

View File

@@ -70,10 +70,19 @@ async function withIsolatedSkillHomes<T>(run: () => Promise<T>): Promise<T> {
describe('dsh-acp-demo composition', () => { describe('dsh-acp-demo composition', () => {
it('brings up the spine + persistence + the ACP bridge', async () => { it('brings up the spine + persistence + the ACP bridge', async () => {
const ctx = await mount({ provider: 'mock', model: 'mock', persona: 'hi', persistenceRoot: '/tmp/dsh-acp-demo-test', skills: await isolatedSkillsConfig(), workspaceContext: false }) const ctx = await mount({
provider: 'mock',
model: 'mock',
persona: 'hi',
persistenceRoot: '/tmp/dsh-acp-demo-test',
persistenceCompression: 'none',
skills: await isolatedSkillsConfig(),
workspaceContext: false,
})
expect(ctx.get('agents')).toBeDefined() expect(ctx.get('agents')).toBeDefined()
expect(ctx.get('sessions')).toBeDefined() expect(ctx.get('sessions')).toBeDefined()
expect(ctx.get('sessionPersistence')).toBeDefined() expect(ctx.get('sessionPersistence')).toBeDefined()
expect((ctx.get('sessionPersistence') as unknown as { config: { compression?: string } }).config.compression).toBe('none')
expect(ctx.get('agentLoop')).toBeDefined() expect(ctx.get('agentLoop')).toBeDefined()
expect(ctx.get('userInteraction')).toBeDefined() expect(ctx.get('userInteraction')).toBeDefined()
expect(ctx.get('tools')?.get('ask_user_question')).toBeUndefined() expect(ctx.get('tools')?.get('ask_user_question')).toBeUndefined()

View File

@@ -1,5 +1,5 @@
import { spawn } from 'node:child_process' import { spawn } from 'node:child_process'
import { mkdtemp, mkdir, rm, symlink, writeFile, readFile } from 'node:fs/promises' import { mkdtemp, mkdir, readdir, rm, symlink, writeFile, readFile } from 'node:fs/promises'
import { existsSync } from 'node:fs' import { existsSync } from 'node:fs'
import { tmpdir } from 'node:os' import { tmpdir } from 'node:os'
import { dirname, join } from 'node:path' import { dirname, join } from 'node:path'
@@ -15,21 +15,24 @@ import {
type SessionNotification, type SessionNotification,
} from '@agentclientprotocol/sdk' } from '@agentclientprotocol/sdk'
import { Readable, Writable } from 'node:stream' import { Readable, Writable } from 'node:stream'
import { promisify } from 'node:util'
import { zstdDecompress } from 'node:zlib'
import { afterEach, describe, expect, it } from 'vitest' import { afterEach, describe, expect, it } from 'vitest'
/** /**
* Published-entry smoke: run `lib/bin.js` under plain Node in a symlinked external consumer and * Published-entry smoke: run `lib/bin.js` under plain Node in a symlinked external consumer and
* require a valid initialize response. This catches built-only settle races and stdout protocol * complete a mock-backed turn. This catches built-only settle races, stdout protocol leaks, and
* leaks that the tsx source-path smoke cannot. It skips before build; initialize is keyless, with a * published persistence behavior that the tsx source-path smoke cannot. It skips before build;
* dummy key used only to boot the adapter. `--expose-internals` enables Cordis bare-plugin loading. * `--expose-internals` enables Cordis bare-plugin loading.
*/ */
const repoRoot = fileURLToPath(new URL('../../../../', import.meta.url)) const repoRoot = fileURLToPath(new URL('../../../../', import.meta.url))
const acpBin = join(repoRoot, 'packages/examples/acp-demo/lib/bin.js') const acpBin = join(repoRoot, 'packages/examples/acp-demo/lib/bin.js')
const decompress = promisify(zstdDecompress)
const dshPackages = [ const dshPackages = [
'examples/agent-spine-demo', 'core/agent', 'core/session', 'core/system-prompt', 'examples/agent-spine-demo', 'core/agent', 'core/session', 'core/system-prompt',
'core/tools', 'core/agent-loop', 'llm/llm', 'llm/llm-deepseek', 'bash/bash', 'core/tools', 'core/agent-loop', 'llm/llm', 'bash/bash',
'bash/bash-local', 'bash/tool-bash', 'context/workspace-context', 'support/invariants', 'ui/app-boot', 'bash/bash-local', 'bash/tool-bash', 'context/workspace-context', 'support/invariants', 'ui/app-boot',
'session-persistence/session-persistence', 'session-persistence/session-persistence',
'session-persistence/session-persistence-jsonl', 'ui/acp', 'examples/acp-demo', 'util/paths', 'session-persistence/session-persistence-jsonl', 'ui/acp', 'examples/acp-demo', 'util/paths',
@@ -73,18 +76,31 @@ async function makeConsumer(): Promise<string> {
const resolved = fileURLToPath(import.meta.resolve(`${dep}/package.json`, fromAcp)) const resolved = fileURLToPath(import.meta.resolve(`${dep}/package.json`, fromAcp))
await link(dirname(resolved), dep, nm) await link(dirname(resolved), dep, nm)
} }
await writeFile(join(dir, 'mock-llm.mjs'), [
"import { LlmAdapter } from '@deepseek-ai/dsh-llm'",
'class Mock extends LlmAdapter {',
' async * stream() {',
" yield { type: 'block-start', index: 0, blockType: 'text' }",
" yield { type: 'text-delta', index: 0, text: 'ACP BUILT OK' }",
" yield { type: 'block-end', index: 0, block: { type: 'text', text: 'ACP BUILT OK' } }",
" yield { type: 'finish', reason: { kind: 'stop' } }",
' }',
'}',
"export const name = 'built-acp-mock'",
"export const inject = ['llm']",
"export function apply(ctx) { ctx.llm.registerAdapter(['built-acp-mock'], new Mock()) }",
'',
].join('\n'))
await writeFile(join(dir, 'cordis.yml'), [ await writeFile(join(dir, 'cordis.yml'), [
'- id: llm-deepseek', '- id: mock-llm',
' name: \'@deepseek-ai/dsh-llm-deepseek\'', ' name: \'./mock-llm.mjs\'',
' config:',
' apiKey: !!js process.env.DEEPSEEK_API_KEY',
'- id: bash', '- id: bash',
' name: \'@deepseek-ai/dsh-bash-local\'', ' name: \'@deepseek-ai/dsh-bash-local\'',
'- id: acp-agent', '- id: acp-agent',
' name: \'@deepseek-ai/dsh-acp-demo\'', ' name: \'@deepseek-ai/dsh-acp-demo\'',
' config:', ' config:',
' provider: deepseek', ' provider: built-acp-mock',
' model: deepseek-v4-flash', ' model: built-acp-mock',
' persona: \'test agent\'', ' persona: \'test agent\'',
' workspaceContext: false', ' workspaceContext: false',
'', '',
@@ -113,14 +129,12 @@ afterEach(async () => {
}) })
describe.skipIf(!existsSync(acpBin))('dsh-acp-demo BUILT bin (node lib/bin.js, no tsx)', () => { describe.skipIf(!existsSync(acpBin))('dsh-acp-demo BUILT bin (node lib/bin.js, no tsx)', () => {
it('boots the published bin and answers an initialize JSON-RPC frame on stdout', async () => { it('boots the published bin, completes a turn, and writes default Zstandard persistence', async () => {
consumer = await makeConsumer() consumer = await makeConsumer()
child = spawn(process.execPath, ['--expose-internals', acpBin, '--config', './cordis.yml'], { child = spawn(process.execPath, ['--expose-internals', acpBin, '--config', './cordis.yml'], {
cwd: consumer, cwd: consumer,
// Dummy key: initialize never reaches the model, so it is never used.
env: { env: {
...process.env, ...process.env,
DEEPSEEK_API_KEY: process.env.DEEPSEEK_API_KEY ?? 'sk-dummy-for-boot',
DSH_HOME: join(consumer, '.dsh'), DSH_HOME: join(consumer, '.dsh'),
DSH_AGENTS_HOME: join(consumer, '.agents'), DSH_AGENTS_HOME: join(consumer, '.agents'),
}, },
@@ -151,6 +165,18 @@ describe.skipIf(!existsSync(acpBin))('dsh-acp-demo BUILT bin (node lib/bin.js, n
// regression would exit before answering); loadSession proves the real app // regression would exit before answering); loadSession proves the real app
// mounted, not a collapsed export shape. // mounted, not a collapsed export shape.
expect(init.agentCapabilities?.loadSession).toBe(true) expect(init.agentCapabilities?.loadSession).toBe(true)
const { sessionId } = await client.newSession({ cwd: consumer, mcpServers: [] })
const result = await client.prompt({ sessionId, prompt: [{ type: 'text', text: 'reply' }] })
expect(result.stopReason).toBe('end_turn')
const sessionsRoot = join(consumer, '.sessions')
let log: string | undefined
await expect.poll(async () => {
log = (await readdir(sessionsRoot, { recursive: true })).find(file => file.endsWith('.jsonl.zstd'))
return log
}).toBeTypeOf('string')
const compressed = await readFile(join(sessionsRoot, log!))
expect(compressed.subarray(0, 4).toString('hex')).toBe('28b52ffd')
expect(JSON.parse((await decompress(compressed)).toString())).toMatchObject({ type: 'session', id: sessionId })
expect(stderr.join('')).not.toContain('without inject') expect(stderr.join('')).not.toContain('without inject')
// stdout purity: every emitted line is a JSON-RPC frame, no logger leak. // stdout purity: every emitted line is a JSON-RPC frame, no logger leak.
for (const line of rawOut.join('').split('\n').filter(l => l.trim().length > 0)) { for (const line of rawOut.join('').split('\n').filter(l => l.trim().length > 0)) {
@@ -182,7 +208,6 @@ function runBinExpectingExit(configArg: string, cwd: string = tmpdir()): Promise
cwd, cwd,
env: { env: {
...process.env, ...process.env,
DEEPSEEK_API_KEY: process.env.DEEPSEEK_API_KEY ?? 'sk-dummy-for-boot',
DSH_HOME: join(cwd, '.dsh'), DSH_HOME: join(cwd, '.dsh'),
DSH_AGENTS_HOME: join(cwd, '.agents'), DSH_AGENTS_HOME: join(cwd, '.agents'),
}, },

View File

@@ -19,6 +19,7 @@ The package mounts no console logger, readline UI, user-interaction service, or
| `toolBash` | owner defaults | model-facing bash config, including this producer's background opt-in | | `toolBash` | owner defaults | model-facing bash config, including this producer's background opt-in |
| `toolTasks` | owner defaults | generic `task_output` wait bounds | | `toolTasks` | owner defaults | generic `task_output` wait bounds |
| `persistenceRoot` | `./.sessions` | JSONL session root | | `persistenceRoot` | `./.sessions` | JSONL session root |
| `persistenceCompression` | `'zstd'` | JSONL artifact encoding (`'zstd'` or raw `'none'`) |
| `workspaceContext` | required | workspace-instruction byte budget, or `false` to disable loading | | `workspaceContext` | required | workspace-instruction byte budget, or `false` to disable loading |
## CLI contract ## CLI contract

View File

@@ -11,7 +11,10 @@ import z from 'schemastery'
import { SessionId } from '@deepseek-ai/dsh-session' import { SessionId } from '@deepseek-ai/dsh-session'
import ToolRegistry, { type Config as ToolsConfig } from '@deepseek-ai/dsh-tools' import ToolRegistry, { type Config as ToolsConfig } from '@deepseek-ai/dsh-tools'
import * as agentCore from '@deepseek-ai/dsh-agent-spine-demo' import * as agentCore from '@deepseek-ai/dsh-agent-spine-demo'
import SessionPersistenceJsonl from '@deepseek-ai/dsh-session-persistence-jsonl' import SessionPersistenceJsonl, {
JsonlCompressionSchema,
type JsonlCompression,
} from '@deepseek-ai/dsh-session-persistence-jsonl'
import * as workspaceContext from '@deepseek-ai/dsh-workspace-context' import * as workspaceContext from '@deepseek-ai/dsh-workspace-context'
const DEFAULT_PERSISTENCE_ROOT = './.sessions' const DEFAULT_PERSISTENCE_ROOT = './.sessions'
@@ -36,6 +39,8 @@ export interface Config {
dshHome?: string dshHome?: string
/** Directory the JSONL session backend writes under. Defaults to `./.sessions`. */ /** Directory the JSONL session backend writes under. Defaults to `./.sessions`. */
persistenceRoot?: string persistenceRoot?: string
/** JSONL artifact encoding; defaults to checksummed Zstandard frames. */
persistenceCompression?: JsonlCompression
/** Skill registry, local-provider, and model-facing consumer config. */ /** Skill registry, local-provider, and model-facing consumer config. */
skills?: agentCore.SkillConfig skills?: agentCore.SkillConfig
/** Model-facing bash tool config forwarded through agent-spine-demo. */ /** Model-facing bash tool config forwarded through agent-spine-demo. */
@@ -54,6 +59,7 @@ export const Config: z<Config> = z.object({
model: z.string().required(), model: z.string().required(),
maxParallelToolCalls: z.number().step(1).min(1), maxParallelToolCalls: z.number().step(1).min(1),
persistenceRoot: z.string().default(DEFAULT_PERSISTENCE_ROOT), persistenceRoot: z.string().default(DEFAULT_PERSISTENCE_ROOT),
persistenceCompression: JsonlCompressionSchema,
persona: z.string(), persona: z.string(),
dshHome: z.string(), dshHome: z.string(),
skills: agentCore.SkillConfigSchema, skills: agentCore.SkillConfigSchema,
@@ -78,5 +84,8 @@ export function apply(ctx: Context, config: Config): void {
...agentCore.pickSpineConfig(config), ...agentCore.pickSpineConfig(config),
agents: [{ id: SessionId('main'), provider: config.provider, model: config.model, cwd: process.cwd() }], agents: [{ id: SessionId('main'), provider: config.provider, model: config.model, cwd: process.cwd() }],
}) })
ctx.plugin(SessionPersistenceJsonl, { root: config.persistenceRoot ?? DEFAULT_PERSISTENCE_ROOT }) ctx.plugin(SessionPersistenceJsonl, {
root: config.persistenceRoot ?? DEFAULT_PERSISTENCE_ROOT,
...(config.persistenceCompression === undefined ? {} : { compression: config.persistenceCompression }),
})
} }

View File

@@ -3,11 +3,14 @@ import { existsSync } from 'node:fs'
import { mkdtemp, mkdir, readFile, readdir, rm, symlink, writeFile } from 'node:fs/promises' import { mkdtemp, mkdir, readFile, readdir, rm, symlink, writeFile } from 'node:fs/promises'
import { tmpdir } from 'node:os' import { tmpdir } from 'node:os'
import { dirname, join } from 'node:path' import { dirname, join } from 'node:path'
import { promisify } from 'node:util'
import { fileURLToPath } from 'node:url' import { fileURLToPath } from 'node:url'
import { zstdDecompress } from 'node:zlib'
import { afterEach, describe, expect, it } from 'vitest' import { afterEach, describe, expect, it } from 'vitest'
const repoRoot = fileURLToPath(new URL('../../../../', import.meta.url)) const repoRoot = fileURLToPath(new URL('../../../../', import.meta.url))
const cliBin = join(repoRoot, 'packages/examples/cli-demo/lib/bin.js') const cliBin = join(repoRoot, 'packages/examples/cli-demo/lib/bin.js')
const decompress = promisify(zstdDecompress)
const dshPackages = [ const dshPackages = [
'examples/agent-spine-demo', 'examples/cli-demo', 'core/agent', 'core/session', 'examples/agent-spine-demo', 'examples/cli-demo', 'core/agent', 'core/session',
'core/system-prompt', 'core/tools', 'core/agent-loop', 'llm/llm', 'bash/bash', 'core/system-prompt', 'core/tools', 'core/agent-loop', 'llm/llm', 'bash/bash',
@@ -140,8 +143,13 @@ describe.skipIf(!existsSync(cliBin))('dsh-cli-demo BUILT bin', () => {
const lines = stream.stdout.trimEnd().split('\n').map(line => JSON.parse(line) as Record<string, unknown>) const lines = stream.stdout.trimEnd().split('\n').map(line => JSON.parse(line) as Record<string, unknown>)
expect(lines[0]).toMatchObject({ type: 'session_event', event: { type: 'turn/start' } }) expect(lines[0]).toMatchObject({ type: 'session_event', event: { type: 'turn/start' } })
expect(lines.at(-1)).toMatchObject({ type: 'result', success: true, result: 'BUILT: stream task' }) expect(lines.at(-1)).toMatchObject({ type: 'result', success: true, result: 'BUILT: stream task' })
const files = await readdir(join(consumer, '.sessions'), { recursive: true }) const sessionsRoot = join(consumer, '.sessions')
expect(files.filter(file => file.endsWith('.jsonl'))).toHaveLength(3) const files = await readdir(sessionsRoot, { recursive: true })
const logs = files.filter(file => file.endsWith('.jsonl.zstd'))
expect(logs).toHaveLength(3)
const compressed = await readFile(join(sessionsRoot, logs[0]!))
expect(compressed.subarray(0, 4).toString('hex')).toBe('28b52ffd')
expect(JSON.parse((await decompress(compressed)).toString())).toMatchObject({ type: 'session' })
}, 30_000) }, 30_000)
it('keeps stdout empty for invalid argv and missing config', async () => { it('keeps stdout empty for invalid argv and missing config', async () => {

View File

@@ -51,12 +51,14 @@ describe('dsh-cli-demo app composition', () => {
persona: 'Headless.', persona: 'Headless.',
tools: { mode: 'native' }, tools: { mode: 'native' },
persistenceRoot: root, persistenceRoot: root,
persistenceCompression: 'none',
skills: await skillConfig(), skills: await skillConfig(),
workspaceContext: false, workspaceContext: false,
}) })
const [agent] = ctx.get('agents')?.roots() ?? [] const [agent] = ctx.get('agents')?.roots() ?? []
expect(ctx.get('agentLoop')).toBeDefined() expect(ctx.get('agentLoop')).toBeDefined()
expect(ctx.get('sessionPersistence')).toBeDefined() expect(ctx.get('sessionPersistence')).toBeDefined()
expect((ctx.get('sessionPersistence') as unknown as { config: { compression?: string } }).config.compression).toBe('none')
expect(agent?.session.header.cwd).toBe(process.cwd()) expect(agent?.session.header.cwd).toBe(process.cwd())
expect(ctx.get('userInteraction')).toBeUndefined() expect(ctx.get('userInteraction')).toBeUndefined()
expect(ctx.get('tools')?.get('ask_user_question')).toBeUndefined() expect(ctx.get('tools')?.get('ask_user_question')).toBeUndefined()

View File

@@ -303,7 +303,7 @@ describe('runOneShot and executeCli', () => {
expect(output).toEqual({ code: 0, stdout: 'final answer\n', stderr: '' }) expect(output).toEqual({ code: 0, stdout: 'final answer\n', stderr: '' })
expect(agent.status).toBe('disposed') expect(agent.status).toBe('disposed')
const files = await readdir(persistenceRoot, { recursive: true }) const files = await readdir(persistenceRoot, { recursive: true })
expect(files.some(file => file.endsWith('.jsonl'))).toBe(true) expect(files.some(file => file.endsWith('.jsonl.zstd'))).toBe(true)
}) })
it('sums usage across tool steps and selects the last text-bearing assistant message', async () => { it('sums usage across tool steps and selects the last text-bearing assistant message', async () => {

View File

@@ -37,6 +37,7 @@ The leaf `cordis.yml` supplies only the **swappable backends** — an LLM adapte
| `toolBash` | owner defaults | model-facing bash config routed through `dsh-agent-spine-demo`, including bash's producer-local `enableRunInBackground` | | `toolBash` | owner defaults | model-facing bash config routed through `dsh-agent-spine-demo`, including bash's producer-local `enableRunInBackground` |
| `toolTasks` | owner defaults | generic `task_output` wait bounds routed through `dsh-agent-spine-demo` | | `toolTasks` | owner defaults | generic `task_output` wait bounds routed through `dsh-agent-spine-demo` |
| `persistenceRoot` | `./.sessions` | the JSONL backend's root directory | | `persistenceRoot` | `./.sessions` | the JSONL backend's root directory |
| `persistenceCompression` | `'zstd'` | JSONL artifact encoding (`'zstd'` or raw `'none'`) |
| `welcome` | `ready.` | terminal banner / TUI subtitle | | `welcome` | `ready.` | terminal banner / TUI subtitle |
| `ui` | `{ mode: 'auto' }` | terminal mode (`auto` / `readline` / `tui`) and nested TUI presentation config | | `ui` | `{ mode: 'auto' }` | terminal mode (`auto` / `readline` / `tui`) and nested TUI presentation config |
| `resumeSessionId` | — | resume a persisted session id instead of starting fresh (sourced from an env var in the leaf) | | `resumeSessionId` | — | resume a persisted session id instead of starting fresh (sourced from an env var in the leaf) |

View File

@@ -18,7 +18,10 @@ import { SessionId } from '@deepseek-ai/dsh-session'
import ToolRegistry, { type Config as ToolsConfig } from '@deepseek-ai/dsh-tools' import ToolRegistry, { type Config as ToolsConfig } from '@deepseek-ai/dsh-tools'
import * as agentCore from '@deepseek-ai/dsh-agent-spine-demo' import * as agentCore from '@deepseek-ai/dsh-agent-spine-demo'
import * as workspaceContext from '@deepseek-ai/dsh-workspace-context' import * as workspaceContext from '@deepseek-ai/dsh-workspace-context'
import SessionPersistenceJsonl from '@deepseek-ai/dsh-session-persistence-jsonl' import SessionPersistenceJsonl, {
JsonlCompressionSchema,
type JsonlCompression,
} from '@deepseek-ai/dsh-session-persistence-jsonl'
import UserInteractionService from '@deepseek-ai/dsh-user-interaction' import UserInteractionService from '@deepseek-ai/dsh-user-interaction'
import * as toolAskUser from '@deepseek-ai/dsh-tool-ask-user' import * as toolAskUser from '@deepseek-ai/dsh-tool-ask-user'
import * as uiStdio from '@deepseek-ai/dsh-stdio' import * as uiStdio from '@deepseek-ai/dsh-stdio'
@@ -89,6 +92,8 @@ export interface Config {
dshHome?: string dshHome?: string
/** Directory the JSONL session backend writes under. Defaults to `./.sessions`. */ /** Directory the JSONL session backend writes under. Defaults to `./.sessions`. */
persistenceRoot?: string persistenceRoot?: string
/** JSONL artifact encoding; defaults to checksummed Zstandard frames. */
persistenceCompression?: JsonlCompression
/** stdin-chat banner printed once on start. Defaults to `'ready.'`. */ /** stdin-chat banner printed once on start. Defaults to `'ready.'`. */
welcome?: string welcome?: string
/** Terminal front-door selection and pi-tui presentation settings. */ /** Terminal front-door selection and pi-tui presentation settings. */
@@ -121,6 +126,7 @@ export const Config: z<Config> = z.object({
tools: ToolRegistry.Config, tools: ToolRegistry.Config,
dshHome: z.string(), dshHome: z.string(),
persistenceRoot: z.string().default(DEFAULT_PERSISTENCE_ROOT), persistenceRoot: z.string().default(DEFAULT_PERSISTENCE_ROOT),
persistenceCompression: JsonlCompressionSchema,
welcome: z.string().default(DEFAULT_WELCOME), welcome: z.string().default(DEFAULT_WELCOME),
ui: UiConfigSchema, ui: UiConfigSchema,
skills: agentCore.SkillConfigSchema, skills: agentCore.SkillConfigSchema,
@@ -145,7 +151,10 @@ export function composeTerminalApp(ctx: Context, config: Config, isTTY: boolean)
const sessionId = SessionId(resumeSessionId ?? `main-session-${randomUUID()}`) const sessionId = SessionId(resumeSessionId ?? `main-session-${randomUUID()}`)
const mode = resolveTerminalMode(config.ui, isTTY) const mode = resolveTerminalMode(config.ui, isTTY)
if (mode === 'readline') ctx.plugin(ConsoleExporter) if (mode === 'readline') ctx.plugin(ConsoleExporter)
ctx.plugin(SessionPersistenceJsonl, { root: config.persistenceRoot ?? DEFAULT_PERSISTENCE_ROOT }) ctx.plugin(SessionPersistenceJsonl, {
root: config.persistenceRoot ?? DEFAULT_PERSISTENCE_ROOT,
...(config.persistenceCompression === undefined ? {} : { compression: config.persistenceCompression }),
})
ctx.plugin(UserInteractionService) ctx.plugin(UserInteractionService)
if (mode === 'tui') { if (mode === 'tui') {
ctx.plugin(uiTui, { ctx.plugin(uiTui, {

View File

@@ -1,9 +1,11 @@
import { spawn } from 'node:child_process' import { spawn } from 'node:child_process'
import { cp, mkdtemp, mkdir, rm, symlink, writeFile, readFile } from 'node:fs/promises' import { cp, mkdtemp, mkdir, readdir, rm, symlink, writeFile, readFile } from 'node:fs/promises'
import { existsSync } from 'node:fs' import { existsSync } from 'node:fs'
import { tmpdir } from 'node:os' import { tmpdir } from 'node:os'
import { dirname, join } from 'node:path' import { dirname, join } from 'node:path'
import { fileURLToPath } from 'node:url' import { fileURLToPath } from 'node:url'
import { promisify } from 'node:util'
import { zstdDecompress } from 'node:zlib'
import { afterEach, describe, expect, it } from 'vitest' import { afterEach, describe, expect, it } from 'vitest'
/** /**
@@ -15,6 +17,7 @@ import { afterEach, describe, expect, it } from 'vitest'
const repoRoot = fileURLToPath(new URL('../../../../', import.meta.url)) const repoRoot = fileURLToPath(new URL('../../../../', import.meta.url))
const stdioBin = join(repoRoot, 'packages/examples/stdio-demo/lib/bin.js') const stdioBin = join(repoRoot, 'packages/examples/stdio-demo/lib/bin.js')
const decompress = promisify(zstdDecompress)
// Symlink each required workspace package by package name so plain Node resolves its built `main`, // Symlink each required workspace package by package name so plain Node resolves its built `main`,
// matching an installed dependency rather than tsconfig paths. // matching an installed dependency rather than tsconfig paths.
@@ -153,6 +156,12 @@ describe.skipIf(!existsSync(stdioBin))('dsh-stdio-demo BUILT bin (node lib/bin.j
expect(stdout).toContain('[tool call] echo') expect(stdout).toContain('[tool call] echo')
expect(stdout).toContain('[tool result] ECHO: HI') expect(stdout).toContain('[tool result] ECHO: HI')
expect(code).toBe(0) expect(code).toBe(0)
const files = await readdir(join(consumer, '.sessions'), { recursive: true })
const log = files.find(file => file.endsWith('.jsonl.zstd'))
expect(log).toBeDefined()
const compressed = await readFile(join(consumer, '.sessions', log!))
expect(compressed.subarray(0, 4).toString('hex')).toBe('28b52ffd')
expect(JSON.parse((await decompress(compressed)).toString())).toMatchObject({ type: 'session' })
}, 30_000) }, 30_000)
it('boots cleanly when the config disables an (otherwise unresolvable) entry', async () => { it('boots cleanly when the config disables an (otherwise unresolvable) entry', async () => {

View File

@@ -86,12 +86,17 @@ describe('dsh-stdio-demo app', () => {
provider: 'mock', provider: 'mock',
model: 'mock', model: 'mock',
workspaceContext: false, workspaceContext: false,
persistenceCompression: 'none',
welcome: 'TUI ready', welcome: 'TUI ready',
ui: { mode: 'tui', tui: { color: false, maxToolOutputLines: 3 } }, ui: { mode: 'tui', tui: { color: false, maxToolOutputLines: 3 } },
}, true) }, true)
expect(calls.map(call => call.name)).toContain('ui-tui') expect(calls.map(call => call.name)).toContain('ui-tui')
expect(calls.map(call => call.name)).not.toContain('ui-stdio') expect(calls.map(call => call.name)).not.toContain('ui-stdio')
expect(calls.map(call => call.name)).not.toContain('ConsoleExporter') expect(calls.map(call => call.name)).not.toContain('ConsoleExporter')
expect(calls.find(call => (call.config as { root?: string } | undefined)?.root === './.sessions')?.config).toEqual({
root: './.sessions',
compression: 'none',
})
const tuiConfig = calls.find(call => call.name === 'ui-tui')?.config as { sessionId: string } const tuiConfig = calls.find(call => call.name === 'ui-tui')?.config as { sessionId: string }
expect(tuiConfig).toMatchObject({ welcome: 'TUI ready', color: false, maxToolOutputLines: 3 }) expect(tuiConfig).toMatchObject({ welcome: 'TUI ready', color: false, maxToolOutputLines: 3 })
expect(tuiConfig.sessionId).toMatch(/^main-session-/) expect(tuiConfig.sessionId).toMatch(/^main-session-/)

View File

@@ -1,31 +1,39 @@
# @deepseek-ai/dsh-session-persistence-jsonl # @deepseek-ai/dsh-session-persistence-jsonl
The JSONL durable session-persistence backend — a concrete `SessionPersistence` (the `dsh-session-persistence` seam). One append-only `.jsonl` event log per session. The JSONL durable session-persistence backend — a concrete `SessionPersistence` (the `dsh-session-persistence` seam). Each session has one append-only logical JSONL log, stored as `.jsonl.zstd` by default or raw `.jsonl` when compression is disabled.
## On-disk layout ## On-disk layout
``` ```
<root>/ <root>/
cwd-<sha256(cwd)[:12]>/ # per-project bucket (or _no-cwd/ when no cwd) cwd-<sha256(cwd)[:12]>/ # per-project bucket (or _no-cwd/ when no cwd)
<encoded-id>.jsonl # header line + one SessionEvent per line (verbatim) <encoded-id>.jsonl.zstd # default: checksummed header frame + append frames
<encoded-id>.jsonl # only with compression: 'none'
``` ```
- The first `.jsonl` line is the immutable `SessionHeader` tagged `{ type: 'session', version, id, cwd?, createdAt, parentSession?, seedLength? }`; every subsequent line is one `SessionEvent` JSON, **verbatim including `assistant/chunk`** so `seq` stays contiguous (`events[i].seq === i`). - The first logical line is the immutable `SessionHeader` tagged `{ type: 'session', version, id, cwd?, createdAt, parentSession?, seedLength? }`; every subsequent line is one `SessionEvent` JSON, **verbatim including `assistant/chunk`** so `seq` stays contiguous (`events[i].seq === i`).
- Session ids are unvalidated branded strings, so they are percent-encoded to a single safe path segment before use (no traversal, no collision). - Session ids are unvalidated branded strings, so they are injectively escaped to a single safe path segment before use (no traversal, no collision).
## Config ## Config
| Key | Type | Notes | | Key | Type | Notes |
|---|---|---| |---|---|---|
| `root` | `string` (required) | Root directory for all session files. **No default** — a `process.cwd()` default would scatter files as the process's cwd changes (bash calls, subprocesses). | | `root` | `string` (required) | Root directory for all session files. **No default** — a `process.cwd()` default would scatter files as the process's cwd changes (bash calls, subprocesses). |
| `compression` | `'zstd' \| 'none'` | Defaults to `'zstd'`; `'none'` retains newline-delimited UTF-8 text. |
`locate(meta)` returns `{ kind: 'jsonl', path }` using the resolved absolute root and the same cwd-bucket/id encoding as materialization. It performs no filesystem I/O: the target can be returned before the file exists, and an existing file contains only the last flushed prefix. `locate(meta)` returns `{ kind: 'jsonl', path }` using the resolved absolute root and the same cwd-bucket/id encoding as materialization. It performs no filesystem I/O: the target can be returned before the file exists, and an existing file contains only the last flushed prefix.
## Physical encoding
The default artifact is a standard concatenation of independent [Zstandard frames](../../../docs/rfc/implemented/architecture/2026-07-19-zstandard-jsonl-session-logs.md): one checksummed frame containing only the header line, followed by one checksummed frame per durable append batch. The backend uses Node's built-in Zstandard API with its default compression level and exposes no level knob. Listing reads and validates only the header frame. `compression: 'none'` keeps the same logical lines in the original raw representation.
A root belongs to one encoding. Startup discovery and targeted lookup reject the opposite suffix with an error naming the incompatible artifact and instructing the caller to select the matching mode or a separate root. There is no migration, mixed-root fallback, or dual write.
## Durability and crash semantics ## Durability and crash semantics
- **Lazy materialization.** `create(meta)` writes nothing; on the first `append`, the backend writes and `fsync`s a temporary file, publishes it without overwrite via a hard link, then `fsync`s the directory when the host supports it. A created-but-never-appended session leaves nothing on disk and is absent from `list`. - **Lazy materialization.** `create(meta)` writes nothing; on the first `append`, the backend writes and `fsync`s a temporary file, publishes it without overwrite via a hard link, then `fsync`s the directory when the host supports it. A created-but-never-appended session leaves nothing on disk and is absent from `list`.
- **Append-only.** Committed events (at or below a flushed `turn/end`) are never rewritten. Subsequent appends are line appends at EOF + `fsync`. - **Append-only.** Committed events (at or below a flushed `turn/end`) are never rewritten. Subsequent raw batches append lines; compressed batches append one frame. Both paths `fsync`, and a caught write or sync failure rolls the file back to its prior byte length.
- **Crash recovery — preserve valid tail work.** `load` keeps the contiguous valid prefix of an interrupted final turn. It truncates from the first unparsable or sequence-gapped uncommitted record, then appends the synthetic tool, step, and turn closers required by the shared [persistence contract](../../../docs/rfc/implemented/architecture/2026-06-14-session-persistence.md); the same defect at or before the last committed `turn/end` rejects. - **Crash recovery — preserve valid tail work.** `load` validates every complete compressed frame and scans their decompressed JSONL. If the last frame is structurally incomplete, the reader keeps its complete decoded records, truncates from that frame's start, and re-encodes those records with the synthetic tool, step, and turn closers required by the shared [persistence contract](../../../docs/rfc/implemented/architecture/2026-06-14-session-persistence.md). Raw mode truncates from its first incomplete line. A checksum/decompression failure in a complete frame, or a defect at or before the last committed `turn/end`, is corruption and rejects.
- **Contiguous-seq.** `append` rejects a batch whose first `seq` does not continue the stored log, and rejects non-JSON-serializable `event.data` naming the offending event type. - **Contiguous-seq.** `append` rejects a batch whose first `seq` does not continue the stored log, and rejects non-JSON-serializable `event.data` naming the offending event type.
## Write path ## Write path
@@ -50,7 +58,8 @@ JSONL storage does not mutate live request prefixes. A resumed loop can reuse pr
## Known Limitations and Deferred Work ## Known Limitations and Deferred Work
- **Only the current `SESSION_FORMAT_VERSION` (v0) loads** — the on-disk format is pre-release/unstable: a breaking format change is absorbed at v0 and non-current logs are rejected; there is no migration. - **Only the configured encoding and current `SESSION_FORMAT_VERSION` (v0) load** — changing compression requires a separate/fresh root or selecting the legacy raw mode; the pre-release format has no migration.
- **Compressed files are not directly line-readable** — use the backend to load them, or select `compression: 'none'` before writing a fresh root when text fixtures or external line readers are required.
- **Nothing deletes session files** — logs accumulate under `root` until removed externally (the seam has no deletion surface). - **Nothing deletes session files** — logs accumulate under `root` until removed externally (the seam has no deletion surface).
- **Single-process assumption** — per-session serialization and the write cursor live in this process; two processes appending to the same `root` are not coordinated. - **Single-process assumption** — per-session serialization and the write cursor live in this process; two processes appending to the same `root` are not coordinated.
- **Initial materialization requires hard-link support** — first append uses `link()` so same-id races fail instead of overwriting a committed log; a filesystem that cannot create hard links cannot host this backend. - **Initial materialization requires hard-link support** — first append uses `link()` so same-id races fail instead of overwriting a committed log; a filesystem that cannot create hard links cannot host this backend.

View File

@@ -12,8 +12,20 @@ import { createHash } from 'node:crypto'
import { join } from 'node:path' import { join } from 'node:path'
import type { SessionEvent, SessionHeader, SessionId } from '@deepseek-ai/dsh-session' import type { SessionEvent, SessionHeader, SessionId } from '@deepseek-ai/dsh-session'
/** Physical encoding selected for JSONL session artifacts. */
export type JsonlCompression = 'zstd' | 'none'
/** /**
* The first line of a session's `.jsonl` file: the immutable * Return the artifact suffix for one physical encoding.
* @param compression - configured JSONL artifact encoding.
* @returns `.jsonl.zstd` for Zstandard or `.jsonl` for plaintext.
*/
export function logSuffix(compression: JsonlCompression): '.jsonl.zstd' | '.jsonl' {
return compression === 'zstd' ? '.jsonl.zstd' : '.jsonl'
}
/**
* The first JSONL record of a session artifact: the immutable
* {@link SessionHeader} tagged as a `session` record so a reader can tell it * {@link SessionHeader} tagged as a `session` record so a reader can tell it
* apart from an event line. * apart from an event line.
*/ */
@@ -119,10 +131,16 @@ export function sessionDir(root: string, cwd: string | undefined): string {
* @param root - the backend's session root directory. * @param root - the backend's session root directory.
* @param cwd - the session's project directory (picks the per-cwd bucket; `undefined` → `_no-cwd`). * @param cwd - the session's project directory (picks the per-cwd bucket; `undefined` → `_no-cwd`).
* @param id - the session id, path-encoded via {@link encodeSegment} before filesystem use. * @param id - the session id, path-encoded via {@link encodeSegment} before filesystem use.
* @returns the session's `.jsonl` log file path. * @param compression - physical artifact encoding and filename suffix.
* @returns the session's configured JSONL artifact path.
*/ */
export function logPath(root: string, cwd: string | undefined, id: SessionId): string { export function logPath(
return join(sessionDir(root, cwd), `${encodeSegment(id)}.jsonl`) root: string,
cwd: string | undefined,
id: SessionId,
compression: JsonlCompression,
): string {
return join(sessionDir(root, cwd), `${encodeSegment(id)}${logSuffix(compression)}`)
} }
/** /**

View File

@@ -17,8 +17,20 @@ import {
} from '@deepseek-ai/dsh-session-persistence' } from '@deepseek-ai/dsh-session-persistence'
import type { SessionEvent, SessionId, SessionHeader } from '@deepseek-ai/dsh-session' import type { SessionEvent, SessionId, SessionHeader } from '@deepseek-ai/dsh-session'
import { import {
encodeSegment, eventLine, logPath, parseHeaderMeta, scanLog, sessionDir, toHeaderLine, encodeSegment, eventLine, logPath, logSuffix, parseHeaderMeta, scanLog, sessionDir, toHeaderLine,
type JsonlCompression,
} from './format.ts' } from './format.ts'
import { compressZstdFrame, decompressZstdFrame, scanZstdFrames } from './zstd.ts'
export type { JsonlCompression } from './format.ts'
const DEFAULT_COMPRESSION: JsonlCompression = 'zstd'
/** Loader schema for the JSONL artifact's physical encoding. */
export const JsonlCompressionSchema: z<JsonlCompression> = z.union([
z.const('zstd'),
z.const('none'),
]).default(DEFAULT_COMPRESSION)
/** Plugin config: where the JSONL backend keeps its session logs (`root` is required — no default). */ /** Plugin config: where the JSONL backend keeps its session logs (`root` is required — no default). */
export interface Config { export interface Config {
@@ -28,6 +40,14 @@ export interface Config {
* (bash calls, subprocesses). Sessions group under per-cwd subdirectories. * (bash calls, subprocesses). Sessions group under per-cwd subdirectories.
*/ */
root: string root: string
/** Physical encoding; defaults to checksummed Zstandard frames. */
compression?: JsonlCompression
}
/** Opaque coordinator token for replacing bytes recovered from a torn frame. */
interface JsonlTornMarker {
truncateTo: number
recoveredEvents: SessionEvent[]
} }
/** Whether a filesystem error means absence; every non-ENOENT failure must surface. */ /** Whether a filesystem error means absence; every non-ENOENT failure must surface. */
@@ -38,13 +58,15 @@ function isENOENT(error: unknown): boolean {
/** /**
* The JSONL persistence backend. Load as a plugin; it registers as * The JSONL persistence backend. Load as a plugin; it registers as
* `ctx.sessionPersistence` and (via the coordinator) installs the write-path * `ctx.sessionPersistence` and (via the coordinator) installs the write-path
* listeners. Its torn-tail marker is the byte offset to truncate the log to. * listeners. Its torn-tail marker carries the byte offset and any events
* recovered from an incomplete final Zstandard frame.
*/ */
export class SessionPersistenceJsonl extends SessionPersistence implements PersistenceBackend<number> { export class SessionPersistenceJsonl extends SessionPersistence implements PersistenceBackend<JsonlTornMarker> {
static inject = ['sessions'] static inject = ['sessions']
static Config: z<Config> = z.object({ static Config: z<Config> = z.object({
root: z.string().required(), root: z.string().required(),
compression: JsonlCompressionSchema,
}) })
/** /**
@@ -55,7 +77,9 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi
override readonly name = 'session-persistence-jsonl' override readonly name = 'session-persistence-jsonl'
private root: string private root: string
private coordinator: PersistenceCoordinator<number> private compression: JsonlCompression
private coordinator: PersistenceCoordinator<JsonlTornMarker>
private rootEncodingCheck: Promise<void> | undefined
/** Runtime host platform used to decide whether directory sync is supported. */ /** Runtime host platform used to decide whether directory sync is supported. */
readonly internals: { platform: NodeJS.Platform } = { platform: process.platform } readonly internals: { platform: NodeJS.Platform } = { platform: process.platform }
@@ -64,7 +88,8 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi
super(ctx) super(ctx)
// Resolve once so later process.cwd() changes cannot split one backend across roots. // Resolve once so later process.cwd() changes cannot split one backend across roots.
this.root = resolve(config.root) this.root = resolve(config.root)
this.coordinator = new PersistenceCoordinator<number>(this.ctx, this) this.compression = config.compression ?? DEFAULT_COMPRESSION
this.coordinator = new PersistenceCoordinator<JsonlTornMarker>(this.ctx, this)
} }
// Each backend keeps the typed service surface beside its storage hooks; // Each backend keeps the typed service surface beside its storage hooks;
@@ -74,7 +99,7 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi
/** Resolve the absolute target path without touching the filesystem. */ /** Resolve the absolute target path without touching the filesystem. */
locate(meta: SessionHeader): SessionLocation { locate(meta: SessionHeader): SessionLocation {
return { kind: 'jsonl', path: logPath(this.root, meta.cwd, meta.id) } return { kind: 'jsonl', path: logPath(this.root, meta.cwd, meta.id, this.compression) }
} }
create(meta: SessionHeader): Promise<void> { create(meta: SessionHeader): Promise<void> {
@@ -96,7 +121,8 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi
// --- PersistenceBackend hooks (the file-bytes storage primitives) --- // --- PersistenceBackend hooks (the file-bytes storage primitives) ---
/** Read a stored prefix by id across all cwd buckets when cwd is unknown. */ /** Read a stored prefix by id across all cwd buckets when cwd is unknown. */
async loadStored(id: SessionId): Promise<StoredPrefix<number> | undefined> { async loadStored(id: SessionId): Promise<StoredPrefix<JsonlTornMarker> | undefined> {
await this.ensureRootEncoding()
const file = await this.findLog(id) const file = await this.findLog(id)
if (file === undefined) return undefined if (file === undefined) return undefined
return this.readPrefix(file.path) return this.readPrefix(file.path)
@@ -106,28 +132,85 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi
* Read a stored prefix within one cwd for HMR adoption. `undefined` names the * Read a stored prefix within one cwd for HMR adoption. `undefined` names the
* no-cwd bucket rather than an unknown cwd, so this never scans other buckets. * no-cwd bucket rather than an unknown cwd, so this never scans other buckets.
*/ */
async loadLive(id: SessionId, cwd: string | undefined): Promise<StoredPrefix<number> | undefined> { async loadLive(id: SessionId, cwd: string | undefined): Promise<StoredPrefix<JsonlTornMarker> | undefined> {
const path = logPath(this.root, cwd, id) await this.ensureRootEncoding()
if (!await this.exists(path)) return undefined const path = logPath(this.root, cwd, id, this.compression)
if (!await this.exists(path)) {
await this.rejectOppositeArtifact(cwd, id)
return undefined
}
return this.readPrefix(path) return this.readPrefix(path)
} }
/** /**
* Read a stored prefix and convert torn-tail state to the byte offset the * Read a stored prefix and convert torn-tail state to the opaque marker the
* coordinator can round-trip without knowing the file format. * coordinator can round-trip without knowing the physical encoding.
*/ */
private async readPrefix(path: string): Promise<StoredPrefix<number>> { private async readPrefix(path: string): Promise<StoredPrefix<JsonlTornMarker>> {
const buffer = await readFile(path) const buffer = await readFile(path)
if (this.compression === 'zstd') return this.readZstdPrefix(buffer)
const { meta, events, committedBytes } = scanLog(buffer) const { meta, events, committedBytes } = scanLog(buffer)
return { return {
meta, meta,
events, events,
...committedBytes < buffer.byteLength ? { tornMarker: committedBytes } : {}, ...committedBytes < buffer.byteLength
? { tornMarker: { truncateTo: committedBytes, recoveredEvents: [] } }
: {},
}
}
/** Decode complete frames and retain complete JSONL records from a torn final frame. */
private async readZstdPrefix(buffer: Buffer): Promise<StoredPrefix<JsonlTornMarker>> {
const { frames, tornStart } = scanZstdFrames(buffer)
if (frames.length === 0) throw new Error('empty or header-less Zstandard session log')
const plaintextFrames: Buffer[] = []
for (const frame of frames) {
try {
plaintextFrames.push(await decompressZstdFrame(buffer.subarray(frame.start, frame.end)))
} catch (error) {
throw new Error(`corrupt Zstandard session log: frame at byte ${frame.start} failed validation`, { cause: error })
}
}
const headerFrame = plaintextFrames[0]
if (headerFrame === undefined || headerFrame.length === 0 || headerFrame.indexOf(0x0A) !== headerFrame.length - 1) {
throw new Error('corrupt Zstandard session log: first frame is not exactly one header line')
}
const completePlaintext = Buffer.concat(plaintextFrames)
const completePrefix = scanLog(completePlaintext)
if (completePrefix.committedBytes !== completePlaintext.length) {
throw new Error('corrupt Zstandard session log: complete frame contains a torn JSONL record')
}
if (tornStart === undefined) {
return { meta: completePrefix.meta, events: completePrefix.events }
}
let recoveredPlaintext: Buffer = Buffer.alloc(0)
try {
recoveredPlaintext = await decompressZstdFrame(buffer.subarray(tornStart))
} catch {
// A structurally incomplete final frame may end before Node's decoder can
// emit any plaintext; the complete prior frames remain recoverable.
}
const recoveredPrefix = scanLog(Buffer.concat([completePlaintext, recoveredPlaintext]))
/* v8 ignore next 3 -- appending plaintext cannot shorten the already-scanned complete prefix */
if (recoveredPrefix.events.length < completePrefix.events.length) {
throw new Error('corrupt Zstandard session log: recovered prefix does not extend complete frames')
}
return {
meta: recoveredPrefix.meta,
events: recoveredPrefix.events,
tornMarker: {
truncateTo: tornStart,
recoveredEvents: recoveredPrefix.events.slice(completePrefix.events.length),
},
} }
} }
/** Durably append a batch, lazily materializing the file when not yet present. */ /** Durably append a batch, lazily materializing the file when not yet present. */
async appendBatch(meta: SessionHeader, events: readonly SessionEvent[], isMaterialized: boolean): Promise<void> { async appendBatch(meta: SessionHeader, events: readonly SessionEvent[], isMaterialized: boolean): Promise<void> {
await this.ensureRootEncoding()
if (isMaterialized) { if (isMaterialized) {
await this.appendLines(meta, events) await this.appendLines(meta, events)
} else { } else {
@@ -136,22 +219,30 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi
} }
/** /**
* Make a crash repair durable: truncate the torn tail to `tornMarker` bytes (if * Make a crash repair durable: truncate a torn tail, restore complete events
* any), then append the synthetic `closers` (if any). Two fsync'd steps — the * decoded from it, then append synthetic closers. Two fsync'd steps — the seam
* seam does not require this to be atomic. * does not require this to be atomic.
*/ */
async commitRepair(meta: SessionHeader, tornMarker: number | undefined, closers: readonly SessionEvent[]): Promise<void> { async commitRepair(
if (tornMarker !== undefined) await this.repair(meta, tornMarker) meta: SessionHeader,
if (closers.length > 0) await this.appendLines(meta, closers) tornMarker: JsonlTornMarker | undefined,
closers: readonly SessionEvent[],
): Promise<void> {
if (tornMarker !== undefined) await this.repair(meta, tornMarker.truncateTo)
const repairedEvents = [...(tornMarker?.recoveredEvents ?? []), ...closers]
if (repairedEvents.length > 0) await this.appendLines(meta, repairedEvents)
} }
/** List all stored sessions' metadata (header line only — no full-log parse). */ /** List all stored sessions' metadata (header line only — no full-log parse). */
async list(): Promise<SessionHeader[]> { async list(): Promise<SessionHeader[]> {
await this.ensureRootEncoding()
const metas: SessionHeader[] = [] const metas: SessionHeader[] = []
for (const dir of await this.listCwdDirs()) { for (const dir of await this.listCwdDirs()) {
for (const name of await this.listJsonl(dir)) { for (const name of await this.listArtifacts(dir)) {
// Read only headers so listing scales with session count, not log size. // Read only headers so listing scales with session count, not log size.
const first = await this.readFirstLine(`${dir}/${name}`) const first = this.compression === 'zstd'
? await this.readFirstZstdLine(`${dir}/${name}`)
: await this.readFirstLine(`${dir}/${name}`)
if (first === undefined) continue // empty/half-written file if (first === undefined) continue // empty/half-written file
const meta = parseHeaderMeta(first) const meta = parseHeaderMeta(first)
if (meta === undefined) continue // not a session header if (meta === undefined) continue // not a session header
@@ -170,15 +261,14 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi
await this.syncDir(dirname(this.root)) await this.syncDir(dirname(this.root))
await mkdir(dir, { recursive: true, mode: 0o700 }) await mkdir(dir, { recursive: true, mode: 0o700 })
await this.syncDir(this.root) await this.syncDir(this.root)
const finalPath = logPath(this.root, meta.cwd, meta.id) const finalPath = logPath(this.root, meta.cwd, meta.id, this.compression)
// Materialization is the first write; an existing log is an id collision. // Materialization is the first write; an existing log is an id collision.
/* v8 ignore next 3 -- createCore guards collisions before materialize; this is a TOCTOU backstop */ /* v8 ignore next 3 -- createCore guards collisions before materialize; this is a TOCTOU backstop */
if (await this.exists(finalPath)) { if (await this.exists(finalPath)) {
throw new Error(`refusing to materialize "${meta.id}": a log already exists on disk (load/resume it instead)`) throw new Error(`refusing to materialize "${meta.id}": a log already exists on disk (load/resume it instead)`)
} }
const header = JSON.stringify(toHeaderLine(meta)) await this.rejectOppositeArtifact(meta.cwd, meta.id)
const body = events.map(eventLine).join('\n') const content = await this.encodeMaterialization(meta, events)
const content = header + '\n' + body + '\n'
const tmp = `${finalPath}.${randomBytes(6).toString('hex')}.tmp` const tmp = `${finalPath}.${randomBytes(6).toString('hex')}.tmp`
const handle = await open(tmp, 'wx', 0o600) const handle = await open(tmp, 'wx', 0o600)
@@ -211,6 +301,22 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi
} }
} }
/** Encode the header and first batch without combining their frame boundaries. */
private async encodeMaterialization(meta: SessionHeader, events: readonly SessionEvent[]): Promise<Buffer | string> {
const header = JSON.stringify(toHeaderLine(meta)) + '\n'
const body = events.map(eventLine).join('\n') + '\n'
if (this.compression === 'none') return header + body
const headerFrame = await compressZstdFrame(header)
const eventFrame = await compressZstdFrame(body)
return Buffer.concat([headerFrame, eventFrame])
}
/** Encode one durable append batch in the configured physical representation. */
private async encodeEventBatch(events: readonly SessionEvent[]): Promise<Buffer | string> {
const body = events.map(eventLine).join('\n') + '\n'
return this.compression === 'zstd' ? compressZstdFrame(body) : body
}
/** fsync a directory when the host exposes that durability primitive. */ /** fsync a directory when the host exposes that durability primitive. */
private async syncDir(dir: string): Promise<void> { private async syncDir(dir: string): Promise<void> {
const handle = await open(dir, 'r') const handle = await open(dir, 'r')
@@ -234,12 +340,13 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi
* batch; leaving partial bytes would create duplicate sequence numbers. * batch; leaving partial bytes would create duplicate sequence numbers.
*/ */
private async appendLines(meta: SessionHeader, events: readonly SessionEvent[]): Promise<void> { private async appendLines(meta: SessionHeader, events: readonly SessionEvent[]): Promise<void> {
const path = logPath(this.root, meta.cwd, meta.id) const content = await this.encodeEventBatch(events)
const path = logPath(this.root, meta.cwd, meta.id, this.compression)
const handle = await open(path, 'a') const handle = await open(path, 'a')
try { try {
const { size: before } = await handle.stat() const { size: before } = await handle.stat()
try { try {
await handle.writeFile(events.map(eventLine).join('\n') + '\n') await handle.writeFile(content)
await handle.sync() await handle.sync()
} catch (error) { } catch (error) {
// Roll back whatever bytes landed so a retry starts from a clean EOF. // Roll back whatever bytes landed so a retry starts from a clean EOF.
@@ -254,7 +361,7 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi
/** Truncate the log file to `offset` bytes and fsync (discard the crash tail). */ /** Truncate the log file to `offset` bytes and fsync (discard the crash tail). */
private async repair(meta: SessionHeader, offset: number): Promise<void> { private async repair(meta: SessionHeader, offset: number): Promise<void> {
const path = logPath(this.root, meta.cwd, meta.id) const path = logPath(this.root, meta.cwd, meta.id, this.compression)
await truncate(path, offset) await truncate(path, offset)
const handle = await open(path, 'r+') const handle = await open(path, 'r+')
try { try {
@@ -292,17 +399,47 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi
} }
} }
/** Read and validate only the independently compressed header frame. */
private async readFirstZstdLine(path: string): Promise<string | undefined> {
const handle = await open(path, 'r')
try {
let content = Buffer.alloc(0)
const chunk = Buffer.alloc(8192)
for (;;) {
const { bytesRead } = await handle.read(chunk, 0, chunk.length, null)
if (bytesRead === 0) return undefined
content = Buffer.concat([content, chunk.subarray(0, bytesRead)])
const first = scanZstdFrames(content, 1).frames[0]
if (first === undefined) continue
let plaintext: Buffer
try {
plaintext = await decompressZstdFrame(content.subarray(first.start, first.end))
} catch (error) {
throw new Error('corrupt Zstandard session log: header frame failed validation', { cause: error })
}
if (plaintext.length === 0 || plaintext.indexOf(0x0A) !== plaintext.length - 1) {
throw new Error('corrupt Zstandard session log: first frame is not exactly one header line')
}
return plaintext.subarray(0, -1).toString('utf8')
}
} finally {
await handle.close()
}
}
/** /**
* Find a session by id across cwd buckets for resume. Cwd-scoped HMR adoption * Find a session by id across cwd buckets for resume. Cwd-scoped HMR adoption
* bypasses this scan so a no-cwd session cannot claim another bucket. * bypasses this scan so a no-cwd session cannot claim another bucket.
*/ */
private async findLog(id: SessionId): Promise<{ path: string; cwd: string | undefined } | undefined> { private async findLog(id: SessionId): Promise<{ path: string; cwd: string | undefined } | undefined> {
const target = encodeSegment(id) + '.jsonl' const target = encodeSegment(id) + logSuffix(this.compression)
for (const dir of await this.listCwdDirs()) { for (const dir of await this.listCwdDirs()) {
const path = `${dir}/${target}` const path = `${dir}/${target}`
const opposite = `${dir}/${encodeSegment(id)}${logSuffix(this.oppositeCompression())}`
if (await this.exists(opposite)) throw this.encodingMismatch(opposite)
if (await this.exists(path)) { if (await this.exists(path)) {
// Recover the cwd from the header so the caller has the session's bucket. // Recover the cwd from the header so the caller has the session's bucket.
const { meta } = scanLog(await readFile(path)) const { meta } = await this.readPrefix(path)
return { path, cwd: meta.cwd } return { path, cwd: meta.cwd }
} }
} }
@@ -321,9 +458,45 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi
} }
} }
private async listJsonl(dir: string): Promise<string[]> { private async listArtifacts(dir: string): Promise<string[]> {
const entries = await readdir(dir) const entries = await readdir(dir)
return entries.filter(n => n.endsWith('.jsonl')) const oppositeSuffix = logSuffix(this.oppositeCompression())
const incompatible = entries.find(name => name.endsWith(oppositeSuffix))
if (incompatible !== undefined) throw this.encodingMismatch(`${dir}/${incompatible}`)
const suffix = logSuffix(this.compression)
return entries.filter(name => name.endsWith(suffix))
}
/** Reject a root that already belongs to the other physical encoding. */
private ensureRootEncoding(): Promise<void> {
this.rootEncodingCheck ??= this.checkRootEncoding()
return this.rootEncodingCheck
}
private async checkRootEncoding(): Promise<void> {
const oppositeSuffix = logSuffix(this.oppositeCompression())
for (const dir of await this.listCwdDirs()) {
const entries = await readdir(dir)
const incompatible = entries.find(name => name.endsWith(oppositeSuffix))
if (incompatible !== undefined) throw this.encodingMismatch(`${dir}/${incompatible}`)
}
}
private async rejectOppositeArtifact(cwd: string | undefined, id: SessionId): Promise<void> {
const path = logPath(this.root, cwd, id, this.oppositeCompression())
if (await this.exists(path)) throw this.encodingMismatch(path)
}
private oppositeCompression(): JsonlCompression {
return this.compression === 'zstd' ? 'none' : 'zstd'
}
private encodingMismatch(path: string): Error {
return new Error(
`session artifact ${JSON.stringify(path)} uses ${logSuffix(this.oppositeCompression())}, `
+ `but this backend is configured for compression ${JSON.stringify(this.compression)}; `
+ 'use a separate root or select the matching compression mode',
)
} }
private async exists(path: string): Promise<boolean> { private async exists(path: string): Promise<boolean> {

View File

@@ -0,0 +1,116 @@
/**
* Zstandard frame primitives for the JSONL persistence backend. The backend
* owns a concatenated-frame container so it can append and recover batches
* without exposing compression mechanics through the persistence seam.
* @module dsh-session-persistence-jsonl/zstd
*/
import { constants, zstdCompress, zstdDecompress, type ZstdOptions } from 'node:zlib'
import { promisify } from 'node:util'
const ZSTD_MAGIC = 0xFD2FB528
const zstdCompressAsync = promisify(zstdCompress)
const zstdDecompressAsync = promisify(zstdDecompress)
const CHECKSUM_OPTIONS: ZstdOptions = {
params: { [constants.ZSTD_c_checksumFlag]: 1 },
}
/** Byte range occupied by one structurally complete Zstandard frame. */
export interface ZstdFrameRange {
/** Inclusive frame start. */
start: number
/** Exclusive frame end. */
end: number
}
/** Structural scan result for a concatenated Zstandard stream. */
export interface ZstdFrameScan {
/** Complete frames in file order. */
frames: ZstdFrameRange[]
/** Start of an incomplete final frame, when EOF interrupts one. */
tornStart?: number
}
/**
* Locate complete frames without decompressing their blocks. Invalid complete
* structure rejects; EOF inside the final frame returns its start for repair.
* @param buffer - complete bytes currently present in the session artifact.
* @param maxFrames - optional complete-frame limit for metadata-only readers.
* @returns complete frame ranges and an optional incomplete-final-frame start.
*/
export function scanZstdFrames(buffer: Buffer, maxFrames = Number.POSITIVE_INFINITY): ZstdFrameScan {
const frames: ZstdFrameRange[] = []
let offset = 0
while (offset < buffer.length) {
const start = offset
if (buffer.length - offset < 4) return { frames, tornStart: start }
if (buffer.readUInt32LE(offset) !== ZSTD_MAGIC) {
throw new Error(`corrupt Zstandard session log: invalid frame magic at byte ${offset}`)
}
offset += 4
if (offset === buffer.length) return { frames, tornStart: start }
const descriptor = buffer.readUInt8(offset)
offset += 1
if ((descriptor & 0x18) !== 0) {
throw new Error(`corrupt Zstandard session log: reserved frame-header bit at byte ${offset - 1}`)
}
const contentSizeFlag = descriptor >>> 6
const singleSegment = (descriptor & 0x20) !== 0
const checksum = (descriptor & 0x04) !== 0
const dictionaryFlag = descriptor & 0x03
const dictionaryBytes = dictionaryFlag === 3 ? 4 : dictionaryFlag
const contentSizeBytes = contentSizeFlag === 0
? (singleSegment ? 1 : 0)
: 1 << contentSizeFlag
const remainingHeaderBytes = (singleSegment ? 0 : 1) + dictionaryBytes + contentSizeBytes
if (buffer.length - offset < remainingHeaderBytes) return { frames, tornStart: start }
offset += remainingHeaderBytes
for (;;) {
if (buffer.length - offset < 3) return { frames, tornStart: start }
const blockHeader = buffer.readUIntLE(offset, 3)
offset += 3
const lastBlock = (blockHeader & 1) !== 0
const blockType = (blockHeader >>> 1) & 0x03
const blockSize = blockHeader >>> 3
if (blockType === 0x03) {
throw new Error(`corrupt Zstandard session log: reserved block type at byte ${offset - 3}`)
}
const payloadBytes = blockType === 0x01 ? 1 : blockSize
if (buffer.length - offset < payloadBytes) return { frames, tornStart: start }
offset += payloadBytes
if (lastBlock) break
}
if (checksum) {
if (buffer.length - offset < 4) return { frames, tornStart: start }
offset += 4
}
frames.push({ start, end: offset })
if (frames.length === maxFrames) return { frames }
}
return { frames }
}
/**
* Compress one independently decodable, checksummed Zstandard frame.
* @param input - JSONL bytes for a header or durable event batch.
* @returns the complete encoded frame.
*/
export async function compressZstdFrame(input: Buffer | string): Promise<Buffer> {
return zstdCompressAsync(input, CHECKSUM_OPTIONS)
}
/**
* Decompress one complete frame or the available prefix of a torn final frame.
* Complete-frame checksums are validated by Node's decoder.
* @param input - bytes beginning at a Zstandard frame boundary.
* @returns plaintext produced from the available input.
*/
export async function decompressZstdFrame(input: Buffer): Promise<Buffer> {
return zstdDecompressAsync(input)
}

View File

@@ -40,6 +40,10 @@ async function freshRoot(): Promise<string> {
return dir return dir
} }
function rawLogPath(root: string, cwd: string | undefined, id: SessionId): string {
return logPath(root, cwd, id, 'none')
}
afterEach(async () => { afterEach(async () => {
vi.restoreAllMocks() vi.restoreAllMocks()
for (const d of dirs.splice(0)) await rm(d, { recursive: true, force: true }) for (const d of dirs.splice(0)) await rm(d, { recursive: true, force: true })
@@ -70,11 +74,11 @@ function appendClosedTurn(session: Session): void {
} }
// Run the shared backend contract against the real JSONL backend. // Run the shared backend contract against the real JSONL backend.
runPersistenceContract('jsonl', async () => { runPersistenceContract('jsonl-none', async () => {
const dir = await mkdtemp(join(tmpdir(), 'dsh-jsonl-')) const dir = await mkdtemp(join(tmpdir(), 'dsh-jsonl-'))
const ctx = new Context() const ctx = new Context()
await ctx.plugin(SessionStore) await ctx.plugin(SessionStore)
const fiber = await ctx.plugin(SessionPersistenceJsonl, { root: dir }) const fiber = await ctx.plugin(SessionPersistenceJsonl, { root: dir, compression: 'none' })
return { return {
persistence: ctx.sessionPersistence, persistence: ctx.sessionPersistence,
dispose: async () => { dispose: async () => {
@@ -86,18 +90,18 @@ runPersistenceContract('jsonl', async () => {
// Two mounts share this temp root to exercise reload. `corruptTail` appends a partial, // Two mounts share this temp root to exercise reload. `corruptTail` appends a partial,
// newline-less fragment past the committed region so coordinator repair runs on real file bytes. // newline-less fragment past the committed region so coordinator repair runs on real file bytes.
runCoordinatorContract('jsonl', async (): Promise<CoordinatorFixture> => { runCoordinatorContract('jsonl-none', async (): Promise<CoordinatorFixture> => {
const dir = await mkdtemp(join(tmpdir(), 'dsh-jsonl-coord-')) const dir = await mkdtemp(join(tmpdir(), 'dsh-jsonl-coord-'))
return { return {
mount: async (ctx) => { mount: async (ctx) => {
const fiber = await ctx.plugin(SessionPersistenceJsonl, { root: dir }) const fiber = await ctx.plugin(SessionPersistenceJsonl, { root: dir, compression: 'none' })
return fiber return fiber
}, },
corruptTail: async (id, cwd) => { corruptTail: async (id, cwd) => {
// A half-written record with no trailing newline: scanLog treats it as an // A half-written record with no trailing newline: scanLog treats it as an
// uncommitted crash fragment and reports committedBytes < byteLength, so // uncommitted crash fragment and reports committedBytes < byteLength, so
// the coordinator sees a tornMarker to truncate. // the coordinator sees a tornMarker to truncate.
await appendFile(logPath(dir, cwd, id), '{"type":"assistant/chunk","seq":8,"ti') await appendFile(rawLogPath(dir, cwd, id), '{"type":"assistant/chunk","seq":8,"ti')
}, },
cleanup: async () => { await rm(dir, { recursive: true, force: true }) }, cleanup: async () => { await rm(dir, { recursive: true, force: true }) },
} }
@@ -134,11 +138,14 @@ describe('SessionPersistenceJsonl: format helpers', () => {
const absoluteRoot = await freshRoot() const absoluteRoot = await freshRoot()
const ctx = new Context() const ctx = new Context()
await ctx.plugin(SessionStore) await ctx.plugin(SessionStore)
const fiber = await ctx.plugin(SessionPersistenceJsonl, { root: relative(process.cwd(), absoluteRoot) }) const fiber = await ctx.plugin(SessionPersistenceJsonl, {
root: relative(process.cwd(), absoluteRoot),
compression: 'none',
})
const m = meta('relative-location', '/work') const m = meta('relative-location', '/work')
expect(ctx.sessionPersistence.locate(m)).toEqual({ expect(ctx.sessionPersistence.locate(m)).toEqual({
kind: 'jsonl', kind: 'jsonl',
path: logPath(resolve(absoluteRoot), '/work', m.id), path: rawLogPath(resolve(absoluteRoot), '/work', m.id),
}) })
await fiber.dispose() await fiber.dispose()
}) })
@@ -150,26 +157,26 @@ describe('SessionPersistenceJsonl: durability and crash semantics', () => {
root = await freshRoot() root = await freshRoot()
ctx = new Context() ctx = new Context()
await ctx.plugin(SessionStore) await ctx.plugin(SessionStore)
await ctx.plugin(SessionPersistenceJsonl, { root }) await ctx.plugin(SessionPersistenceJsonl, { root, compression: 'none' })
}) })
afterEach(async () => { await ctx.fiber.dispose() }) afterEach(async () => { await ctx.fiber.dispose() })
it('lazy materialization: create() writes no file until the first append', async () => { it('lazy materialization: create() writes no file until the first append', async () => {
const m = meta('lazy', '/work') const m = meta('lazy', '/work')
const location = ctx.sessionPersistence.locate(m) const location = ctx.sessionPersistence.locate(m)
expect(location).toEqual({ kind: 'jsonl', path: logPath(root, '/work', m.id) }) expect(location).toEqual({ kind: 'jsonl', path: rawLogPath(root, '/work', m.id) })
expect(isAbsolute(location!.path)).toBe(true) expect(isAbsolute(location!.path)).toBe(true)
await ctx.sessionPersistence.create(m) await ctx.sessionPersistence.create(m)
// locate() is a pure target-path calculation: neither it nor create() // locate() is a pure target-path calculation: neither it nor create()
// materializes a file before the first append. // materializes a file before the first append.
const dir = sessionDir(root, '/work') const dir = sessionDir(root, '/work')
await expect(stat(logPath(root, '/work', m.id))).rejects.toThrow() await expect(stat(rawLogPath(root, '/work', m.id))).rejects.toThrow()
expect((await ctx.sessionPersistence.list()).map(h => h.id)).not.toContain(m.id) expect((await ctx.sessionPersistence.list()).map(h => h.id)).not.toContain(m.id)
await ctx.sessionPersistence.append(m.id, oneTurnLog()) await ctx.sessionPersistence.append(m.id, oneTurnLog())
// now materialized // now materialized
expect((await stat(logPath(root, '/work', m.id))).isFile()).toBe(true) expect((await stat(rawLogPath(root, '/work', m.id))).isFile()).toBe(true)
expect((await ctx.sessionPersistence.list()).map(h => h.id)).toContain(m.id) expect((await ctx.sessionPersistence.list()).map(h => h.id)).toContain(m.id)
void dir void dir
}) })
@@ -191,7 +198,7 @@ describe('SessionPersistenceJsonl: durability and crash semantics', () => {
} }
const childLocation = ctx.sessionPersistence.locate(child) const childLocation = ctx.sessionPersistence.locate(child)
expect(childLocation?.path).not.toBe(parentLocation?.path) expect(childLocation?.path).not.toBe(parentLocation?.path)
expect(childLocation).toEqual({ kind: 'jsonl', path: logPath(root, '/work', child.id) }) expect(childLocation).toEqual({ kind: 'jsonl', path: rawLogPath(root, '/work', child.id) })
}) })
it('round-trip is byte-identical (incl. assistant/chunk verbatim)', async () => { it('round-trip is byte-identical (incl. assistant/chunk verbatim)', async () => {
@@ -213,7 +220,7 @@ describe('SessionPersistenceJsonl: durability and crash semantics', () => {
it('rejects a stored v0 log containing a legacy request/header-delta event', async () => { it('rejects a stored v0 log containing a legacy request/header-delta event', async () => {
const m = meta('legacy-header-delta', '/legacy') const m = meta('legacy-header-delta', '/legacy')
const path = logPath(root, m.cwd, m.id) const path = rawLogPath(root, m.cwd, m.id)
await mkdir(sessionDir(root, m.cwd), { recursive: true }) await mkdir(sessionDir(root, m.cwd), { recursive: true })
await writeFile(path, [ await writeFile(path, [
JSON.stringify(toHeaderLine(m)), JSON.stringify(toHeaderLine(m)),
@@ -228,7 +235,7 @@ describe('SessionPersistenceJsonl: durability and crash semantics', () => {
it('rejects a stored v0 full header carrying the legacy fallback reason', async () => { it('rejects a stored v0 full header carrying the legacy fallback reason', async () => {
const m = meta('legacy-header-fallback', '/legacy') const m = meta('legacy-header-fallback', '/legacy')
const path = logPath(root, m.cwd, m.id) const path = rawLogPath(root, m.cwd, m.id)
await mkdir(sessionDir(root, m.cwd), { recursive: true }) await mkdir(sessionDir(root, m.cwd), { recursive: true })
await writeFile(path, [ await writeFile(path, [
JSON.stringify(toHeaderLine(m)), JSON.stringify(toHeaderLine(m)),
@@ -270,7 +277,7 @@ describe('SessionPersistenceJsonl: durability and crash semantics', () => {
// Simulate a crash mid-second-turn: append raw lines that are NOT closed by // Simulate a crash mid-second-turn: append raw lines that are NOT closed by
// a turn/end (turn/start + step/start are fully written), plus a final // a turn/end (turn/start + step/start are fully written), plus a final
// partial line with no newline (a torn fragment never fully flushed). // partial line with no newline (a torn fragment never fully flushed).
const path = logPath(root, '/proj', m.id) const path = rawLogPath(root, '/proj', m.id)
await writeFile(path, [ await writeFile(path, [
JSON.stringify({ type: 'turn/start', seq: 6, time: 8, data: { turn: 2, trigger: { kind: 'message', source: { kind: 'user' } } } }), JSON.stringify({ type: 'turn/start', seq: 6, time: 8, data: { turn: 2, trigger: { kind: 'message', source: { kind: 'user' } } } }),
JSON.stringify({ type: 'step/start', seq: 7, time: 9, data: { turn: 2, step: 1 } }), JSON.stringify({ type: 'step/start', seq: 7, time: 9, data: { turn: 2, step: 1 } }),
@@ -303,17 +310,17 @@ describe('SessionPersistenceJsonl: durability and crash semantics', () => {
const m = meta('append-only') const m = meta('append-only')
await ctx.sessionPersistence.create(m) await ctx.sessionPersistence.create(m)
await ctx.sessionPersistence.append(m.id, oneTurnLog()) await ctx.sessionPersistence.append(m.id, oneTurnLog())
const before = await readFile(logPath(root, undefined, m.id), 'utf8') const before = await readFile(rawLogPath(root, undefined, m.id), 'utf8')
const committedPrefix = before // the whole committed log const committedPrefix = before // the whole committed log
// A crash tail then a repair-append. // A crash tail then a repair-append.
await writeFile(logPath(root, undefined, m.id), '\n{"partial', { flag: 'a' }) await writeFile(rawLogPath(root, undefined, m.id), '\n{"partial', { flag: 'a' })
await ctx.sessionPersistence.load(m.id) await ctx.sessionPersistence.load(m.id)
await ctx.sessionPersistence.append(m.id, [ await ctx.sessionPersistence.append(m.id, [
{ type: 'turn/start', seq: 6, time: 9, data: { turn: 2, trigger: { kind: 'message', source: { kind: 'user' } } } }, { type: 'turn/start', seq: 6, time: 9, data: { turn: 2, trigger: { kind: 'message', source: { kind: 'user' } } } },
{ type: 'turn/end', seq: 7, time: 10, data: { turn: 2, reason: { kind: 'completed' } } }, { type: 'turn/end', seq: 7, time: 10, data: { turn: 2, reason: { kind: 'completed' } } },
] as SessionEvent[]) ] as SessionEvent[])
const after = await readFile(logPath(root, undefined, m.id), 'utf8') const after = await readFile(rawLogPath(root, undefined, m.id), 'utf8')
// the committed prefix is byte-for-byte intact at the head of the file // the committed prefix is byte-for-byte intact at the head of the file
expect(after.startsWith(committedPrefix)).toBe(true) expect(after.startsWith(committedPrefix)).toBe(true)
}) })
@@ -322,12 +329,12 @@ describe('SessionPersistenceJsonl: durability and crash semantics', () => {
const m = meta('truncate-retry') const m = meta('truncate-retry')
await ctx.sessionPersistence.create(m) await ctx.sessionPersistence.create(m)
await ctx.sessionPersistence.append(m.id, oneTurnLog()) // materialized, seqs 0..5 await ctx.sessionPersistence.append(m.id, oneTurnLog()) // materialized, seqs 0..5
const sizeBefore = (await stat(logPath(root, undefined, m.id))).size const sizeBefore = (await stat(rawLogPath(root, undefined, m.id))).size
// Force the NEXT fsync (inside appendLines) to fail once, AFTER writeFile // Force the NEXT fsync (inside appendLines) to fail once, AFTER writeFile
// has already put bytes on disk — simulating an ENOSPC/fsync error // has already put bytes on disk — simulating an ENOSPC/fsync error
// mid-append. The recovery truncate() also fsyncs, so allow that one. // mid-append. The recovery truncate() also fsyncs, so allow that one.
const handle = await (await import('node:fs/promises')).open(logPath(root, undefined, m.id), 'r') const handle = await (await import('node:fs/promises')).open(rawLogPath(root, undefined, m.id), 'r')
const proto = Object.getPrototypeOf(handle) as { sync: () => Promise<void> } const proto = Object.getPrototypeOf(handle) as { sync: () => Promise<void> }
await handle.close() await handle.close()
const realSync = proto.sync const realSync = proto.sync
@@ -344,7 +351,7 @@ describe('SessionPersistenceJsonl: durability and crash semantics', () => {
// The append rejects, but the partial bytes are truncated back: the file is // The append rejects, but the partial bytes are truncated back: the file is
// its pre-append size and the cursor is unchanged. // its pre-append size and the cursor is unchanged.
await expect(ctx.sessionPersistence.append(m.id, turn2)).rejects.toThrow(/ENOSPC/) await expect(ctx.sessionPersistence.append(m.id, turn2)).rejects.toThrow(/ENOSPC/)
expect((await stat(logPath(root, undefined, m.id))).size).toBe(sizeBefore) expect((await stat(rawLogPath(root, undefined, m.id))).size).toBe(sizeBefore)
spy.mockRestore() spy.mockRestore()
// The retry now succeeds with NO seq gap — the log is contiguous 0..7. // The retry now succeeds with NO seq gap — the log is contiguous 0..7.
@@ -425,7 +432,7 @@ describe('SessionPersistenceJsonl: write path (session/event → flush)', () =>
root = await freshRoot() root = await freshRoot()
const ctx = new Context() const ctx = new Context()
await ctx.plugin(SessionStore) await ctx.plugin(SessionStore)
await ctx.plugin(SessionPersistenceJsonl, { root }) await ctx.plugin(SessionPersistenceJsonl, { root, compression: 'none' })
const a = ctx.sessions.create(SessionId('sa')) const a = ctx.sessions.create(SessionId('sa'))
const b = ctx.sessions.create(SessionId('sb')) const b = ctx.sessions.create(SessionId('sb'))
@@ -531,7 +538,7 @@ describe('SessionPersistenceJsonl: edge cases', () => {
root = await freshRoot() root = await freshRoot()
ctx = new Context() ctx = new Context()
await ctx.plugin(SessionStore) await ctx.plugin(SessionStore)
await ctx.plugin(SessionPersistenceJsonl, { root }) await ctx.plugin(SessionPersistenceJsonl, { root, compression: 'none' })
}) })
afterEach(async () => { await ctx.fiber.dispose() }) afterEach(async () => { await ctx.fiber.dispose() })
@@ -551,8 +558,8 @@ describe('SessionPersistenceJsonl: edge cases', () => {
await p await p
await ctx.sessionPersistence.append(SessionId('create-snap'), oneTurnLog()) await ctx.sessionPersistence.append(SessionId('create-snap'), oneTurnLog())
// The log materialized under the ORIGINAL cwd, not the mutated one. // The log materialized under the ORIGINAL cwd, not the mutated one.
expect((await stat(logPath(root, '/orig', SessionId('create-snap')))).isFile()).toBe(true) expect((await stat(rawLogPath(root, '/orig', SessionId('create-snap')))).isFile()).toBe(true)
await expect(stat(logPath(root, '/mutated', SessionId('create-snap')))).rejects.toThrow() await expect(stat(rawLogPath(root, '/mutated', SessionId('create-snap')))).rejects.toThrow()
}) })
it('list discovers sessions across multiple cwd buckets', async () => { it('list discovers sessions across multiple cwd buckets', async () => {
@@ -633,7 +640,7 @@ describe('SessionPersistenceJsonl: edge cases', () => {
// of grafting no-cwd events onto a log with mismatched cwd. // of grafting no-cwd events onto a log with mismatched cwd.
const ctx2 = new Context() const ctx2 = new Context()
await ctx2.plugin(SessionStore) await ctx2.plugin(SessionStore)
await ctx2.plugin(SessionPersistenceJsonl, { root }) await ctx2.plugin(SessionPersistenceJsonl, { root, compression: 'none' })
let b!: Session let b!: Session
await ctx2.plugin(Object.assign((inner: Context) => { await ctx2.plugin(Object.assign((inner: Context) => {
b = inner.sessions.create(SessionId('x')) // no cwd b = inner.sessions.create(SessionId('x')) // no cwd
@@ -642,10 +649,10 @@ describe('SessionPersistenceJsonl: edge cases', () => {
// The "/w" log is untouched — no no-cwd events were grafted onto it, and no // The "/w" log is untouched — no no-cwd events were grafted onto it, and no
// `_no-cwd` log for "x" was created. // `_no-cwd` log for "x" was created.
const inW = scanLog(await readFile(logPath(root, '/w', SessionId('x')))) const inW = scanLog(await readFile(rawLogPath(root, '/w', SessionId('x'))))
expect(inW.meta.cwd).toBe('/w') expect(inW.meta.cwd).toBe('/w')
expect(inW.events).toHaveLength(6) expect(inW.events).toHaveLength(6)
await expect(stat(logPath(root, undefined, SessionId('x')))).rejects.toThrow() await expect(stat(rawLogPath(root, undefined, SessionId('x')))).rejects.toThrow()
await ctx2.fiber.dispose() await ctx2.fiber.dispose()
}) })
@@ -689,7 +696,10 @@ describe('SessionPersistenceJsonl: edge cases', () => {
it('list returns nothing when the root directory does not exist', async () => { it('list returns nothing when the root directory does not exist', async () => {
const ctx2 = new Context() const ctx2 = new Context()
await ctx2.plugin(SessionStore) await ctx2.plugin(SessionStore)
await ctx2.plugin(SessionPersistenceJsonl, { root: join(root, 'does-not-exist-yet') }) await ctx2.plugin(SessionPersistenceJsonl, {
root: join(root, 'does-not-exist-yet'),
compression: 'none',
})
expect(await ctx2.sessionPersistence.list()).toEqual([]) expect(await ctx2.sessionPersistence.list()).toEqual([])
await ctx2.fiber.dispose() await ctx2.fiber.dispose()
}) })
@@ -701,7 +711,7 @@ describe('SessionPersistenceJsonl: edge cases', () => {
await writeFile(filePath, 'x') await writeFile(filePath, 'x')
const ctx2 = new Context() const ctx2 = new Context()
await ctx2.plugin(SessionStore) await ctx2.plugin(SessionStore)
await ctx2.plugin(SessionPersistenceJsonl, { root: filePath }) await ctx2.plugin(SessionPersistenceJsonl, { root: filePath, compression: 'none' })
await expect(ctx2.sessionPersistence.list()).rejects.toThrow(/ENOTDIR/) await expect(ctx2.sessionPersistence.list()).rejects.toThrow(/ENOTDIR/)
await ctx2.fiber.dispose() await ctx2.fiber.dispose()
}) })
@@ -712,7 +722,7 @@ describe('SessionPersistenceJsonl: edge cases', () => {
const cwd = '/x' const cwd = '/x'
const ctx2 = new Context() const ctx2 = new Context()
await ctx2.plugin(SessionStore) await ctx2.plugin(SessionStore)
await ctx2.plugin(SessionPersistenceJsonl, { root }) await ctx2.plugin(SessionPersistenceJsonl, { root, compression: 'none' })
await writeFile(sessionDir(root, cwd), 'x') // bucket path is now a FILE await writeFile(sessionDir(root, cwd), 'x') // bucket path is now a FILE
let s!: Session let s!: Session
await ctx2.plugin(Object.assign((inner: Context) => { await ctx2.plugin(Object.assign((inner: Context) => {
@@ -727,14 +737,14 @@ describe('SessionPersistenceJsonl: edge cases', () => {
const m = meta('disk-append', '/d') const m = meta('disk-append', '/d')
await ctx.sessionPersistence.create(m) await ctx.sessionPersistence.create(m)
await ctx.sessionPersistence.append(m.id, oneTurnLog()) await ctx.sessionPersistence.append(m.id, oneTurnLog())
await writeFile(logPath(root, '/d', m.id), '\n{"partial crash', { flag: 'a' }) await writeFile(rawLogPath(root, '/d', m.id), '\n{"partial crash', { flag: 'a' })
// A FRESH backend with no in-memory state: append directly (no prior load) // A FRESH backend with no in-memory state: append directly (no prior load)
// → append must adopt from disk, and the adopt's load schedules a repair // → append must adopt from disk, and the adopt's load schedules a repair
// that the same append then performs before writing. // that the same append then performs before writing.
const ctx2 = new Context() const ctx2 = new Context()
await ctx2.plugin(SessionStore) await ctx2.plugin(SessionStore)
await ctx2.plugin(SessionPersistenceJsonl, { root }) await ctx2.plugin(SessionPersistenceJsonl, { root, compression: 'none' })
await ctx2.sessionPersistence.append(m.id, [ await ctx2.sessionPersistence.append(m.id, [
{ type: 'turn/start', seq: 6, time: 9, data: { turn: 2, trigger: { kind: 'message', source: { kind: 'user' } } } }, { type: 'turn/start', seq: 6, time: 9, data: { turn: 2, trigger: { kind: 'message', source: { kind: 'user' } } } },
{ type: 'turn/end', seq: 7, time: 10, data: { turn: 2, reason: { kind: 'completed' } } }, { type: 'turn/end', seq: 7, time: 10, data: { turn: 2, reason: { kind: 'completed' } } },
@@ -770,7 +780,7 @@ describe('SessionPersistenceJsonl: edge cases', () => {
// nondeterministic. create scans every bucket, not just meta.cwd's. // nondeterministic. create scans every bucket, not just meta.cwd's.
const ctx2 = new Context() const ctx2 = new Context()
await ctx2.plugin(SessionStore) await ctx2.plugin(SessionStore)
await ctx2.plugin(SessionPersistenceJsonl, { root }) await ctx2.plugin(SessionPersistenceJsonl, { root, compression: 'none' })
await expect(ctx2.sessionPersistence.create(meta('dup-id', '/projB'))) await expect(ctx2.sessionPersistence.create(meta('dup-id', '/projB')))
.rejects.toThrow(/already has a persisted log on disk/) .rejects.toThrow(/already has a persisted log on disk/)
await ctx2.fiber.dispose() await ctx2.fiber.dispose()
@@ -780,7 +790,7 @@ describe('SessionPersistenceJsonl: edge cases', () => {
root = await freshRoot() root = await freshRoot()
const ctx2 = new Context() const ctx2 = new Context()
await ctx2.plugin(SessionStore) await ctx2.plugin(SessionStore)
await ctx2.plugin(SessionPersistenceJsonl, { root }) await ctx2.plugin(SessionPersistenceJsonl, { root, compression: 'none' })
const session = ctx2.sessions.create(SessionId('flush-fail')) const session = ctx2.sessions.create(SessionId('flush-fail'))
// A full turn lands in the write-behind buffer. // A full turn lands in the write-behind buffer.
session.append('user/message', { content: [{ type: 'text', text: 'hi' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) session.append('user/message', { content: [{ type: 'text', text: 'hi' }], source: { kind: 'user' } }, { surfaceOp: 'append' })

View File

@@ -0,0 +1,24 @@
import { describe, expect, it } from 'vitest'
import { compressZstdFrame, decompressZstdFrame, scanZstdFrames } from '../src/zstd.ts'
describe('JSONL Zstandard compatibility', () => {
it('round-trips concatenated checksummed frames through the built-in Node API', async () => {
const encoded = Buffer.concat([
await compressZstdFrame('{"type":"session","version":0,"id":"compat","createdAt":1}\n'),
await compressZstdFrame('{"type":"turn/start","seq":0,"turn":1}\n'),
])
const { frames, tornStart } = scanZstdFrames(encoded)
expect(tornStart).toBeUndefined()
expect(frames).toHaveLength(2)
expect(frames.map(frame => encoded.subarray(frame.start, frame.start + 4).toString('hex')))
.toEqual(['28b52ffd', '28b52ffd'])
const decoded = await Promise.all(frames.map(frame => decompressZstdFrame(encoded.subarray(frame.start, frame.end))))
expect(Buffer.concat(decoded).toString()).toContain('"type":"turn/start"')
const eventFrame = encoded.subarray(frames[1]!.start, frames[1]!.end)
const missingChecksumByte = eventFrame.subarray(0, -1)
expect(scanZstdFrames(missingChecksumByte)).toEqual({ frames: [], tornStart: 0 })
expect((await decompressZstdFrame(missingChecksumByte)).toString()).toContain('"type":"turn/start"')
})
})

View File

@@ -0,0 +1,483 @@
import { afterEach, describe, expect, it, vi } from 'vitest'
import { Context } from 'cordis'
import { appendFile, mkdir, mkdtemp, open, readFile, readdir, rm, stat, writeFile } from 'node:fs/promises'
import type { FileHandle } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import SessionStore, { SessionId } from '@deepseek-ai/dsh-session'
import type { SessionEvent } from '@deepseek-ai/dsh-session'
import SessionPersistenceJsonl from '@deepseek-ai/dsh-session-persistence-jsonl'
import { eventLine, logPath, scanLog, sessionDir, toHeaderLine, type JsonlCompression } from '../src/format.ts'
import { compressZstdFrame, decompressZstdFrame, scanZstdFrames } from '../src/zstd.ts'
import { runPersistenceContract, meta, oneTurnLog } from '../../session-persistence/tests/contract.ts'
import { runCoordinatorContract, type CoordinatorFixture } from '../../session-persistence/tests/coordinator-contract.ts'
const MAGIC = Buffer.from([0x28, 0xB5, 0x2F, 0xFD])
const roots: string[] = []
const contexts: Context[] = []
async function freshRoot(prefix = 'dsh-jsonl-zstd-'): Promise<string> {
const root = await mkdtemp(join(tmpdir(), prefix))
roots.push(root)
return root
}
async function mount(root: string, compression?: JsonlCompression): Promise<Context> {
const ctx = new Context()
contexts.push(ctx)
await ctx.plugin(SessionStore)
await ctx.plugin(SessionPersistenceJsonl, {
root,
...(compression === undefined ? {} : { compression }),
})
return ctx
}
async function decodeCompleteFrames(buffer: Buffer): Promise<Buffer> {
const { frames, tornStart } = scanZstdFrames(buffer)
expect(tornStart).toBeUndefined()
const plaintext: Buffer[] = []
for (const frame of frames) {
plaintext.push(await decompressZstdFrame(buffer.subarray(frame.start, frame.end)))
}
return Buffer.concat(plaintext)
}
async function tornFrame(
plaintext: string,
accepts: (decoded: string) => boolean,
): Promise<Buffer> {
const frame = await compressZstdFrame(plaintext)
const candidateEnds = [
frame.length - 1,
frame.length - 4,
...[0.9, 0.75, 0.6, 0.5, 0.4, 0.25].map(ratio => Math.floor(frame.length * ratio)),
]
for (const end of candidateEnds) {
const candidate = frame.subarray(0, end)
if (scanZstdFrames(candidate).tornStart !== 0) continue
try {
const decoded = (await decompressZstdFrame(candidate)).toString('utf8')
if (accepts(decoded)) return candidate
} catch {
// Some early cuts precede the first decodable block; keep searching for
// a cut that exercises partial-plaintext recovery.
}
}
throw new Error('test fixture could not produce the requested torn Zstandard frame')
}
function deterministicNoise(length: number): string {
let state = 0x12345678
let output = ''
for (let index = 0; index < length; index++) {
state = (Math.imul(state, 1_664_525) + 1_013_904_223) >>> 0
output += String.fromCharCode(33 + (state % 90))
}
return output
}
function emptyStructuralFrame(descriptor: number): Buffer {
const contentSizeFlag = descriptor >>> 6
const singleSegment = (descriptor & 0x20) !== 0
const dictionaryBytes = [0, 1, 2, 4][descriptor & 0x03]!
const contentSizeBytes = contentSizeFlag === 0 ? (singleSegment ? 1 : 0) : 1 << contentSizeFlag
const variableHeader = Buffer.alloc((singleSegment ? 0 : 1) + dictionaryBytes + contentSizeBytes)
const lastEmptyRawBlock = Buffer.from([1, 0, 0])
const checksum = (descriptor & 0x04) === 0 ? Buffer.alloc(0) : Buffer.alloc(4)
return Buffer.concat([MAGIC, Buffer.from([descriptor]), variableHeader, lastEmptyRawBlock, checksum])
}
afterEach(async () => {
vi.restoreAllMocks()
for (const ctx of contexts.splice(0).reverse()) await ctx.fiber.dispose()
for (const root of roots.splice(0)) await rm(root, { recursive: true, force: true })
})
runPersistenceContract('jsonl-zstd', async () => {
const root = await mkdtemp(join(tmpdir(), 'dsh-jsonl-zstd-contract-'))
const ctx = new Context()
await ctx.plugin(SessionStore)
const fiber = await ctx.plugin(SessionPersistenceJsonl, { root })
return {
persistence: ctx.sessionPersistence,
dispose: async () => {
await fiber.dispose()
await rm(root, { recursive: true, force: true })
},
}
})
runCoordinatorContract('jsonl-zstd', async (): Promise<CoordinatorFixture> => {
const root = await mkdtemp(join(tmpdir(), 'dsh-jsonl-zstd-coordinator-'))
return {
mount: async ctx => ctx.plugin(SessionPersistenceJsonl, { root }),
corruptTail: async (id, cwd) => {
const line = JSON.stringify({
type: 'assistant/chunk',
seq: 8,
time: 9,
data: { turn: 2, step: 1, chunk: { type: 'text-delta', index: 0, text: deterministicNoise(300_000) } },
}) + '\n'
const partial = await tornFrame(line, decoded => decoded.length > 0 && !decoded.endsWith('\n'))
await appendFile(logPath(root, cwd, id, 'zstd'), partial)
},
cleanup: async () => { await rm(root, { recursive: true, force: true }) },
}
})
describe('Zstandard frame structure', () => {
it('scans concatenated checksummed frames and honors a frame limit', async () => {
const first = await compressZstdFrame('header\n')
const second = await compressZstdFrame('event\n')
const stream = Buffer.concat([first, second])
expect(scanZstdFrames(Buffer.alloc(0))).toEqual({ frames: [] })
expect(scanZstdFrames(stream)).toEqual({
frames: [{ start: 0, end: first.length }, { start: first.length, end: stream.length }],
})
expect(scanZstdFrames(stream, 1)).toEqual({ frames: [{ start: 0, end: first.length }] })
expect(first[4]! & 0x04).toBe(0x04)
expect(second[4]! & 0x04).toBe(0x04)
expect((await decompressZstdFrame(first)).toString()).toBe('header\n')
})
it('distinguishes incomplete frame regions from invalid complete structure', () => {
expect(scanZstdFrames(MAGIC.subarray(0, 2))).toEqual({ frames: [], tornStart: 0 })
expect(scanZstdFrames(MAGIC)).toEqual({ frames: [], tornStart: 0 })
expect(() => scanZstdFrames(Buffer.alloc(4))).toThrow(/invalid frame magic/)
expect(() => scanZstdFrames(Buffer.concat([MAGIC, Buffer.from([0x08])]))).toThrow(/reserved frame-header bit/)
// Non-single-segment descriptor with no window descriptor.
expect(scanZstdFrames(Buffer.concat([MAGIC, Buffer.from([0x00])]))).toEqual({ frames: [], tornStart: 0 })
// Single-segment header followed by only two bytes of the three-byte block header.
expect(scanZstdFrames(Buffer.concat([MAGIC, Buffer.from([0x20, 0x00, 0x01, 0x00])]))).toEqual({
frames: [],
tornStart: 0,
})
const rawFiveBytes = Buffer.from([(5 << 3) | 1, 0, 0])
expect(scanZstdFrames(Buffer.concat([
MAGIC,
Buffer.from([0x20, 0x00]),
rawFiveBytes,
Buffer.from([0x01, 0x02]),
]))).toEqual({ frames: [], tornStart: 0 })
const reservedBlock = Buffer.concat([
MAGIC,
Buffer.from([0x20, 0x00, 0x07, 0x00, 0x00]),
])
expect(() => scanZstdFrames(reservedBlock)).toThrow(/reserved block type/)
})
it('covers standard header variants, RLE blocks, multiple blocks, and checksums', () => {
for (const descriptor of [0x00, 0x21, 0x42, 0x83, 0xE3]) {
const frame = emptyStructuralFrame(descriptor)
expect(scanZstdFrames(frame)).toEqual({ frames: [{ start: 0, end: frame.length }] })
}
const rle = Buffer.concat([
MAGIC,
Buffer.from([0x20, 0x01]),
Buffer.from([(1 << 3) | (1 << 1) | 1, 0, 0]),
Buffer.from([0x41]),
])
expect(scanZstdFrames(rle)).toEqual({ frames: [{ start: 0, end: rle.length }] })
const twoBlocks = Buffer.concat([
MAGIC,
Buffer.from([0x20, 0x00]),
Buffer.from([0, 0, 0]),
Buffer.from([1, 0, 0]),
])
expect(scanZstdFrames(twoBlocks)).toEqual({ frames: [{ start: 0, end: twoBlocks.length }] })
const checksummed = emptyStructuralFrame(0x24)
expect(scanZstdFrames(checksummed.subarray(0, -1))).toEqual({ frames: [], tornStart: 0 })
expect(scanZstdFrames(checksummed)).toEqual({ frames: [{ start: 0, end: checksummed.length }] })
})
})
describe('SessionPersistenceJsonl: default Zstandard encoding', () => {
it('writes .jsonl.zstd by default with one header frame and one first-batch frame', async () => {
const root = await freshRoot()
const ctx = await mount(root)
const header = meta('default-zstd', '/work')
await ctx.sessionPersistence.create(header)
await ctx.sessionPersistence.append(header.id, oneTurnLog())
const path = logPath(root, header.cwd, header.id, 'zstd')
const buffer = await readFile(path)
expect(buffer.subarray(0, 4)).toEqual(MAGIC)
await expect(stat(logPath(root, header.cwd, header.id, 'none'))).rejects.toThrow()
expect(ctx.sessionPersistence.locate(header)).toEqual({ kind: 'jsonl', path })
const scan = scanZstdFrames(buffer)
expect(scan.frames).toHaveLength(2)
const plaintext = await decodeCompleteFrames(buffer)
expect(plaintext.toString()).toBe([
JSON.stringify(toHeaderLine(header)),
...oneTurnLog().map(eventLine),
'',
].join('\n'))
expect((await ctx.sessionPersistence.load(header.id)).events).toEqual(oneTurnLog())
})
it('resolves the default when a programmatic wrapper bypasses Loader schema normalization', async () => {
const root = await freshRoot()
const ctx = new Context()
contexts.push(ctx)
await ctx.plugin(SessionStore)
let backend!: SessionPersistenceJsonl
await ctx.plugin(Object.assign((inner: Context) => {
backend = new SessionPersistenceJsonl(inner, { root })
}, { inject: ['sessions'] }))
const header = meta('direct-default')
expect(backend.locate(header)).toEqual({
kind: 'jsonl',
path: logPath(root, header.cwd, header.id, 'zstd'),
})
})
it('appends one frame per durable batch without rewriting prior bytes', async () => {
const root = await freshRoot()
const ctx = await mount(root)
const header = meta('append-frame')
await ctx.sessionPersistence.create(header)
await ctx.sessionPersistence.append(header.id, oneTurnLog())
const path = logPath(root, header.cwd, header.id, 'zstd')
const before = await readFile(path)
const secondTurn = [
{ type: 'turn/start', seq: 6, time: 7, data: { turn: 2, trigger: { kind: 'message', source: { kind: 'user' } } } },
{ type: 'turn/end', seq: 7, time: 8, data: { turn: 2, reason: { kind: 'completed' } } },
] as SessionEvent[]
await ctx.sessionPersistence.append(header.id, secondTurn)
const after = await readFile(path)
expect(after.subarray(0, before.length)).toEqual(before)
expect(scanZstdFrames(after).frames).toHaveLength(3)
expect((await ctx.sessionPersistence.load(header.id)).events).toEqual([...oneTurnLog(), ...secondTurn])
})
it('lists from a multi-chunk header frame without decoding a corrupt event frame', async () => {
const root = await freshRoot()
const ctx = await mount(root)
const header = meta('large-header', `/work/${'x'.repeat(24_000)}`)
await ctx.sessionPersistence.create(header)
await ctx.sessionPersistence.append(header.id, oneTurnLog())
const path = logPath(root, header.cwd, header.id, 'zstd')
const buffer = Buffer.from(await readFile(path))
const eventFrame = scanZstdFrames(buffer).frames[1]!
buffer[eventFrame.end - 1] = buffer[eventFrame.end - 1]! ^ 0xFF
await writeFile(path, buffer)
expect((await ctx.sessionPersistence.list()).map(item => item.id)).toEqual([header.id])
await expect(ctx.sessionPersistence.load(header.id)).rejects.toThrow(/frame at byte .* failed validation/)
})
it('preserves complete records from a torn frame and re-encodes them with crash closers', async () => {
const root = await freshRoot()
const ctx = await mount(root)
const header = meta('recover-torn', '/proj')
await ctx.sessionPersistence.create(header)
await ctx.sessionPersistence.append(header.id, oneTurnLog())
const path = logPath(root, header.cwd, header.id, 'zstd')
const committed = await readFile(path)
const openTurn = [
{ type: 'turn/start', seq: 6, time: 7, data: { turn: 2, trigger: { kind: 'message', source: { kind: 'user' } } } },
{ type: 'step/start', seq: 7, time: 8, data: { turn: 2, step: 1 } },
{ type: 'assistant/chunk', seq: 8, time: 9, data: { turn: 2, step: 1, chunk: { type: 'text-delta', index: 0, text: deterministicNoise(300_000) } } },
] as SessionEvent[]
const plaintext = openTurn.map(eventLine).join('\n') + '\n'
const partial = await tornFrame(plaintext, (decoded) => {
const newlines = decoded.match(/\n/g)?.length ?? 0
return newlines >= 2 && !decoded.endsWith('\n')
})
await appendFile(path, partial)
const loaded = await ctx.sessionPersistence.load(header.id)
expect(loaded.events.map(event => event.seq)).toEqual([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])
expect(loaded.events[6]).toEqual(openTurn[0])
expect(loaded.events[7]).toEqual(openTurn[1])
expect(loaded.events.some(event => event.type === 'assistant/chunk' && event.seq === 8)).toBe(false)
expect(loaded.events[8]?.type).toBe('step/end')
expect(loaded.events[9]?.type).toBe('turn/end')
const repaired = await readFile(path)
expect(repaired.subarray(0, committed.length)).toEqual(committed)
expect(scanZstdFrames(repaired).tornStart).toBeUndefined()
expect(scanLog(await decodeCompleteFrames(repaired)).events).toEqual(loaded.events)
})
it('drops a frame torn in its header before it has produced plaintext', async () => {
const root = await freshRoot()
const ctx = await mount(root)
const header = meta('partial-magic')
await ctx.sessionPersistence.create(header)
await ctx.sessionPersistence.append(header.id, oneTurnLog())
const path = logPath(root, header.cwd, header.id, 'zstd')
const committed = await readFile(path)
await appendFile(path, MAGIC.subarray(0, 2))
expect((await ctx.sessionPersistence.load(header.id)).events).toEqual(oneTurnLog())
expect(await readFile(path)).toEqual(committed)
})
it('recovers complete events when EOF tears only the final frame checksum', async () => {
const root = await freshRoot()
const ctx = await mount(root)
const header = meta('partial-checksum')
await ctx.sessionPersistence.create(header)
await ctx.sessionPersistence.append(header.id, oneTurnLog())
const path = logPath(root, header.cwd, header.id, 'zstd')
const secondTurn = [
{ type: 'turn/start', seq: 6, time: 7, data: { turn: 2, trigger: { kind: 'message', source: { kind: 'user' } } } },
{ type: 'turn/end', seq: 7, time: 8, data: { turn: 2, reason: { kind: 'completed' } } },
] as SessionEvent[]
const frame = await compressZstdFrame(secondTurn.map(eventLine).join('\n') + '\n')
await appendFile(path, frame.subarray(0, -1))
const loaded = await ctx.sessionPersistence.load(header.id)
expect(loaded.events).toEqual([...oneTurnLog(), ...secondTurn])
const repaired = await readFile(path)
expect(scanZstdFrames(repaired).tornStart).toBeUndefined()
expect(scanLog(await decodeCompleteFrames(repaired)).events).toEqual(loaded.events)
})
it('rejects a complete frame containing a torn JSONL record', async () => {
const root = await freshRoot()
const ctx = await mount(root)
const header = meta('complete-bad-jsonl')
await ctx.sessionPersistence.create(header)
await ctx.sessionPersistence.append(header.id, oneTurnLog())
await appendFile(
logPath(root, header.cwd, header.id, 'zstd'),
await compressZstdFrame('{"type":"turn/start"'),
)
await expect(ctx.sessionPersistence.load(header.id)).rejects.toThrow(/complete frame contains a torn JSONL record/)
})
it('rolls back a checksummed append frame when fsync fails', async () => {
const root = await freshRoot()
const ctx = await mount(root)
const header = meta('zstd-fsync-rollback')
await ctx.sessionPersistence.create(header)
await ctx.sessionPersistence.append(header.id, oneTurnLog())
const path = logPath(root, header.cwd, header.id, 'zstd')
const before = await readFile(path)
const handle = await open(path, 'r')
const prototype = Object.getPrototypeOf(handle) as { sync: () => Promise<void> }
await handle.close()
const realSync = prototype.sync
let failed = false
const spy = vi.spyOn(prototype, 'sync').mockImplementation(async function (this: FileHandle) {
if (!failed) {
failed = true
throw new Error('simulated Zstandard fsync failure')
}
return realSync.call(this)
})
const secondTurn = [
{ type: 'turn/start', seq: 6, time: 7, data: { turn: 2, trigger: { kind: 'message', source: { kind: 'user' } } } },
{ type: 'turn/end', seq: 7, time: 8, data: { turn: 2, reason: { kind: 'completed' } } },
] as SessionEvent[]
await expect(ctx.sessionPersistence.append(header.id, secondTurn)).rejects.toThrow(/simulated Zstandard fsync failure/)
expect(await readFile(path)).toEqual(before)
spy.mockRestore()
await ctx.sessionPersistence.append(header.id, secondTurn)
expect((await ctx.sessionPersistence.load(header.id)).events).toEqual([...oneTurnLog(), ...secondTurn])
})
it('skips empty, incomplete, and non-header compressed artifacts while rejecting malformed header frames', async () => {
const root = await freshRoot()
const bucket = sessionDir(root, undefined)
await mkdir(bucket, { recursive: true })
await writeFile(join(bucket, 'empty.jsonl.zstd'), '')
await writeFile(join(bucket, 'partial.jsonl.zstd'), MAGIC)
await writeFile(join(bucket, 'not-header.jsonl.zstd'), await compressZstdFrame('{"type":"turn/start"}\n'))
const ctx = await mount(root)
expect(await ctx.sessionPersistence.list()).toEqual([])
await writeFile(join(bucket, 'two-lines.jsonl.zstd'), await compressZstdFrame([
JSON.stringify(toHeaderLine(meta('two-lines'))),
JSON.stringify({ type: 'turn/start' }),
'',
].join('\n')))
await expect(ctx.sessionPersistence.list()).rejects.toThrow(/first frame is not exactly one header line/)
await expect(ctx.sessionPersistence.load(SessionId('two-lines')))
.rejects.toThrow(/first frame is not exactly one header line/)
})
it('rejects missing, empty, and checksum-corrupt header frames on targeted reads', async () => {
const root = await freshRoot()
const bucket = sessionDir(root, undefined)
await mkdir(bucket, { recursive: true })
await writeFile(logPath(root, undefined, SessionId('partial-only'), 'zstd'), MAGIC)
await writeFile(logPath(root, undefined, SessionId('empty-header'), 'zstd'), await compressZstdFrame(''))
const corruptHeader = Buffer.from(await compressZstdFrame(`${JSON.stringify(toHeaderLine(meta('bad-checksum')))}\n`))
corruptHeader[corruptHeader.length - 1] = corruptHeader[corruptHeader.length - 1]! ^ 0xFF
await writeFile(logPath(root, undefined, SessionId('bad-checksum'), 'zstd'), corruptHeader)
const ctx = await mount(root)
await expect(ctx.sessionPersistence.load(SessionId('partial-only')))
.rejects.toThrow(/empty or header-less Zstandard session log/)
await expect(ctx.sessionPersistence.load(SessionId('empty-header')))
.rejects.toThrow(/first frame is not exactly one header line/)
await expect(ctx.sessionPersistence.list()).rejects.toThrow(/header frame failed validation/)
})
})
describe('SessionPersistenceJsonl: encoding selection', () => {
it('rejects roots owned by the opposite encoding in both directions', async () => {
const rawRoot = await freshRoot('dsh-jsonl-raw-mismatch-')
const raw = await mount(rawRoot, 'none')
const rawHeader = meta('raw-log')
await raw.sessionPersistence.create(rawHeader)
await raw.sessionPersistence.append(rawHeader.id, oneTurnLog())
const defaultBackend = await mount(rawRoot)
await expect(defaultBackend.sessionPersistence.list()).rejects.toThrow(/configured for compression "zstd"/)
const zstdRoot = await freshRoot('dsh-jsonl-zstd-mismatch-')
const zstd = await mount(zstdRoot)
const zstdHeader = meta('zstd-log')
await zstd.sessionPersistence.create(zstdHeader)
await zstd.sessionPersistence.append(zstdHeader.id, oneTurnLog())
const rawBackend = await mount(zstdRoot, 'none')
await expect(rawBackend.sessionPersistence.list()).rejects.toThrow(/configured for compression "none"/)
})
it('rechecks targeted artifacts and listing after an initially empty root', async () => {
const root = await freshRoot()
const ctx = await mount(root)
expect(await ctx.sessionPersistence.list()).toEqual([])
const loadHeader = meta('late-raw-load', '/late')
await mkdir(sessionDir(root, loadHeader.cwd), { recursive: true })
await writeFile(logPath(root, loadHeader.cwd, loadHeader.id, 'none'), [
JSON.stringify(toHeaderLine(loadHeader)),
...oneTurnLog().map(eventLine),
'',
].join('\n'))
await expect(ctx.sessionPersistence.load(loadHeader.id)).rejects.toThrow(/uses \.jsonl/)
await expect((ctx.sessionPersistence as SessionPersistenceJsonl).loadLive(loadHeader.id, loadHeader.cwd))
.rejects.toThrow(/uses \.jsonl/)
await expect(ctx.sessionPersistence.list()).rejects.toThrow(/uses \.jsonl/)
})
it('refuses materialization when an opposite artifact appears after create', async () => {
const root = await freshRoot()
const ctx = await mount(root)
await ctx.sessionPersistence.list()
const header = meta('late-raw-materialize', '/late')
await ctx.sessionPersistence.create(header)
await mkdir(sessionDir(root, header.cwd), { recursive: true })
await writeFile(logPath(root, header.cwd, header.id, 'none'), [
JSON.stringify(toHeaderLine(header)),
...oneTurnLog().map(eventLine),
'',
].join('\n'))
await expect(ctx.sessionPersistence.append(header.id, oneTurnLog())).rejects.toThrow(/uses \.jsonl/)
expect((await readdir(sessionDir(root, header.cwd))).some(name => name.endsWith('.jsonl.zstd'))).toBe(false)
})
})

View File

@@ -52,5 +52,5 @@ None; this package neither assembles nor sends a provider request.
## Known Limitations and Deferred Work ## Known Limitations and Deferred Work
- **Session harvest is JSONL-only** — `runScenario` collects persisted `.jsonl` logs, so an example composed over the SQLite persistence backend has no snapshot path. - **Session harvest requires raw JSONL mode** — `runScenario` collects persisted `.jsonl` logs, so snapshot configs set `persistenceCompression: 'none'`; compressed JSONL and SQLite compositions have no snapshot-harvest path.
- **The subprocess boots the unbuilt tsx/Loader path only** — the built-bin artifact is guarded by the separate `built-bin` e2e smokes, never by this tier. - **The subprocess boots the unbuilt tsx/Loader path only** — the built-bin artifact is guarded by the separate `built-bin` e2e smokes, never by this tier.

View File

@@ -395,11 +395,11 @@ async function runStep(
* header line, and return them ordered primary-first: the top-level session (no * header line, and return them ordered primary-first: the top-level session (no
* `parentSession`) leads, then each subagent child by ascending `createdAt`. * `parentSession`) leads, then each subagent child by ascending `createdAt`.
* *
* The JSONL backend lays sessions out as `<root>/<cwd-bucket>/<encoded-id>.jsonl` * Snapshot configs select the JSONL backend's raw mode, which lays sessions
* (one bucket per cwd), so a parent and its same-cwd in-process child land in * out as `<root>/<cwd-bucket>/<encoded-id>.jsonl` (one bucket per cwd). A
* the SAME bucket — collecting all files across all buckets catches both (a * parent and its same-cwd in-process child land in the SAME bucket, so
* first-match short-circuit would silently drop the child). Returns `[]` if no * collecting all files across all buckets catches both. Returns `[]` if no log
* log was produced (a no-session scenario). * was produced (a no-session scenario).
*/ */
async function harvestSessionLogs(root: string): Promise<HarvestedLog[]> { async function harvestSessionLogs(root: string): Promise<HarvestedLog[]> {
let cwdDirs: string[] let cwdDirs: string[]

View File

@@ -83,15 +83,12 @@ def run_smoke(repo_root: Path, keep_sessions: bool) -> None:
assert request["authorization"] == "Bearer sdk-smoke-key" assert request["authorization"] == "Bearer sdk-smoke-key"
assert request["body"]["model"] == "sdk-smoke-model" assert request["body"]["model"] == "sdk-smoke-model"
jsonl_files = sorted(session_root.rglob("*.jsonl")) jsonl_files = sorted(session_root.rglob("*.jsonl.zstd"))
assert jsonl_files, f"no jsonl sessions were written under {session_root}" assert jsonl_files, f"no Zstandard JSONL sessions were written under {session_root}"
print("session_jsonl_files:") print("session_jsonl_zstd_files:")
for path in jsonl_files: for path in jsonl_files:
print(f" {path} bytes={path.stat().st_size}") print(f" {path} bytes={path.stat().st_size}")
with path.open("r", encoding="utf-8") as handle: assert path.read_bytes().startswith(bytes.fromhex("28b52ffd"))
first_line = handle.readline().strip()
if first_line:
print(f" first_line={first_line[:500]}")
finally: finally:
server.shutdown() server.shutdown()
server.server_close() server.server_close()

View File

@@ -181,6 +181,11 @@ function gatesForMode(selected: Mode): Gate[] {
'run', 'run',
'packages/workflow/workflow-workerthread/tests/source-worker.compat.spec.ts', 'packages/workflow/workflow-workerthread/tests/source-worker.compat.spec.ts',
], { label: 'source worker smoke' }), ], { label: 'source worker smoke' }),
pnpmExec('jsonl-zstd-smoke', [
'vitest',
'run',
'packages/session-persistence/session-persistence-jsonl/tests/zstd.compat.spec.ts',
], { label: 'JSONL Zstandard smoke' }),
] ]
case 'pre-push': case 'pre-push':
return [ return [
@@ -379,7 +384,7 @@ function demoSmokeGate(options: { needs?: string[] } = {}): Gate {
for (const bucket of buckets) { for (const bucket of buckets) {
if (!bucket.isDirectory() || !bucket.name.startsWith('cwd-')) continue if (!bucket.isDirectory() || !bucket.name.startsWith('cwd-')) continue
const entries = await readdir(join(sessionsRoot, bucket.name)) const entries = await readdir(join(sessionsRoot, bucket.name))
if (entries.some(entry => /^main-session-.+\.jsonl$/.test(entry))) { if (entries.some(entry => /^main-session-.+\.jsonl\.zstd$/.test(entry))) {
found = true found = true
break break
} }

View File

@@ -64,6 +64,7 @@ CUSTOM_CORDIS = """\
name: '@deepseek-ai/dsh-session-persistence-jsonl' name: '@deepseek-ai/dsh-session-persistence-jsonl'
config: config:
root: !!js process.env.DSH_SESSION_ROOT root: !!js process.env.DSH_SESSION_ROOT
compression: 'none'
- id: bash - id: bash
name: '@deepseek-ai/dsh-bash-local' name: '@deepseek-ai/dsh-bash-local'
config: config:
@@ -391,7 +392,7 @@ def smoke_sdk_default(base_url: str) -> None:
result = harness.run("reply with the smoke text", session_id="default-smoke") result = harness.run("reply with the smoke text", session_id="default-smoke")
assert result.status == "ok", result assert result.status == "ok", result
assert result.final_response == EXPECTED_TEXT, result.final_response assert result.final_response == EXPECTED_TEXT, result.final_response
assert_session_log(sessions, root, EXPECTED_TEXT) assert_zstd_session_log(sessions)
def smoke_sdk_custom(base_url: str, executable: Path) -> None: def smoke_sdk_custom(base_url: str, executable: Path) -> None:
@@ -585,6 +586,14 @@ def assert_session_log(sessions: Path, cwd: Path, *expected_texts: str) -> None:
raise AssertionError(f"session log has no {expected!r} response: {logs[0]}") raise AssertionError(f"session log has no {expected!r} response: {logs[0]}")
def assert_zstd_session_log(sessions: Path) -> None:
logs = list(sessions.rglob("*.jsonl.zstd"))
if len(logs) != 1:
raise AssertionError(f"expected one Zstandard JSONL session log under {sessions}, found {logs}")
if not logs[0].read_bytes().startswith(bytes.fromhex("28b52ffd")):
raise AssertionError(f"session log has no Zstandard magic: {logs[0]}")
def read_session_logs(sessions: Path) -> dict[str, list[dict[str, object]]]: def read_session_logs(sessions: Path) -> dict[str, list[dict[str, object]]]:
"""Parse every persisted JSONL session into a map keyed by header id.""" """Parse every persisted JSONL session into a map keyed by header id."""
logs: dict[str, list[dict[str, object]]] = {} logs: dict[str, list[dict[str, object]]] = {}