docs: rebalance prose cleanup and add trimming skill

This commit is contained in:
Tianyi Cui
2026-07-13 23:27:00 +08:00
parent fcdc318dda
commit 148046b9c8
392 changed files with 2801 additions and 1754 deletions

View File

@@ -39,4 +39,4 @@ A scenario booting a differently-composed tree sets its own `configPath` (an ove
Examples use a `cordis.snapshot.yml` overlay with [`dsh-llm-replay`](../llm-replay/README.md). Recording calls the live model and updates model fixtures; keyless refresh replays those fixtures and updates derived stdout, session-log, and prompt snapshots. See the [snapshot RFC](../../../docs/rfc/implemented/testing/2026-06-19-acp-snapshot-tests.md).
`suite.ts` imports Vitest, so use this package only inside a Vitest run. The ACP-specific script can queue permission answers by stable option kind and can set session config options or assert their rejection. Missing permission answers cancel; selecting an unavailable kind fails the scenario.
`suite.ts` imports Vitest, so use this package only inside a Vitest run. The ACP-specific script can queue permission answers by stable option kind and can set session config options or assert their rejection in the transcript. Missing permission answers cancel; selecting an unavailable kind fails the scenario.

View File

@@ -1,7 +1,8 @@
/**
* Shared subprocess harness for ACP snapshot suites. A library module driven by the suite
* factory in ./suite.ts (and directly by harness-level specs); each example's `*.snapshot.ts`
* names its own agent-under-test paths.
* Shared ACP snapshot subprocess harness. It boots the real agent bin through the Cordis
* loader, drives deterministic ACP JSON-RPC over stdio, captures protocol-pure stdout, and
* harvests persisted session logs after graceful shutdown. Normalization stays in
* `normalize.ts`; suite registration stays in `suite.ts`.
* @module @deepseek-ai/dsh-acp-snapshot/harness
*/
@@ -57,8 +58,8 @@ export interface AgentUnderTest {
/**
* One step of a scenario's deterministic input script (`input.json`). The harness interprets
* these in order. `newSession` captures the server-issued (random) session id into a
* `{{sessionId}}` variable that later steps reference, since a committed file cannot know the
* id in advance.
* `{{sessionId}}` variable that later steps reference. `promptAndCancel` sends without awaiting,
* waits for the first streamed message, then cancels, making transcript order deterministic.
*/
export type InputStep =
| { op: 'initialize'; terminalOutput?: boolean }
@@ -75,8 +76,9 @@ export type InputStep =
export interface InputScript {
steps: InputStep[]
/**
* Ordered answers for the agent's `session/request_permission` round-trips, consumed FIFO —
* the Nth request gets the Nth answer.
* FIFO permission answers selected by stable option kind; the harness maps each kind to the
* agent-issued option id. Exhaustion cancels, while a kind the agent did not offer fails the
* scenario.
*/
permissionAnswers?: PermissionAnswer[]
}
@@ -202,8 +204,8 @@ export async function runScenario(input: InputScript, opts: RunOptions): Promise
child.stderr.setEncoding('utf8')
child.stderr.on('data', (c: string) => stderrChunks.push(c))
// Tee raw stdout: accumulate the bytes for the golden + purity check, and ALSO feed the
// same bytes to the SDK client through a passthrough.
// Tee the same raw bytes to the golden and SDK client. Decode once at the end so a UTF-8
// sequence split across stream chunks cannot corrupt the transcript.
const passthrough = new Readable({ read() {} })
child.stdout.on('data', (buf: Buffer) => {
rawBuffers.push(buf)
@@ -226,8 +228,8 @@ export async function runScenario(input: InputScript, opts: RunOptions): Promise
// Permission answers are consumed FIFO across the whole run; exhaustion
// falls back to `cancelled` so approval-free scenarios keep the plain stub.
const permissionQueue = [...input.permissionAnswers ?? []]
// A scenario bug detected inside a client callback (a scripted permission kind the agent
// never offered).
// A callback throw would become only an RPC error the agent could absorb. Record an
// impossible permission choice here, answer cancelled, and fail the outer scenario.
let scriptError: Error | undefined
const makeClient = (_agent: AcpAgent): Client => ({
sessionUpdate(params: SessionNotification): Promise<void> {
@@ -353,7 +355,8 @@ async function runStep(
case 'promptAndCancel': {
const sessionId = getSessionId()
if (sessionId === undefined) throw new Error('snapshot-harness: promptAndCancel before newSession')
// Dispatch the prompt WITHOUT awaiting (a hang fixture never resolves on its own).
// A hang fixture never resolves alone. Wait for its streamed chunk before cancellation
// so updates deterministically precede the cancelled prompt response.
const promptDone = client.prompt({ sessionId, prompt: [{ type: 'text', text: step.text }] })
await waitForUpdate(u => u.sessionUpdate === 'agent_message_chunk')
await client.cancel({ sessionId })
@@ -439,7 +442,8 @@ async function harvestSessionLogs(root: string): Promise<HarvestedLog[]> {
})
}
}
// Primary (no parentSession) first, then children by ascending createdAt.
// Match replay fixture assignment: primary first, then children by creation time, with id as
// a deterministic collision tiebreaker.
logs.sort((a, b) => {
const ap = a.parentSession === undefined ? 0 : 1
const bp = b.parentSession === undefined ? 0 : 1

View File

@@ -1,6 +1,7 @@
/**
* ACP snapshot suite kit — the shared machinery behind the keyless snapshot tier (`pnpm run
* test:snapshot`).
* ACP snapshot suite kit: subprocess scenario harness, pure golden normalizers, and the Vitest
* suite factory behind `pnpm run test:snapshot`. Because this entry exports `suite.ts`, importing
* it requires a Vitest run.
* @module @deepseek-ai/dsh-acp-snapshot
*/

View File

@@ -1,8 +1,8 @@
/**
* Pure normalizers for the ACP snapshot goldens. They replace the non-deterministic values in
* the two captured surfaces — the stdout JSON-RPC transcript and the persisted session JSONL —
* with stable tokens, so a golden compare reflects behavior, not run-to-run noise. Kept
* dependency-free and side-effect-free so they unit-test trivially.
* Pure ACP transcript and session-log normalizers. They scrub session ids, temp cwd, RPC ids,
* timestamps, and hook duration while preserving deterministic event sequence numbers.
* Request-header scrubbers stay separate so one scenario per header class can pin tools and a
* readable prompt while other fixtures omit duplicated header bulk.
* @module @deepseek-ai/dsh-acp-snapshot/normalize
*/
@@ -50,6 +50,7 @@ function scrubValue(value: unknown, ctx: NormalizeContext): unknown {
* Normalize a raw stdout transcript (newline-delimited JSON-RPC frames) into a stable golden
* in the same shape as the wire: one compact JSON frame per line (NDJSON), with the JSON-RPC
* `id` rewritten to a per-transcript sequence (1, 2, 3, …) and all volatile strings scrubbed.
* Invalid JSON throws, doubling as a protocol-stdout purity check.
*
* @param rawStdout The captured stdout bytes, decoded utf8.
* @param ctx The run's volatile values to scrub.

View File

@@ -1,5 +1,12 @@
/**
* The ACP snapshot suite factory (replay by default, keyless).
* Keyless-by-default ACP snapshot suite factory. Each scenario drives the real subprocess and
* compares normalized stdout; comparable session fixtures are both replay input and expected
* output. Record mode refreshes reproducible model scenarios from the live API, while refresh
* mode replays committed scripts and rewrites derived artifacts without a key.
*
* Exactly one scenario per header-composition class pins tool schemas in JSONL and the system
* prompt in Markdown. Every live header is checked against that pin, so session-dependent
* composition must declare a separate class instead of escaping coverage.
* @module @deepseek-ai/dsh-acp-snapshot/suite
*/
@@ -61,7 +68,8 @@ export interface Scenario {
*/
childSessions?: number
/**
* Whether this scenario pins its header class's model-facing request-header content.
* Whether this scenario is its header class's sole request-header pin. Its Markdown file owns
* the prompt, its JSONL keeps tool schemas, and every classmate is checked for equality.
*/
pinsHeader?: boolean
/**
@@ -123,8 +131,9 @@ export function childFixturePaths(dir: string, childSessions: number): string[]
}
/**
* Derive the {@link NormalizeContext} for a `session.jsonl` fixture from its own header line
* (`{ type: 'session', id, cwd }`).
* Derive normalization values from a fixture's own session header. Recorded ids and cwd differ
* from the live replay run; the non-empty sentinel for missing cwd avoids accidental empty-
* string replacement.
*
* @param fixture The committed `session.jsonl` content.
* @returns The fixture's own volatile values, ready for {@link normalizeSessionLog}.
@@ -403,8 +412,8 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
cwd: result.cwd,
}
// RECORD mode (recorded model scenarios only): persist the freshly-harvested live logs
// back to their fixtures.
// Record writes live model fixtures; keyless refresh writes every comparable replayed
// fixture. Pins keep tools but all JSONL files scrub prompt text.
const scrub = scenario.pinsHeader === true
? scrubSystemPrompts
: scrubRequestHeaders

View File

@@ -1,5 +1,7 @@
/**
* Scripted fake ACP agent bin for `dsh-acp-snapshot`'s unit specs.
* Scripted ACP agent for snapshot-kit tests. A fixture-adjacent `behavior.json` controls the
* subprocess reached through the real harness path; the bin reports observations over ACP and
* writes scripted logs before exiting on stdin EOF.
*/
import { mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'

View File

@@ -308,7 +308,8 @@ describe('runScenario', () => {
it('rejects the run on a scripted permission kind the agent never offered', { timeout: 20_000 }, async () => {
const { fixtureFile } = await scenario({ permissionProbe: true })
// The fake bin offers allow_once/reject_once; scripting allow_always is a scenario bug.
// The fake offers only allow_once/reject_once. The harness must reject an impossible click,
// not merely send an RPC error that a tolerant agent could absorb.
await expect(runScenario(
{ steps: [...boot, { op: 'prompt', text: 'impossible click' }], permissionAnswers: [{ kind: 'allow_always' }] },
{ agent: AGENT, mode: 'replay', fixtureFile },

View File

@@ -23,6 +23,9 @@ import {
* so every factory path — golden and log compares, the per-suite header pin and its uniformity
* guard, record-mode fixture write-back, skip semantics, and the fixture guard block —
* executes as an ordinary green test.
*
* Record tests use a temp copy. To intentionally rebuild their committed fixtures, run this
* spec once with `ACP_SNAPSHOT_SPEC_BOOTSTRAP=1`, then review and commit the resulting tree.
*/
const AGENT = {

View File

@@ -39,7 +39,7 @@ Agent status (per agent):
Model requests (on `llm/stream`):
- **a loop-built request is exactly what the log reconstructs** — frozen requests with a live `sessionId` must match a fresh derivation bounded before the in-flight `step/start`, while non-content fields match the folded request headers. The check is prepended so ordinary short-circuiting stream listeners cannot skip it; correctness comes from the sequence boundary, not listener order. See the [reconstructability RFC](../../../docs/rfc/implemented/architecture/2026-07-05-reconstructable-requests.md).
- **a loop-built request is exactly what the log reconstructs** — a frozen request with a live `sessionId` is rebuilt through a fresh `Session` from the prefix before its in-flight `step/start`; later content belongs to the next request, and hand-built unfrozen one-shots are excluded. Frozen messages must match that derivation, while every other field matches folded `request/header*` events. The prepended check runs before ordinary short-circuiting stream listeners, but correctness comes from the sequence boundary rather than listener timing. See the [reconstructability RFC](../../../docs/rfc/implemented/architecture/2026-07-05-reconstructable-requests.md).
On any violation it throws `InvariantError` (`code: 'INVARIANT'`).

View File

@@ -1,6 +1,8 @@
/**
* Dev-only listener plugin for cross-event lifecycle, scope, and request
* invariants that types cannot express.
* Dev-only listeners for relationships that event types and immutability cannot express: turn and
* step nesting, scoped dispatch, status transitions, and request reconstruction. Enable in tests
* and demos, not production. Sessions already snapshot and freeze individual events; this plugin
* checks the cross-event contract and serves as its executable documentation.
* @module @deepseek-ai/dsh-invariants
*/
@@ -315,19 +317,15 @@ function replayEvent(trace: SessionTrace, event: SessionEvent): void {
applyTransition(trace, validateEvent(trace, event))
}
/** Legal agent status transitions (the only state machine the loop guarantees). */
/** Allow an initial observation, idle/running transitions, and terminal disposal; reject repeats and leaving disposed. */
function checkTransition(from: AgentStatus | undefined, to: AgentStatus): void {
// First observation: any status is a valid starting point.
if (from === undefined) return
// A no-op transition is illegal — setStatus dedups, so we never see it.
if (from === to) {
throw new InvariantError(`agent/status repeated ${to} (no-op transition)`)
}
// Leaving `disposed` is illegal — disposal is terminal.
if (from === 'disposed') {
throw new InvariantError(`agent/status left terminal state disposed → ${to}`)
}
// idle↔running and (idle|running)→disposed are all legal; nothing else exists.
}
/**

View File

@@ -325,19 +325,17 @@ describe('HMR state rebuild', () => {
it('rebuilds trace state for a session that exists at (re-)apply time', async () => {
const ctx = new Context()
await ctx.plugin(SessionStore)
// First registration, mid-turn: a turn is open when the plugin reloads.
const first = await ctx.plugin(Invariants)
const session = ctx.sessions.create()
session.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } })
session.append('step/start', { turn: 1, step: 1 })
await first.dispose()
// Re-apply (HMR): the fresh fiber must replay the existing log so the open
// step is known — the next chunk must NOT be a false positive.
// Re-apply mid-step: the new fiber must reconstruct the open boundaries from the log.
await ctx.plugin(Invariants)
expect(() => session.append('assistant/chunk', { turn: 1, step: 1, chunk: { type: 'text-delta', index: 0, text: 'h' } }))
.not.toThrow()
// And a genuine violation is still caught after the rebuild.
// Rebuild must not disable later violations.
expect(() => session.append('turn/start', { turn: 2, trigger: { kind: 'message', source: { kind: 'user' } } }))
.toThrow(/turn 1 is still open/)
})
@@ -541,25 +539,17 @@ describe('surface invariants', () => {
})
it('rejects sourceEventSeqs referencing unknown seq (gap in event log)', async () => {
// The unknown-seq check fires when a ref passes the "earlier" test but is
// not in knownSeqs — only possible with a gap in seqs. We create a gap by
// directly manipulating the private log array to skip a seq.
// Create an impossible-through-public-API gap so seq 2 is earlier but unknown.
const { ctx } = await setup()
const session = ctx.sessions.create()
session.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } })
session.append('step/start', { turn: 1, step: 1 })
// Push a fake event at seq 3 into the internal log, creating a gap at seq 2.
// The invariants plugin replays session.events on every append, so it sees
// this gap during trace reconstruction.
;(session as unknown as { log: unknown[] }).log.push({
type: 'assistant/chunk',
seq: 3,
time: Date.now(),
data: { turn: 1, step: 1, chunk: { type: 'text-delta', index: 0, text: 'x' } },
})
// Now the log has seqs 0, 1, 3 (gap at 2). Append at what session believes
// is seq 3 (log.length). Reference seq 2: passes earlier (2 < 3) but not
// in knownSeqs ({0, 1, 3} — gap at 2).
expect(() => {
session.append('assistant/message', { turn: 1, step: 1, content: [] }, { surfaceOp: 'append', sourceEventSeqs: [2] })
}).toThrow(/unknown seq 2/)
@@ -803,7 +793,8 @@ describe('request-reconstruction cross-check (llm/stream)', () => {
describe('request cross-check ordering (prepend)', () => {
it('runs ahead of a short-circuiting llm/stream listener registered before it', async () => {
// The prepended check must run before a short-circuiting replay listener.
// Replay short-circuits without next(), so the check prepends ahead of ordinary listeners;
// correctness still comes from its sequence-bounded rebuild, not listener timing.
const ctx = new Context()
await ctx.plugin(SessionStore)
ctx.on('llm/stream', () => (async function* () {})() as never) // short-circuits, no next()

View File

@@ -1,5 +1,8 @@
/**
* Replay LLM plugin for snapshot tests.
* Keyless snapshot-test LLM replay. It derives one model-call script per
* recorded session from `assistant/chunk` events and binds fresh live sessions
* to parent/child scripts by first-call order. Throw and hang cases require an
* explicit override because a session log cannot reconstruct them alone.
* @module @deepseek-ai/dsh-llm-replay
*/
@@ -11,11 +14,9 @@ import type { GenerateOptions, StreamChunk } from '@deepseek-ai/dsh-llm'
import { LlmError, assertNever } from '@deepseek-ai/dsh-llm'
/**
* One recorded model call. A discriminated union (not a bare `StreamChunk[]`) so it can
* faithfully replay BOTH branches of the documented LLM failure contract — an adapter may
* THROW from `stream()` or end with a `finish` error chunk — plus a `hang` marker for
* cancellation scenarios (mirrors the `MockAdapter` `hang` support in
* packages/core/agent-loop/tests).
* One recorded model call. `throw` may replay prefix chunks before failing;
* `hang` models cancellation. Only ordinary chunk entries derive from JSONL;
* the other variants come from an override sidecar.
*/
export type ReplayEntry =
| { kind: 'chunks'; chunks: StreamChunk[] }
@@ -46,7 +47,10 @@ export interface ReplayConfig {
childFiles?: string[]
}
/** Recorded calls plus header facts used to order parent and child replay scripts. */
/**
* Recorded calls plus header facts used to order parent and child scripts.
* Recorded ids are diagnostic; fresh live ids bind by ordered first use.
*/
export interface SessionScript {
/** The recorded session id (diagnostics only — the live id differs). */
recordedId: string
@@ -71,9 +75,7 @@ export interface SessionScript {
export function parseSessionLog(text: string): SessionEvent[] {
const lines = text.split('\n').filter(line => line.trim().length > 0)
const events: SessionEvent[] = []
// Skip line 0 (the header). A reader distinguishes it by its `type:'session'`
// tag; we simply drop the first line, which the JSONL backend guarantees is
// the header.
// The JSONL backend guarantees line 0 is the session header.
for (let i = 1; i < lines.length; i++) {
const parsed: unknown = JSON.parse(lines[i] as string)
events.push(parsed as SessionEvent)
@@ -100,6 +102,9 @@ export function parseSessionHeader(text: string): { id: string; createdAt: numbe
/**
* Reconstruct the per-`stream()` replay script from a recorded session log.
*
* Groups `assistant/chunk` events by turn and step. Every group must end in a
* `finish`; a missing terminator means the live stream threw, so derivation
* rejects and the scenario must provide an explicit override.
* @param events - the recorded session's events; only `assistant/chunk` is consulted.
* @returns one `chunks` entry per recorded model call, in call order.
*/
@@ -158,8 +163,8 @@ export function loadReplayScript(config: ReplayConfig): ReplayEntry[] {
}
/**
* Load every recorded session's script for a scenario, ordered by `createdAt` (earliest
* first), ready to bind to live sessions in first-call order.
* Load the primary and child scripts in bind order. Child derivation begins at
* `seedLength` so inherited parent chunks are never replayed as child calls.
*
* @param config - the fixture paths: the primary log plus any recorded child logs.
* @returns the primary script first, then the child scripts in bind order.
@@ -236,9 +241,10 @@ async function* replayEntry(entry: ReplayEntry, signal: AbortSignal | undefined)
}
/**
* Install the replay `llm/stream` listener on `ctx`. Returns the listener disposer (so a fiber
* dispose removes it — HMR safety). Exported separately from {@link apply} so unit tests can
* drive it without the Loader or env vars.
* Install per-session positional replay. A newly seen live session takes the
* next ordered recorded script, then advances its own cursor synchronously at
* invocation time; calls without `sessionId` share one anonymous session.
* Returns the effect disposer for HMR-safe removal.
*
* @param ctx - the context whose `llm/stream` waterfall the listener short-circuits.
* @param config - the resolved fixture paths (env-var defaulting is `apply`'s job).

View File

@@ -427,8 +427,8 @@ describe('loadSessionScripts', () => {
})
it('derives a FORK child script from its OWN events only (skips the seeded parent prefix)', () => {
// A fork child's log begins with the seeded parent prefix — the parent's events, INCLUDING
// its assistant/chunk events.
// A fork log includes the parent's assistant chunks before `seedLength`. Deriving from the
// whole log would replay parent responses as child calls, so only child-owned chunks qualify.
const parentChunk: StreamChunk = { type: 'text-delta', index: 0, text: 'PARENT-RESPONSE' }
const childChunks: StreamChunk[] = [{ type: 'text-delta', index: 0, text: 'CHILD-RESPONSE' }, { type: 'finish', reason: { kind: 'stop' } }]
const f = writeSession('session.jsonl', { id: 'parent', createdAt: 100 }, [TEXT_CHUNKS])
@@ -485,7 +485,8 @@ describe('loadSessionScripts', () => {
})
it('keeps the primary first even when a child sorts BEFORE it in input order', () => {
// Equal creation times keep the primary first regardless of input order.
// The primary is appended first internally. A strictly earlier child sorts before it, while
// equal creation times preserve primary-first order regardless of input order.
const f = writeSession('session.jsonl', { id: 'parent', createdAt: 100 }, [TEXT_CHUNKS])
const earlier = writeSession('session.1.jsonl', { id: 'early', createdAt: 100 }, [TEXT_CHUNKS])
const scripts = loadSessionScripts({ file: f, childFiles: [earlier] })

View File

@@ -1,6 +1,7 @@
/**
* A scripted {@link SubagentProvider} for testing the subagent seam WITHOUT a model or a real
* child agent.
* Scripted, model-free subagent provider for deterministic coverage of registration,
* capability checks, lifecycle, the model-facing tool, and structured results through the real
* loader path. It is a named-export functional plugin; no default export.
* @module @deepseek-ai/dsh-subagent-mock
*/

View File

@@ -105,7 +105,7 @@ describe('dsh-subagent-mock', () => {
})
it('has the namespace-plugin export shape (no stray default) so the Loader keeps name/inject/Config/apply', () => {
// Loader must retain this namespace's injection metadata.
// A default export would make Loader unwrap only that value and drop `inject`.
expect('default' in mock).toBe(false)
expect(mock.name).toBe('subagent-mock')
expect(mock.inject).toEqual(['subagents'])