Coder/deepseek-harness
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into feat/plan-mode

Browse Source
This commit is contained in:
Tianyi Cui
2026-07-22 11:29:58 +08:00
committed by GitHub
parent ac21ae0794 6646ef04ac
commit c1f259bd0d
71 changed files with 2447 additions and 312 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
packages/support/acp-snapshot/src/harness.ts
View File

@@ -177,11 +177,21 @@ export interface RunOptions {
configPath?: string
}
/** Derive one stable, fixed-length spill root owned by this scenario. */
function scenarioSpillRoot(fixtureFile: string): string {
/**
* Derive one stable, fixed-length spill root owned by this scenario.
* Windows uses a two-character-shorter root because drive resolution adds its drive prefix.
* @param fixtureFile - The scenario fixture whose parent directory provides the stable identity.
* @param platform - the host platform, injectable for unit coverage.
* @returns the root-relative snapshot spill directory.
*/
export function snapshotSpillRoot(
fixtureFile: string,
platform: NodeJS.Platform = process.platform,
): string {
const scenario = basename(dirname(fixtureFile))
const key = createHash('sha256').update(scenario).digest('hex').slice(0, 9)
return `/tmp/dsh-acp-snap-${key}`
const root = platform === 'win32' ? '/t' : '/tmp'
return `${root}/dsh-acp-snap-${key}`
}
/**
@@ -200,7 +210,7 @@ export async function runScenario(input: InputScript, opts: RunOptions): Promise
// before stdout normalization, so tmpdir() length differences churn expected outputs.
// Scenario ownership also matters: replay runs concurrently, and one teardown
// must never delete another scenario's in-flight full-output recovery file.
const spillRoot = scenarioSpillRoot(opts.fixtureFile)
const spillRoot = snapshotSpillRoot(opts.fixtureFile)
// Everything past the temp-dir creation is followed by failure-safe cleanup,
// so a failure in workspace seeding, spawn, or any step never leaks resources.
let launched: LaunchedAcpTestAgent | undefined

2
packages/support/acp-snapshot/src/index.ts
View File

@@ -38,7 +38,9 @@ export {
scrubRequestHeaders,
scrubSystemPrompts,
scrubToolSchemas,
type CwdPathMode,
type NormalizeContext,
type NormalizeOptions,
} from './normalize.ts'
export {
defineAcpSnapshotSuite,

30
packages/support/acp-snapshot/src/launcher.ts
View File

@@ -23,6 +23,8 @@ import {
} from '@agentclientprotocol/sdk'
import { resolveExampleLaunch } from '@deepseek-ai/dsh-loader-smoke'
const EXIT_MARKER_GRACE_MS = 250
/** The source/built agent entry, leaf config, and workspace tsconfig an ACP test boots. */
export interface AgentUnderTest {
/** The agent source bin entry (for example `packages/examples/acp-demo/src/bin.ts`). */
@@ -238,6 +240,15 @@ export function launchAcpTestAgent(options: AcpTestLaunchOptions): LaunchedAcpTe
return
}
const propagateFailureAfterDrain = async (): Promise<never> => {
await drained
closeUpdateStream()
throw failure
}
// Windows implements the supported signal names as forced termination. The exit markers
// may therefore arrive after the error wins the race above but before fallback begins.
if (!isRunning(child) || await exitMarkerWithinGrace(exited)) return propagateFailureAfterDrain()
// An `error` after spawn is not an exit edge: in particular, a failed
// signal can leave the subprocess live. Force termination, await the
// already-observed exit edge, and only then propagate the child error so
@@ -247,6 +258,10 @@ export function launchAcpTestAgent(options: AcpTestLaunchOptions): LaunchedAcpTe
child.once('error', observeFallbackError)
if (!child.kill('SIGKILL')) {
child.off('error', observeFallbackError)
// A successful earlier signal may win between the live check and this fallback call.
// In that case `kill()` correctly reports no process to signal; the original child error
// remains the shutdown result once inherited stdio and callbacks have drained.
if (!isRunning(child) || await exitMarkerWithinGrace(exited)) return propagateFailureAfterDrain()
closeUpdateStream()
throw new AggregateError(
[failure, new Error('Fallback SIGKILL was not accepted by the child process')],
@@ -265,9 +280,7 @@ export function launchAcpTestAgent(options: AcpTestLaunchOptions): LaunchedAcpTe
'ACP test agent failed and fallback termination was refused',
)
}
await drained
closeUpdateStream()
throw failure
return propagateFailureAfterDrain()
},
}
}
@@ -277,6 +290,17 @@ function waitForExit(child: ChildProcessWithoutNullStreams): Promise<void> {
return new Promise<void>(resolve => child.once('exit', () => { resolve() }))
}
/** Give an accepted Windows termination request a bounded window to publish its exit marker. */
function exitMarkerWithinGrace(exited: Promise<void>): Promise<boolean> {
return Promise.race([
exited.then(() => true),
new Promise<false>((resolve) => {
const timer = setTimeout(() => { resolve(false) }, EXIT_MARKER_GRACE_MS)
timer.unref()
}),
])
}
/** Whether the child still lacks either OS termination marker. */
function isRunning(child: ChildProcessWithoutNullStreams): boolean {
return child.exitCode === null && child.signalCode === null

66
packages/support/acp-snapshot/src/normalize.ts
View File

@@ -13,19 +13,33 @@ const TOOLS = '{{tools}}'
const MESSAGE_PREFIX = '{{messagePrefix}}'
const UPDATED_AT = '{{updatedAt}}'
/** A cwd-rooted path after volatile cwd replacement, through its last separator-delimited segment. */
const CWD_ROOTED_PATH_RE = /\{\{cwd\}\}(?:[\\/][^\s<>"'`]+)+/g
const PATH_TAG_RE = /(<path>)([^<]*)(<\/path>)/g
const ADDITIONAL_INSTRUCTIONS_PATH_RE = /(Additional instructions from: )([^\r\n]+)/g
/** A UUID v4 string, the shape `randomUUID()` produces for session ids. */
const UUID_RE = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/gi
const LOCAL_SPILL_PATH_RE = new RegExp(
String.raw`\{\{cwd\}\}/\.spill/session-[0-9a-f]{12}/[0-9a-f]{12}-([A-Za-z0-9._~-]+?)`
String.raw`\{\{cwd\}\}[\\/]\.spill[\\/]session-[0-9a-f]{12}[\\/][0-9a-f]{12}-([A-Za-z0-9._~-]+?)`
+ String.raw`(?=\. Use read with offset/limit|[\s)]|$)`,
'g',
)
const SNAPSHOT_SPILL_PATH_RE = new RegExp(
String.raw`/tmp/(?:dsh-acp-snap-[0-9a-f]{9}|dsh-acp-snapshot-spill)/session-[0-9a-f]{12}/[0-9a-f]{12}-([A-Za-z0-9._~-]+?)`
String.raw`(?:[A-Za-z]:)?[\\/](?:tmp|t)[\\/](?:dsh-acp-snap-[0-9a-f]{9}|dsh-acp-snapshot-spill)[\\/]session-[0-9a-f]{12}[\\/][0-9a-f]{12}-([A-Za-z0-9._~-]+?)`
+ String.raw`(?=\. Use read with offset/limit|[\s)]|$)`,
'g',
)
/** Convert separators only inside generated path-bearing text markers. */
function canonicalizeEmbeddedPaths(value: string): string {
return value
.replace(PATH_TAG_RE, (_match, open: string, path: string, close: string) =>
`${open}${path.replaceAll('\\', '/')}${close}`)
.replace(ADDITIONAL_INSTRUCTIONS_PATH_RE, (_match, prefix: string, path: string) =>
`${prefix}${path.replaceAll('\\', '/')}`)
}
/** Inputs the normalizers need to recognize a run's volatile values. */
export interface NormalizeContext {
/** The session id(s) the run issued — replaced with `{{sessionId}}`. */
@@ -34,13 +48,28 @@ export interface NormalizeContext {
cwd: string
}
/** How cwd-rooted path separators are represented after the cwd is tokenized. */
export type CwdPathMode = 'canonical' | 'native'
/** Optional controls shared by stdout and session-log normalization. */
export interface NormalizeOptions {
/** Use `/` for shared goldens, or preserve captured separators for a platform-specific golden. */
cwdPathMode?: CwdPathMode
}
/** Replace cwd, session ids, and any stray UUID with stable tokens in a string. */
function scrubString(value: string, ctx: NormalizeContext): string {
function scrubString(value: string, ctx: NormalizeContext, cwdPathMode: CwdPathMode): string {
let out = value
// cwd first (longest, most specific), then explicit session ids, then any
// residual UUID (covers ids that appear in places we didn't enumerate).
out = out.split(ctx.cwd).join(CWD)
out = out.split(`/private${CWD}`).join(CWD)
if (cwdPathMode === 'canonical') {
// Restrict separator conversion to paths rooted at the cwd token. A global
// backslash rewrite would corrupt regexes, commands, and model-authored text.
out = out.replace(CWD_ROOTED_PATH_RE, path => path.replaceAll('\\', '/'))
out = canonicalizeEmbeddedPaths(out)
}
out = out.replace(LOCAL_SPILL_PATH_RE, (_match, name: string) => `{{spillLocator:${name}}}`)
out = out.replace(SNAPSHOT_SPILL_PATH_RE, (_match, name: string) => `{{spillLocator:${name}}}`)
for (const id of ctx.sessionIds) out = out.split(id).join(SESSION_ID)
@@ -49,12 +78,15 @@ function scrubString(value: string, ctx: NormalizeContext): string {
}
/** Recursively scrub a parsed JSON value (strings replaced; structure kept). */
function scrubValue(value: unknown, ctx: NormalizeContext): unknown {
if (typeof value === 'string') return scrubString(value, ctx)
if (Array.isArray(value)) return value.map(v => scrubValue(v, ctx))
function scrubValue(value: unknown, ctx: NormalizeContext, cwdPathMode: CwdPathMode, key?: string): unknown {
if (typeof value === 'string') {
const scrubbed = scrubString(value, ctx, cwdPathMode)
return cwdPathMode === 'canonical' && key === 'path' ? scrubbed.replaceAll('\\', '/') : scrubbed
}
if (Array.isArray(value)) return value.map(v => scrubValue(v, ctx, cwdPathMode))
if (value !== null && typeof value === 'object') {
const out: Record<string, unknown> = {}
for (const [k, v] of Object.entries(value)) out[k] = scrubValue(v, ctx)
for (const [k, v] of Object.entries(value)) out[k] = scrubValue(v, ctx, cwdPathMode, k)
return out
}
return value
@@ -68,9 +100,15 @@ function scrubValue(value: unknown, ctx: NormalizeContext): unknown {
*
* @param rawStdout The captured stdout bytes, decoded utf8.
* @param ctx The run's volatile values to scrub.
* @param options Separator output controls; shared canonical paths are the default.
* @returns The normalized NDJSON transcript, one frame per line.
*/
export function normalizeStdout(rawStdout: string, ctx: NormalizeContext): string {
export function normalizeStdout(
rawStdout: string,
ctx: NormalizeContext,
options: NormalizeOptions = {},
): string {
const cwdPathMode = options.cwdPathMode ?? 'canonical'
const lines = rawStdout.split('\n').filter(line => line.trim().length > 0)
// Map each distinct JSON-RPC id (request/response correlate by id) to a stable
// sequence number, in first-seen order, so id churn doesn't perturb the expected output.
@@ -88,7 +126,7 @@ export function normalizeStdout(rawStdout: string, ctx: NormalizeContext): strin
}
const update = (frame.params as { update?: Record<string, unknown> } | undefined)?.update
if (update?.sessionUpdate === 'session_info_update') update.updatedAt = UPDATED_AT
return scrubValue(frame, ctx) as Record<string, unknown>
return scrubValue(frame, ctx, cwdPathMode) as Record<string, unknown>
})
return frames.map(f => JSON.stringify(f)).join('\n') + '\n'
}
@@ -102,9 +140,15 @@ export function normalizeStdout(rawStdout: string, ctx: NormalizeContext): strin
*
* @param rawLog The raw session `.jsonl` content.
* @param ctx The run's volatile values to scrub.
* @param options Separator output controls; shared canonical paths are the default.
* @returns The normalized JSONL log, one record per line.
*/
export function normalizeSessionLog(rawLog: string, ctx: NormalizeContext): string {
export function normalizeSessionLog(
rawLog: string,
ctx: NormalizeContext,
options: NormalizeOptions = {},
): string {
const cwdPathMode = options.cwdPathMode ?? 'canonical'
const lines = rawLog.split('\n').filter(line => line.trim().length > 0)
const records = lines.map((line) => {
const record = JSON.parse(line) as Record<string, unknown>
@@ -122,7 +166,7 @@ export function normalizeSessionLog(rawLog: string, ctx: NormalizeContext): stri
if ('durationMs' in data) data.durationMs = 0
}
}
return scrubValue(record, ctx) as Record<string, unknown>
return scrubValue(record, ctx, cwdPathMode) as Record<string, unknown>
})
return records.map(r => JSON.stringify(r)).join('\n') + '\n'
}

80
packages/support/acp-snapshot/src/suite.ts
View File

@@ -21,6 +21,7 @@ import { join } from 'node:path'
import { describe, expect, it } from 'vitest'
import { type AgentUnderTest, type HarvestedLog, type InputScript, runScenario } from './harness.ts'
import {
type CwdPathMode,
type NormalizeContext,
normalizeSessionLog,
normalizeStdout,
@@ -35,6 +36,9 @@ const SYSTEM_PROMPT_SNAPSHOT = 'system-prompt.expected.md'
/** The structured tool-schema snapshot beside each header-pinning fixture. */
const TOOL_SCHEMAS_SNAPSHOT = 'tool-schemas.expected.json'
/** The optional full Windows-native stdout transcript. */
const WINDOWS_STDOUT_SNAPSHOT = 'stdout.expected.windows.jsonl'
/** Stable session-log token standing in for the sidecar's initial schemas. */
const TOOLS_TOKEN = '{{tools}}'
@@ -100,6 +104,61 @@ export interface Scenario {
* {@link headerClass}.
*/
configPath?: string
/**
* Whether Windows additionally compares stdout with native separators against
* `stdout.expected.windows.jsonl`. The shared canonical stdout expected output is still
* compared on every platform, and the fixture guard requires this sidecar
* exactly when the option is set.
*/
pinsNativeWindowsStdout?: boolean
/**
* Whether the driven behavior needs POSIX process semantics the harness
* cannot exercise on Windows (e.g. cancelling a live bash tool call kills a
* detached process group). The scenario's run test is skipped on Windows;
* its fixtures stay guarded on every platform.
*/
posixOnly?: boolean
}
/**
* Whether a scenario's run test is skipped for this mode and host: record mode
* skips authored (non-`recorded`) scenarios, and {@link Scenario.posixOnly}
* scenarios skip on Windows.
*
* @param scenario The scenario whose run test is being registered.
* @param recording Whether the suite runs in record mode.
* @param platform The running Node platform, injectable for unit coverage.
* @returns True when the scenario's run test must not execute.
*/
export function scenarioSkipped(
scenario: Scenario,
recording: boolean,
platform: NodeJS.Platform = process.platform,
): boolean {
if (recording && !scenario.recorded) return true
return scenario.posixOnly === true && platform === 'win32'
}
/** One stdout expected output selected for a platform run. */
interface StdoutExpectedVariant {
file: string
cwdPathMode: CwdPathMode
}
/**
* Select the shared stdout expected output plus any platform-native assertion declared by a scenario.
*
* @param scenario The scenario whose stdout contract is being selected.
* @param platform The running Node platform, injectable for unit coverage.
* @returns The ordered expected-output variants: shared canonical first, then optional Windows native.
*/
export function stdoutExpectedVariants(
scenario: Scenario,
platform: NodeJS.Platform = process.platform,
): StdoutExpectedVariant[] {
const canonical: StdoutExpectedVariant = { file: 'stdout.expected.jsonl', cwdPathMode: 'canonical' }
if (platform !== 'win32' || scenario.pinsNativeWindowsStdout !== true) return [canonical]
return [canonical, { file: WINDOWS_STDOUT_SNAPSHOT, cwdPathMode: 'native' }]
}
/** One suite's inputs: the agent to boot, where its fixtures live, and its scenario table. */
@@ -479,8 +538,9 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
scenarioSuite('snapshot scenarios', () => {
for (const scenario of scenarios) {
// In RECORD mode, only re-run the `recorded` (live-API) scenarios; the `authored` ones
// (sidecar-driven errors/cancel) are never re-recorded.
it.skipIf(RECORDING && !scenario.recorded)(`snapshot: ${scenario.name} matches the expected outputs`, async ({ expect }) => {
// (sidecar-driven errors/cancel) are never re-recorded. `posixOnly` scenarios skip on
// Windows, where their process semantics cannot be driven.
it.skipIf(scenarioSkipped(scenario, RECORDING))(`snapshot: ${scenario.name} matches the expected outputs`, async ({ expect }) => {
const dir = join(snapshotsDir, scenario.name)
const input = JSON.parse(await readFile(join(dir, 'input.json'), 'utf8')) as InputScript
const overrideFile = join(dir, 'replay.override.json')
@@ -584,11 +644,13 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
}
}
const stdout = normalizeStdout(result.rawStdout, ctx)
if (REFRESHING) {
await writeFile(join(dir, 'stdout.expected.jsonl'), stdout)
for (const expected of stdoutExpectedVariants(scenario)) {
const stdout = normalizeStdout(result.rawStdout, ctx, { cwdPathMode: expected.cwdPathMode })
if (REFRESHING) {
await writeFile(join(dir, expected.file), stdout)
}
await expect(stdout, `${expected.file} mismatch`).toMatchFileSnapshot(join(dir, expected.file))
}
await expect(stdout).toMatchFileSnapshot(join(dir, 'stdout.expected.jsonl'))
// A model turn always produces a log worth comparing; a hook scenario can
// produce one without a model turn (a `rejected` turn carrying `hook/*`).
@@ -675,10 +737,14 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
it('every registered scenario has its required fixture files', async () => {
// Every scenario needs input, stdout, a primary session fixture, and matching optional sidecars.
for (const { name, overridden, pinsHeader } of scenarios) {
for (const { name, overridden, pinsHeader, pinsNativeWindowsStdout } of scenarios) {
const dir = join(snapshotsDir, name)
expect(existsSync(join(dir, 'input.json')), `${name}/input.json`).toBe(true)
expect(existsSync(join(dir, 'stdout.expected.jsonl')), `${name}/stdout.expected.jsonl`).toBe(true)
expect(
existsSync(join(dir, WINDOWS_STDOUT_SNAPSHOT)),
`${name}/${WINDOWS_STDOUT_SNAPSHOT} presence must match \`pinsNativeWindowsStdout\``,
).toBe(pinsNativeWindowsStdout === true)
expect(existsSync(join(dir, 'session.jsonl')), `${name}/session.jsonl`).toBe(true)
expect(existsSync(join(dir, 'replay.override.json')), `${name}/replay.override.json presence must match \`overridden\``)
.toBe(overridden === true)