Coder/deepseek-harness
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: share the ACP test launcher

Browse Source
This commit is contained in:
Tianyi Cui
2026-07-14 00:18:00 +08:00
parent e481288a3a
commit 0e7d539bbc
11 changed files with 292 additions and 397 deletions
Download Patch File Download Diff File Expand all files Collapse all files

182
examples/acp-agent/tests/acp.e2e.ts
View File

@@ -1,20 +1,14 @@
import { spawn, type ChildProcessWithoutNullStreams } from 'node:child_process'
import { Readable, Writable } from 'node:stream'
import { mkdtemp, rm, readFile } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { fileURLToPath } from 'node:url'
import { afterEach, describe, expect, it } from 'vitest'
import { PROTOCOL_VERSION } from '@agentclientprotocol/sdk'
import {
ClientSideConnection,
ndJsonStream,
PROTOCOL_VERSION,
type Agent as AcpAgent,
type Client,
type RequestPermissionRequest,
type RequestPermissionResponse,
type SessionNotification,
} from '@agentclientprotocol/sdk'
launchAcpTestAgent,
type AgentUnderTest,
type LaunchedAcpTestAgent,
} from '@deepseek-ai/dsh-acp-snapshot'
/**
* End-to-end: boot examples/acp-agent as a real subprocess speaking ACP over
@@ -26,127 +20,18 @@ import {
* WITHOUT a key, since it only needs the server to boot and answer initialize.
*/
// The dsh-acp-agent bin (the demo:acp entry) and this example's cordis.yml. The
// bin resolves its config-path arg from CWD; the subprocess runs from a temp
// workdir, so pass the example config's ABSOLUTE path.
const binScript = fileURLToPath(new URL('../../../packages/ui/acp-agent/src/bin.ts', import.meta.url))
const configPath = fileURLToPath(new URL('../cordis.yml', import.meta.url))
// Resolve tsx's loader to an ABSOLUTE path: the subprocess runs with cwd set to
// a temp workdir (this test launches there and uses it as the session cwd; the
// bridge no longer requires cwd === the launch dir, but a temp dir keeps the
// test hermetic), where a bare `--import tsx` would not resolve from
// node_modules. import.meta.resolve gives the worktree's tsx regardless of cwd.
const tsxLoader = fileURLToPath(import.meta.resolve('tsx'))
// Absolute path to the repo-root tsconfig. Dev/test/demo run UNBUILT: the
// `@deepseek-ai/dsh-*` workspace imports resolve through the `paths` map in the
// root tsconfig (tsx reads it), NOT through built `lib/` output. But tsx finds
// that tsconfig by searching UP from the child's cwd — and the child's cwd is a
// temp workdir OUTSIDE the repo, so the search misses and the dsh-* imports fail
// (the child dies before writing a byte). Point tsx at the repo tsconfig
// explicitly via TSX_TSCONFIG_PATH so resolution is cwd-independent. (Without
// this the suite only passed by accident when a stale built `lib/` happened to
// exist — exactly the contamination that masked the inject bug this suite now
// guards.) The repo root is four levels up from this file (examples/acp-agent/tests).
const repoTsconfig = fileURLToPath(new URL('../../../tsconfig.json', import.meta.url))
interface Spawned {
child: ChildProcessWithoutNullStreams
client: ClientSideConnection
updates: SessionNotification['update'][]
stderr: string[]
const AGENT: AgentUnderTest = {
binScript: fileURLToPath(new URL('../../../packages/ui/acp-agent/src/bin.ts', import.meta.url)),
configPath: fileURLToPath(new URL('../cordis.yml', import.meta.url)),
tsconfigPath: fileURLToPath(new URL('../../../tsconfig.json', import.meta.url)),
}
// TODO(acp-test-harness): this subprocess/client boot glue is duplicated with
// hooks.e2e.ts and partly with dsh-acp-snapshot's harness. Migrate both e2e
// files onto that launcher before the TSX/env/permission-stub details drift.
function spawnAcpAgent(cwd: string, env: NodeJS.ProcessEnv = process.env): Spawned {
const child = spawn(
process.execPath,
['--import', tsxLoader, binScript, configPath],
{
cwd,
env: {
...env,
TSX_TSCONFIG_PATH: repoTsconfig,
DSH_HOME: join(cwd, '.dsh'),
DSH_AGENTS_HOME: join(cwd, '.agents'),
},
stdio: ['pipe', 'pipe', 'pipe'],
},
)
const stderr: string[] = []
child.stderr.setEncoding('utf8')
child.stderr.on('data', (chunk: string) => stderr.push(chunk))
const updates: SessionNotification['update'][] = []
const stream = ndJsonStream(
Writable.toWeb(child.stdin) as WritableStream<Uint8Array>,
Readable.toWeb(child.stdout) as ReadableStream<Uint8Array>,
)
const makeClient = (_agent: AcpAgent): Client => ({
sessionUpdate(params: SessionNotification): Promise<void> {
updates.push(params.update)
return Promise.resolve()
},
requestPermission(_params: RequestPermissionRequest): Promise<RequestPermissionResponse> {
// This example composes no ask-producing policy (no hooks), so the
// bridge never prompts here; answer cancelled (fail closed) if it ever
// does — an unexpected prompt must not grant anything.
return Promise.resolve({ outcome: { outcome: 'cancelled' } })
},
})
const client = new ClientSideConnection(makeClient, stream)
return { child, client, updates, stderr }
}
let spawned: Spawned | undefined
let spawned: LaunchedAcpTestAgent | undefined
let workdir: string | undefined
function hasStdoutLine(out: string[]): boolean {
return out.join('').split('\n').some(line => line.trim().length > 0)
}
async function waitForStdoutLine(child: ChildProcessWithoutNullStreams, out: string[], stderr: string[], timeoutMs: number): Promise<void> {
await new Promise<void>((resolve, reject) => {
const cleanup = () => {
clearTimeout(timeout)
child.stdout.off('data', onData)
child.off('exit', onExit)
child.off('error', onError)
}
const pass = () => {
cleanup()
resolve()
}
const fail = (reason: string) => {
cleanup()
reject(new Error(`${reason}; stderr: ${stderr.join('')}`))
}
const onData = () => {
if (hasStdoutLine(out)) pass()
}
const onExit = (code: number | null, signal: NodeJS.Signals | null) => {
fail(`ACP child exited before emitting a stdout frame (code ${code ?? 'null'}, signal ${signal ?? 'null'})`)
}
const onError = (error: Error) => {
fail(`ACP child failed before emitting a stdout frame: ${error.message}`)
}
const timeout = setTimeout(() => {
fail(`ACP child did not emit a stdout frame within ${timeoutMs}ms`)
}, timeoutMs)
child.stdout.on('data', onData)
child.on('exit', onExit)
child.on('error', onError)
onData()
})
}
afterEach(async () => {
if (spawned) {
spawned.child.kill('SIGKILL')
spawned = undefined
}
await spawned?.close('SIGKILL')
spawned = undefined
if (workdir !== undefined) await rm(workdir, { recursive: true, force: true })
workdir = undefined
})
@@ -154,39 +39,18 @@ afterEach(async () => {
describe('acp-agent over real stdio (no key required)', () => {
it('emits only framed JSON-RPC on stdout', async () => {
workdir = await mkdtemp(join(tmpdir(), 'acp-e2e-'))
// Collect raw stdout bytes directly (bypass the SDK framing) to inspect.
// Inspect the launcher's raw-byte tee in addition to driving its SDK client.
// A dummy key lets the deepseek adapter APPLY (it only checks the key is
// present at boot, not valid — the key is used only on a real model call,
// which this purity test never triggers). So this runs WITHOUT real creds.
const child = spawn(process.execPath, ['--import', tsxLoader, binScript, configPath], {
spawned = launchAcpTestAgent({
agent: AGENT,
cwd: workdir,
env: {
...process.env,
DEEPSEEK_API_KEY: process.env.DEEPSEEK_API_KEY ?? 'sk-dummy-for-boot',
TSX_TSCONFIG_PATH: repoTsconfig,
DSH_HOME: join(workdir, '.dsh'),
DSH_AGENTS_HOME: join(workdir, '.agents'),
},
stdio: ['pipe', 'pipe', 'pipe'],
env: { DEEPSEEK_API_KEY: process.env.DEEPSEEK_API_KEY ?? 'sk-dummy-for-boot' },
})
const out: string[] = []
const stderr: string[] = []
child.stdout.setEncoding('utf8')
child.stderr.setEncoding('utf8')
child.stdout.on('data', (c: string) => out.push(c))
child.stderr.on('data', (c: string) => stderr.push(c))
await spawned.client.initialize({ protocolVersion: PROTOCOL_VERSION, clientCapabilities: {} })
// Send a single initialize request as a newline-delimited JSON-RPC frame.
const req = JSON.stringify({ jsonrpc: '2.0', id: 1, method: 'initialize', params: { protocolVersion: PROTOCOL_VERSION, clientCapabilities: {} } })
child.stdin.write(req + '\n')
try {
await waitForStdoutLine(child, out, stderr, 15_000)
} finally {
child.kill('SIGKILL')
}
const lines = out.join('').split('\n').filter(l => l.trim().length > 0)
const lines = spawned.rawStdout().split('\n').filter(line => line.trim().length > 0)
expect(lines.length).toBeGreaterThan(0)
for (const line of lines) {
// Every stdout line MUST parse as JSON (a JSON-RPC frame). A non-JSON
@@ -210,7 +74,11 @@ describe('acp-agent over real stdio (no key required)', () => {
workdir = await mkdtemp(join(tmpdir(), 'acp-e2e-'))
// A dummy key lets the deepseek adapter boot (it only checks presence, not
// validity, at apply time); no model call is made, so the key is never used.
spawned = spawnAcpAgent(workdir, { ...process.env, DEEPSEEK_API_KEY: process.env.DEEPSEEK_API_KEY ?? 'sk-dummy-for-boot' })
spawned = launchAcpTestAgent({
agent: AGENT,
cwd: workdir,
env: { DEEPSEEK_API_KEY: process.env.DEEPSEEK_API_KEY ?? 'sk-dummy-for-boot' },
})
const { client } = spawned
await client.initialize({ protocolVersion: PROTOCOL_VERSION, clientCapabilities: {} })
@@ -223,7 +91,7 @@ describe('acp-agent over real stdio (no key required)', () => {
describe.skipIf(!process.env.DEEPSEEK_API_KEY)('acp-agent e2e: real prompt over ACP', () => {
it('runs a real turn and the agent writes the requested file (verified on disk)', async () => {
workdir = await mkdtemp(join(tmpdir(), 'acp-e2e-'))
spawned = spawnAcpAgent(workdir)
spawned = launchAcpTestAgent({ agent: AGENT, cwd: workdir })
const { client, updates } = spawned
await client.initialize({ protocolVersion: PROTOCOL_VERSION, clientCapabilities: {} })
@@ -264,7 +132,7 @@ describe.skipIf(!process.env.DEEPSEEK_API_KEY)('acp-agent e2e: real prompt over
it('with the terminal_output capability, a real bash call renders as a terminal card (content + _meta + exit)', async () => {
workdir = await mkdtemp(join(tmpdir(), 'acp-e2e-'))
spawned = spawnAcpAgent(workdir)
spawned = launchAcpTestAgent({ agent: AGENT, cwd: workdir })
const { client, updates } = spawned
// Advertise the Zed `_meta.terminal_output` capability so the bridge emits

68
examples/acp-agent/tests/hooks.e2e.ts
View File

@@ -1,20 +1,14 @@
import { spawn, type ChildProcessWithoutNullStreams } from 'node:child_process'
import { Readable, Writable } from 'node:stream'
import { mkdtemp, rm, writeFile, access } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { fileURLToPath } from 'node:url'
import { afterEach, describe, expect, it } from 'vitest'
import { PROTOCOL_VERSION } from '@agentclientprotocol/sdk'
import {
ClientSideConnection,
ndJsonStream,
PROTOCOL_VERSION,
type Agent as AcpAgent,
type Client,
type RequestPermissionRequest,
type RequestPermissionResponse,
type SessionNotification,
} from '@agentclientprotocol/sdk'
launchAcpTestAgent,
type AgentUnderTest,
type LaunchedAcpTestAgent,
} from '@deepseek-ai/dsh-acp-snapshot'
/**
* With-key e2e: the Claude Code hook bridge running against the REAL acp-agent
@@ -34,54 +28,18 @@ import {
* only a real model deciding to call bash exercises the PreToolUse seam live.
*/
const binScript = fileURLToPath(new URL('../../../packages/ui/acp-agent/src/bin.ts', import.meta.url))
const configPath = fileURLToPath(new URL('../cordis.yml', import.meta.url))
const tsxLoader = fileURLToPath(import.meta.resolve('tsx'))
const repoTsconfig = fileURLToPath(new URL('../../../tsconfig.json', import.meta.url))
interface Spawned {
child: ChildProcessWithoutNullStreams
client: ClientSideConnection
updates: SessionNotification['update'][]
stderr: string[]
const AGENT: AgentUnderTest = {
binScript: fileURLToPath(new URL('../../../packages/ui/acp-agent/src/bin.ts', import.meta.url)),
configPath: fileURLToPath(new URL('../cordis.yml', import.meta.url)),
tsconfigPath: fileURLToPath(new URL('../../../tsconfig.json', import.meta.url)),
}
function spawnAcpAgent(cwd: string): Spawned {
const child = spawn(
process.execPath,
['--import', tsxLoader, binScript, configPath],
{ cwd, env: { ...process.env, TSX_TSCONFIG_PATH: repoTsconfig }, stdio: ['pipe', 'pipe', 'pipe'] },
)
const stderr: string[] = []
child.stderr.setEncoding('utf8')
child.stderr.on('data', (chunk: string) => stderr.push(chunk))
const updates: SessionNotification['update'][] = []
const stream = ndJsonStream(
Writable.toWeb(child.stdin) as WritableStream<Uint8Array>,
Readable.toWeb(child.stdout) as ReadableStream<Uint8Array>,
)
const makeClient = (_agent: AcpAgent): Client => ({
sessionUpdate(params: SessionNotification): Promise<void> {
updates.push(params.update)
return Promise.resolve()
},
requestPermission(_params: RequestPermissionRequest): Promise<RequestPermissionResponse> {
return Promise.resolve({ outcome: { outcome: 'cancelled' } })
},
})
const client = new ClientSideConnection(makeClient, stream)
return { child, client, updates, stderr }
}
let spawned: Spawned | undefined
let spawned: LaunchedAcpTestAgent | undefined
let workdir: string | undefined
afterEach(async () => {
if (spawned) {
spawned.child.kill('SIGKILL')
spawned = undefined
}
await spawned?.close('SIGKILL')
spawned = undefined
if (workdir !== undefined) await rm(workdir, { recursive: true, force: true })
workdir = undefined
})
@@ -96,7 +54,7 @@ describe.skipIf(!process.env.DEEPSEEK_API_KEY)('acp-agent e2e: a PreToolUse hook
hooks: { PreToolUse: [{ hooks: [{ type: 'command', command: 'echo "bash blocked by policy" >&2; exit 2' }] }] },
}))
spawned = spawnAcpAgent(workdir)
spawned = launchAcpTestAgent({ agent: AGENT, cwd: workdir })
const { client, updates } = spawned
await client.initialize({ protocolVersion: PROTOCOL_VERSION, clientCapabilities: {} })

79
examples/sandbox-acp-agent/tests/escalation.e2e.ts
View File

@@ -1,20 +1,18 @@
import { spawn, spawnSync, type ChildProcessWithoutNullStreams } from 'node:child_process'
import { Readable, Writable } from 'node:stream'
import { spawnSync } from 'node:child_process'
import { mkdtemp, readFile, rm } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { fileURLToPath } from 'node:url'
import { afterEach, describe, expect, it } from 'vitest'
import {
ClientSideConnection,
ndJsonStream,
PROTOCOL_VERSION,
type Agent as AcpAgent,
type Client,
type RequestPermissionRequest,
type RequestPermissionResponse,
type SessionNotification,
} from '@agentclientprotocol/sdk'
import {
launchAcpTestAgent,
type AgentUnderTest,
type LaunchedAcpTestAgent,
} from '@deepseek-ai/dsh-acp-snapshot'
/**
* examples/sandbox-acp-agent end to end.
@@ -35,12 +33,11 @@ import {
* escalation target the model picks can land the write.
*/
const binScript = fileURLToPath(new URL('../../../packages/ui/acp-agent/src/bin.ts', import.meta.url))
const configPath = fileURLToPath(new URL('../cordis.yml', import.meta.url))
const tsxLoader = fileURLToPath(import.meta.resolve('tsx'))
// The subprocess runs from a temp cwd OUTSIDE the repo; point tsx at the repo
// tsconfig so the unbuilt `paths` map resolves (see examples/AGENTS.md).
const repoTsconfig = fileURLToPath(new URL('../../../tsconfig.json', import.meta.url))
const AGENT: AgentUnderTest = {
binScript: fileURLToPath(new URL('../../../packages/ui/acp-agent/src/bin.ts', import.meta.url)),
configPath: fileURLToPath(new URL('../cordis.yml', import.meta.url)),
tsconfigPath: fileURLToPath(new URL('../../../tsconfig.json', import.meta.url)),
}
// A usable confining runner, probed the same way the executor suites do:
// bwrap on Linux, Seatbelt's sandbox-exec on macOS. Without one the strict
@@ -56,44 +53,19 @@ const hasSeatbelt = process.platform === 'darwin' && spawnSync('sandbox-exec', [
}).status === 0
const hasRunner = hasBwrap || hasSeatbelt
interface Spawned {
child: ChildProcessWithoutNullStreams
client: ClientSideConnection
updates: SessionNotification['update'][]
interface Spawned extends LaunchedAcpTestAgent {
permissionRequests: RequestPermissionRequest[]
stderr: string[]
}
/** Boot the example as an ACP subprocess; the scripted client answers every permission prompt with `answer`. */
function spawnSandboxAcpAgent(cwd: string, answer: 'allow-once' | 'reject-once'): Spawned {
const child = spawn(
process.execPath,
['--import', tsxLoader, binScript, configPath],
{
cwd,
// A dummy key lets the deepseek adapter boot keyless (presence-checked at
// apply, used only on a real model call); the with-key tests carry the
// real key, so the fallback is inert there.
env: { ...process.env, DEEPSEEK_API_KEY: process.env.DEEPSEEK_API_KEY ?? 'sk-dummy-for-boot', TSX_TSCONFIG_PATH: repoTsconfig },
stdio: ['pipe', 'pipe', 'pipe'],
},
)
const stderr: string[] = []
child.stderr.setEncoding('utf8')
child.stderr.on('data', (chunk: string) => stderr.push(chunk))
const updates: SessionNotification['update'][] = []
function launchSandboxAcpAgent(cwd: string, answer: 'allow-once' | 'reject-once'): Spawned {
const permissionRequests: RequestPermissionRequest[] = []
const stream = ndJsonStream(
Writable.toWeb(child.stdin) as WritableStream<Uint8Array>,
Readable.toWeb(child.stdout) as ReadableStream<Uint8Array>,
)
const makeClient = (_agent: AcpAgent): Client => ({
sessionUpdate(params: SessionNotification): Promise<void> {
updates.push(params.update)
return Promise.resolve()
},
requestPermission(params: RequestPermissionRequest): Promise<RequestPermissionResponse> {
const launched = launchAcpTestAgent({
agent: AGENT,
cwd,
// A dummy key lets the adapter boot keylessly; live tests carry the real key.
env: { DEEPSEEK_API_KEY: process.env.DEEPSEEK_API_KEY ?? 'sk-dummy-for-boot' },
requestPermission(params) {
permissionRequests.push(params)
const option = params.options.find(o => o.optionId === answer)
// The scripted human: pick the requested option when the prompt offers
@@ -102,15 +74,14 @@ function spawnSandboxAcpAgent(cwd: string, answer: 'allow-once' | 'reject-once')
return Promise.resolve({ outcome: { outcome: 'selected', optionId: option.optionId } })
},
})
const client = new ClientSideConnection(makeClient, stream)
return { child, client, updates, permissionRequests, stderr }
return Object.assign(launched, { permissionRequests })
}
let spawned: Spawned | undefined
let workdir: string | undefined
afterEach(async () => {
if (spawned !== undefined && spawned.child.exitCode === null) spawned.child.kill('SIGKILL')
await spawned?.close('SIGKILL')
spawned = undefined
if (workdir !== undefined) await rm(workdir, { recursive: true, force: true })
workdir = undefined
@@ -119,7 +90,7 @@ afterEach(async () => {
describe('sandbox-acp-agent keyless smoke (real cordis.yml via the Loader)', () => {
it('boots the tree — sandbox executor + approval service + bridge — and opens a session', async () => {
workdir = await mkdtemp(join(tmpdir(), 'sandbox-acp-smoke-'))
spawned = spawnSandboxAcpAgent(workdir, 'reject-once')
spawned = launchSandboxAcpAgent(workdir, 'reject-once')
const { client } = spawned
// A dummy key boots the adapter; no prompt is ever sent, so no model call
// and no sandbox runner probe happen. This drives the fiber tree the same
@@ -132,7 +103,7 @@ describe('sandbox-acp-agent keyless smoke (real cordis.yml via the Loader)', ()
it('advertises both session config options and honors a switch end to end (no key, no model)', async () => {
workdir = await mkdtemp(join(tmpdir(), 'sandbox-acp-config-'))
spawned = spawnSandboxAcpAgent(workdir, 'reject-once')
spawned = launchSandboxAcpAgent(workdir, 'reject-once')
const { client } = spawned
await client.initialize({ protocolVersion: PROTOCOL_VERSION, clientCapabilities: {} })
// This tree composes bash-sandbox (mode: read-only) + approval → both
@@ -164,7 +135,7 @@ describe('sandbox-acp-agent keyless smoke (real cordis.yml via the Loader)', ()
describe.skipIf(!process.env.DEEPSEEK_API_KEY || !hasRunner)('sandbox-acp-agent e2e: the live approval loop', () => {
it('denial → model escalation → editor prompt → allow-once → the retried write lands on disk', async () => {
workdir = await mkdtemp(join(tmpdir(), 'sandbox-acp-e2e-'))
spawned = spawnSandboxAcpAgent(workdir, 'allow-once')
spawned = launchSandboxAcpAgent(workdir, 'allow-once')
const { client, permissionRequests } = spawned
await client.initialize({ protocolVersion: PROTOCOL_VERSION, clientCapabilities: {} })
@@ -193,7 +164,7 @@ describe.skipIf(!process.env.DEEPSEEK_API_KEY || !hasRunner)('sandbox-acp-agent
it('a rejected escalation stays denied: no write lands, the turn still ends', async () => {
workdir = await mkdtemp(join(tmpdir(), 'sandbox-acp-e2e-'))
spawned = spawnSandboxAcpAgent(workdir, 'reject-once')
spawned = launchSandboxAcpAgent(workdir, 'reject-once')
const { client, permissionRequests } = spawned
await client.initialize({ protocolVersion: PROTOCOL_VERSION, clientCapabilities: {} })