Coder/deepseek-harness
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(mode): stabilize plan-mode model experience

Browse Source
This commit is contained in:
Tianyi Cui
2026-07-20 22:13:59 +08:00
parent 28a74d6d7e
commit 897dc82d9b
40 changed files with 620 additions and 298 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
examples/plan-acp-agent/tests/acp.snapshot.ts
View File

@@ -45,13 +45,10 @@ const SCENARIOS: Scenario[] = [
// class to carry a pin.
{ name: 'modes-advertise', hasModelTurn: false, recorded: false, headerClass: 'plan' },
// The full plan-mode arc, and NECESSARILY the pinned-header scenario for
// the 'plan' class: the first request ships the plan-shaped header (reason
// initial) — the full toolset plus exit_plan_mode and the mode section —
// and the approved exit narrows it back by exactly that tool and section,
// a pure removal the delta encoding CAN express (one header-delta; the
// ENTERING flip is a non-tail insertion the append-only tools delta cannot
// express, so it falls back to a snapshot — pinned at the unit tier). The
// arc: setMode(plan) → the model runs a real `cat` inside plan and
// the 'plan' class: the first request ships the configured mode section and
// the full toolset, including exit_plan_mode. Approval removes only the
// section; the following changed header carries byte-identical tool schemas.
// The arc: setMode(plan) → the model runs a real `cat` inside plan and
// presents the plan via exit_plan_mode → the scripted elicitation approves
// → the very next step already edits for real, mid-turn.
{ name: 'plan-mode', hasModelTurn: true, recorded: true, pinsHeader: true, headerClass: 'plan', expectedHeaderChanges: 1 },

84
examples/plan-acp-agent/tests/plan-mode.e2e.ts Normal file
View File

@@ -0,0 +1,84 @@
import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { fileURLToPath } from 'node:url'
import { afterEach, describe, expect, it } from 'vitest'
import {
PROTOCOL_VERSION,
type CreateElicitationRequest,
type CreateElicitationResponse,
} from '@agentclientprotocol/sdk'
import {
launchAcpTestAgent,
type AgentUnderTest,
type LaunchedAcpTestAgent,
} from '@deepseek-ai/dsh-acp-snapshot'
/** The shipped plan-mode ACP leaf exercised through its real subprocess entry. */
const AGENT: AgentUnderTest = {
binScript: fileURLToPath(new URL('../../../packages/examples/acp-demo/src/bin.ts', import.meta.url)),
configPath: fileURLToPath(new URL('../cordis.yml', import.meta.url)),
tsconfigPath: fileURLToPath(new URL('../../../tsconfig.json', import.meta.url)),
}
let spawned: LaunchedAcpTestAgent | undefined
let workdir: string | undefined
afterEach(async () => {
const ownedSpawned = spawned
const ownedWorkdir = workdir
spawned = undefined
workdir = undefined
try {
if (ownedSpawned !== undefined) {
await ownedSpawned.close('SIGKILL').catch((error: unknown) => {
throw new Error(`plan ACP cleanup failed; child stderr:\n${ownedSpawned.stderr()}`, { cause: error })
})
}
} finally {
if (ownedWorkdir !== undefined) await rm(ownedWorkdir, { recursive: true, force: true })
}
})
describe.skipIf(!process.env.DEEPSEEK_API_KEY)('plan-acp-agent e2e: approval gates implementation (real model)', () => {
it('keeps the file unchanged through review, then applies the approved plan', async () => {
workdir = await mkdtemp(join(tmpdir(), 'plan-acp-e2e-'))
const proofPath = join(workdir, 'proof.txt')
await writeFile(proofPath, 'BEFORE\n')
const reviews: CreateElicitationRequest[] = []
let contentAtReview: string | undefined
const createElicitation = async (request: CreateElicitationRequest): Promise<CreateElicitationResponse> => {
if (request.mode !== 'form' || request.requestedSchema.title !== 'Plan review') return { action: 'cancel' }
reviews.push(request)
contentAtReview = await readFile(proofPath, 'utf8')
return { action: 'accept', content: { choice: 'Approve' } }
}
spawned = launchAcpTestAgent({ agent: AGENT, cwd: workdir, createElicitation })
const { client, updates } = spawned
const rpc = async <T>(stage: string, operation: Promise<T>): Promise<T> => operation.catch((error: unknown) => {
throw new Error(`plan ACP ${stage} failed; child stderr:\n${spawned?.stderr() ?? '<unavailable>'}`, { cause: error })
})
await rpc('initialize', client.initialize({ protocolVersion: PROTOCOL_VERSION, clientCapabilities: {} }))
const created = await rpc('session/new', client.newSession({ cwd: workdir, mcpServers: [] }))
expect(created.modes?.availableModes.map(mode => mode.id)).toEqual(['default', 'plan'])
await rpc('session/set_mode', client.setSessionMode({ sessionId: created.sessionId, modeId: 'plan' }))
const result = await rpc('prompt', client.prompt({
sessionId: created.sessionId,
prompt: [{
type: 'text',
text: 'Inspect proof.txt and plan the smallest change that replaces its contents with exactly AFTER followed by one newline. Present the complete plan through exit_plan_mode. After I approve it, implement the change with the filesystem tools, verify the exact file contents, and stop. Do not ask questions.',
}],
}))
expect(['end_turn', 'max_tokens']).toContain(result.stopReason)
expect(reviews).toHaveLength(1)
expect(contentAtReview).toBe('BEFORE\n')
expect(await readFile(proofPath, 'utf8')).toBe('AFTER\n')
expect(updates
.filter(update => update.sessionUpdate === 'current_mode_update')
.map(update => update.currentModeId)).toEqual(['plan', 'default'])
}, 240_000)
})

2
examples/plan-acp-agent/tests/snapshots/plan-mode-reject/stdout.expected.jsonl
View File

@@ -82,7 +82,7 @@
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":"o"}}}}
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":"."}}}}
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"tool_call","toolCallId":"call_00_ucsiIH3qJOCb1B6d3dmm4387","title":"Fix the greeting typo","kind":"other","status":"in_progress","content":[{"type":"content","content":{"type":"text","text":"# Fix the greeting typo\n\n**Issue:** The greeting message currently displays \"helo wrld\" instead of the correct \"Hello World\".\n\n**Plan:**\n\n1. **Find the source file** — Locate the file that contains the greeting string \"helo wrld\".\n2. **Fix the typo** — Replace \"helo wrld\" with \"Hello World\" in that source file.\n3. **Verify** — Confirm the fix is correct by reading the changed file."}}]}}}
{"jsonrpc":"2.0","id":1,"method":"elicitation/create","params":{"sessionId":"{{sessionId}}","mode":"form","message":"Approve this plan and leave plan mode?","requestedSchema":{"type":"object","title":"Plan review","properties":{"choice":{"type":"string","title":"Approve this plan and leave plan mode?","description":"Choose one option, or fill a custom answer below.","oneOf":[{"const":"Approve","title":"Approve: Leave plan mode; the plan is carried out from the next step."},{"const":"Keep planning","title":"Keep planning: Stay in plan mode; feedback goes back to the model."}]},"custom":{"type":"string","title":"Custom answer","description":"Optional free-form answer. Leave empty to use the selected option."}},"required":[]}}}
{"jsonrpc":"2.0","id":1,"method":"elicitation/create","params":{"sessionId":"{{sessionId}}","mode":"form","message":"Approve this plan and leave plan mode?\n\n# Fix the greeting typo\n\n**Issue:** The greeting message currently displays \"helo wrld\" instead of the correct \"Hello World\".\n\n**Plan:**\n\n1. **Find the source file** — Locate the file that contains the greeting string \"helo wrld\".\n2. **Fix the typo** — Replace \"helo wrld\" with \"Hello World\" in that source file.\n3. **Verify** — Confirm the fix is correct by reading the changed file.","requestedSchema":{"type":"object","title":"Plan review","properties":{"choice":{"type":"string","title":"Approve this plan and leave plan mode?","description":"Choose one option, or fill a custom answer below.","oneOf":[{"const":"Approve","title":"Approve: Leave plan mode; the plan is carried out from the next step."},{"const":"Keep planning","title":"Keep planning: Stay in plan mode; feedback goes back to the model."}]},"custom":{"type":"string","title":"Custom answer","description":"Optional free-form answer. Leave empty to use the selected option."}},"required":[]}}}
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"tool_call_update","toolCallId":"call_00_ucsiIH3qJOCb1B6d3dmm4387","status":"failed","content":[{"type":"content","content":{"type":"text","text":"Error: The user chose to keep planning; their feedback: Also add a verification step that re-reads the file after the fix."}}],"title":"Plan review"}}}
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":"The"}}}}
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" user"}}}}

2
examples/plan-acp-agent/tests/snapshots/plan-mode/session.jsonl
View File

@@ -229,7 +229,7 @@
{"type":"step/end","seq":227,"time":1784525378329,"data":{"turn":1,"step":2}}
{"type":"mode/set","seq":228,"time":1784525378329,"data":{"mode":"default"}}
{"type":"step/start","seq":229,"time":1784525378330,"data":{"turn":1,"step":3}}
{"type":"request/header","seq":230,"time":1784560735611,"data":{"header":{"config":{"provider":"deepseek","model":"deepseek-v4-flash"},"system":"{{system}}","tools":"{{tools}}"},"reason":"change"}}
{"type":"request/header","seq":230,"time":1784553020470,"data":{"header":{"config":{"provider":"deepseek","model":"deepseek-v4-flash"},"system":"{{system}}","tools":"{{tools}}"},"reason":"change"}}
{"type":"assistant/chunk","seq":231,"time":1784525378723,"data":{"turn":1,"step":3,"chunk":{"type":"block-start","index":0,"blockType":"reasoning"}}}
{"type":"assistant/chunk","seq":232,"time":1784525378723,"data":{"turn":1,"step":3,"chunk":{"type":"reasoning-delta","index":0,"text":"The"}}}
{"type":"assistant/chunk","seq":233,"time":1784525378807,"data":{"turn":1,"step":3,"chunk":{"type":"reasoning-delta","index":0,"text":" plan"}}}

2
examples/plan-acp-agent/tests/snapshots/plan-mode/stdout.expected.jsonl
View File

@@ -122,7 +122,7 @@
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" plan"}}}}
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":"."}}}}
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"tool_call","toolCallId":"call_00_6RLxuiGtAFswvfLnWdt63143","title":"Fix the greeting typo","kind":"other","status":"in_progress","content":[{"type":"content","content":{"type":"text","text":"# Fix the greeting typo\n\n## Single step\n1. **Edit line 2 of `notes.txt`** — Replace the content of line 2 (`- the greeting message still says \"helo wrld\"`) with `hello world`."}}]}}}
{"jsonrpc":"2.0","id":1,"method":"elicitation/create","params":{"sessionId":"{{sessionId}}","mode":"form","message":"Approve this plan and leave plan mode?","requestedSchema":{"type":"object","title":"Plan review","properties":{"choice":{"type":"string","title":"Approve this plan and leave plan mode?","description":"Choose one option, or fill a custom answer below.","oneOf":[{"const":"Approve","title":"Approve: Leave plan mode; the plan is carried out from the next step."},{"const":"Keep planning","title":"Keep planning: Stay in plan mode; feedback goes back to the model."}]},"custom":{"type":"string","title":"Custom answer","description":"Optional free-form answer. Leave empty to use the selected option."}},"required":[]}}}
{"jsonrpc":"2.0","id":1,"method":"elicitation/create","params":{"sessionId":"{{sessionId}}","mode":"form","message":"Approve this plan and leave plan mode?\n\n# Fix the greeting typo\n\n## Single step\n1. **Edit line 2 of `notes.txt`** — Replace the content of line 2 (`- the greeting message still says \"helo wrld\"`) with `hello world`.","requestedSchema":{"type":"object","title":"Plan review","properties":{"choice":{"type":"string","title":"Approve this plan and leave plan mode?","description":"Choose one option, or fill a custom answer below.","oneOf":[{"const":"Approve","title":"Approve: Leave plan mode; the plan is carried out from the next step."},{"const":"Keep planning","title":"Keep planning: Stay in plan mode; feedback goes back to the model."}]},"custom":{"type":"string","title":"Custom answer","description":"Optional free-form answer. Leave empty to use the selected option."}},"required":[]}}}
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"tool_call_update","toolCallId":"call_00_6RLxuiGtAFswvfLnWdt63143","status":"completed","content":[{"type":"content","content":{"type":"text","text":"Plan approved — plan mode exited; carry out the plan starting with your next step."}}],"title":"Plan review"}}}
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"current_mode_update","currentModeId":"default"}}}
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":"The"}}}}

13
examples/plan-acp-agent/tests/snapshots/plan-mode/system-prompt.expected.md
View File

@@ -5,7 +5,18 @@ You are a coding assistant powered by the deepseek-v4-flash model. Your working
Verify your work by running the code or tests. Keep answers brief and factual.
You are in plan mode: a planning state. Explore, analyze, and design; reading files and running read-only commands is fine, but hold off on changes — edits and other side effects belong in the plan and run after its approval, not in this mode. When a decision or a missing detail blocks the plan, ask the user through the ask_user_question tool where it is available. A finished plan is delivered by calling exit_plan_mode — that call is what puts it in front of the user for review, so prefer it over pasting the plan as a plain reply or asking the user to switch modes themselves. If exit_plan_mode is unavailable or its review fails, ask the user to switch the session out of plan mode instead of pressing on.
You are in plan mode. Stay in plan mode until exit_plan_mode succeeds or the user switches the session mode. Imperative language to implement changes means plan the implementation, not execute it.
Explore first. Use non-mutating reads, searches, static analysis, and checks to ground the plan in the actual repository. Do not edit or write files, change configuration, run formatters or code generation that rewrites tracked files, commit, or otherwise carry out the plan. Prefer existing functions and patterns over new machinery.
The tool catalog stays the same across modes for request-cache stability. These plan-mode rules override any later tool description or guidance that suggests using mutation tools; those tools remain listed only to keep the request shape stable. Do not use todo_write to track this planning phase: it tracks implementation after an approved plan, while the plan itself belongs in exit_plan_mode.
Resolve discoverable facts by inspection. Use ask_user_question only for user-owned choices or material ambiguity that inspection cannot answer. Do not ask the user where code lives or how current behavior works when you can find out.
Make the plan decision-complete: state the goal and success criteria; group implementation changes by subsystem; identify public API, schema, and data-flow changes; cover edge cases, failure modes, tests, acceptance criteria, and explicit assumptions. Keep it concise enough to review but detailed enough that another engineer can implement it without making design decisions.
When ready, call exit_plan_mode with the complete plan markdown, starting with a # title. Make exit_plan_mode the only and final tool call in that assistant response: it presents the plan for approval, and implementation begins only in a later step after approval. Do not paste the final plan as a plain reply or ask "should I proceed?" through prose or ask_user_question. If review rejects it, incorporate the feedback and present again. If the review channel is unavailable or aborted, stay in plan mode and ask the user to switch modes manually; do not proceed with implementation.
Use the read tool — not shell commands like cat — to inspect text files. Results include line numbers. Use offset and limit to continue reading large files.

18
examples/plan-acp-agent/tests/snapshots/plan-mode/tool-schemas.expected.json
View File

@@ -150,7 +150,7 @@
},
{
"name": "exit_plan_mode",
"description": "Present your plan for the user's review and, on approval, leave plan mode. Send the COMPLETE plan as markdown, starting with a # heading that names it. The user may approve (carry out the plan from your next step) or keep planning — their feedback comes back in the tool result; revise and present again.",
"description": "Use only in plan mode. Present your plan for the user's review and, on approval, leave plan mode. Send the COMPLETE plan as markdown, starting with a # heading that names it. The user may approve (carry out the plan from your next step) or keep planning — their feedback comes back in the tool result; revise and present again.",
"parameters": {
"type": "object",
"properties": {
@@ -478,6 +478,22 @@
]
}
},
{
"name": "exit_plan_mode",
"description": "Use only in plan mode. Present your plan for the user's review and, on approval, leave plan mode. Send the COMPLETE plan as markdown, starting with a # heading that names it. The user may approve (carry out the plan from your next step) or keep planning — their feedback comes back in the tool result; revise and present again.",
"parameters": {
"type": "object",
"properties": {
"plan": {
"type": "string",
"description": "The complete plan, as markdown, starting with a # heading that names it."
}
},
"required": [
"plan"
]
}
},
{
"name": "read",
"description": "Read a UTF-8 text file and return line-numbered content.",