Coder/deepseek-harness
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(tool-goal): let the model deliver a wrap-up message after a goal-round complete/blocked

Browse Source 
A goal round reporting complete or blocked used to conclude the turn at
the tool result, so the model never spoke after the call and sessions
ended on a bare update_goal card. The terminal update now defers one
plugin-sourced <goal_complete>/<goal_blocked> instruction onto its
result asking for a grounded closing message without further tool
calls; the turn then ends through the ordinary no-tool-calls stop.
Direct-human mutations stay uninstructed. Wording chosen by A/B
sampling on deepseek-v4-pro; one extra request per goal lifecycle.

New keyless ACP snapshot goal-wrapup drives the shipped app through
create -> round one -> autonomous complete and pins the injection, the
same-turn closing message, and the completed turn end.
This commit is contained in:
ZiyaZhang
2026-08-02 05:58:45 -07:00
parent 04d43e3b19
commit 666ef95f81
19 changed files with 334 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
examples/acp-agent/tests/goal-snapshots/goal-wrapup/input.json Normal file
View File

@@ -0,0 +1,13 @@
{
"steps": [
{ "op": "initialize" },
{ "op": "newSession" },
{
"op": "promptAndWaitForAgentMessage",
"text": "Create a durable goal for the wrap-up snapshot, then report readiness.",
"waitForText": "GOAL READY"
},
{ "op": "waitForTurnStart", "minimumTurn": 2 },
{ "op": "waitForTurnEnd" }
]
}

42
examples/acp-agent/tests/goal-snapshots/goal-wrapup/replay.override.json Normal file
View File

@@ -0,0 +1,42 @@
[
{
"kind": "chunks",
"chunks": [
{ "type": "block-start", "index": 0, "blockType": "tool-call" },
{ "type": "tool-call-delta", "index": 0, "id": "call_goal_create", "name": "create_goal", "argumentsDelta": "{\"objective\":\"Finish the ACP goal wrap-up snapshot proof\",\"max_goal_rounds\":2}" },
{ "type": "block-end", "index": 0, "block": { "type": "tool-call", "id": "call_goal_create", "name": "create_goal", "arguments": "{\"objective\":\"Finish the ACP goal wrap-up snapshot proof\",\"max_goal_rounds\":2}" } },
{ "type": "usage", "usage": { "inputTokens": 20, "outputTokens": 8 } },
{ "type": "finish", "reason": { "kind": "tool-calls" } }
]
},
{
"kind": "chunks",
"chunks": [
{ "type": "block-start", "index": 0, "blockType": "text" },
{ "type": "text-delta", "index": 0, "text": "GOAL READY" },
{ "type": "block-end", "index": 0, "block": { "type": "text", "text": "GOAL READY" } },
{ "type": "usage", "usage": { "inputTokens": 28, "outputTokens": 2 } },
{ "type": "finish", "reason": { "kind": "stop" } }
]
},
{
"kind": "chunks",
"chunks": [
{ "type": "block-start", "index": 0, "blockType": "tool-call" },
{ "type": "tool-call-delta", "index": 0, "id": "call_goal_complete", "name": "update_goal", "argumentsDelta": "{\"goal_id\":\"{{fromRequest:goal-[0-9a-f-]+}}\",\"revision\":1,\"action\":\"complete\"}" },
{ "type": "block-end", "index": 0, "block": { "type": "tool-call", "id": "call_goal_complete", "name": "update_goal", "arguments": "{\"goal_id\":\"{{fromRequest:goal-[0-9a-f-]+}}\",\"revision\":1,\"action\":\"complete\"}" } },
{ "type": "usage", "usage": { "inputTokens": 40, "outputTokens": 9 } },
{ "type": "finish", "reason": { "kind": "tool-calls" } }
]
},
{
"kind": "chunks",
"chunks": [
{ "type": "block-start", "index": 0, "blockType": "text" },
{ "type": "text-delta", "index": 0, "text": "GOAL WRAP-UP: the snapshot objective is achieved and this closing message reaches the user." },
{ "type": "block-end", "index": 0, "block": { "type": "text", "text": "GOAL WRAP-UP: the snapshot objective is achieved and this closing message reaches the user." } },
{ "type": "usage", "usage": { "inputTokens": 52, "outputTokens": 14 } },
{ "type": "finish", "reason": { "kind": "stop" } }
]
}
]

50
examples/acp-agent/tests/goal-snapshots/goal-wrapup/session.expected.jsonl Normal file
View File

@@ -0,0 +1,50 @@
{"type":"session","version":0,"id":"{{sessionId}}","createdAt":0,"cwd":"{{cwd}}","delegationDepth":0}
{"type":"turn/start","seq":0,"time":0,"data":{"turn":1,"trigger":{"kind":"message","source":{"kind":"user"}}}}
{"type":"user/message","seq":1,"time":0,"data":{"content":[{"type":"text","text":"Create a durable goal for the wrap-up snapshot, then report readiness."}],"source":{"kind":"user"},"role":"user","id":"{{sessionId}}"},"surfaceOp":"append"}
{"type":"session/title","seq":2,"time":0,"data":{"title":"Create a durable goal for","messageSeqs":[1],"source":{"kind":"fallback"}}}
{"type":"user/message","seq":3,"time":0,"data":{"content":[{"type":"text","text":"Current runtime context. This snapshot supersedes earlier runtime-context snapshots.\n\nCurrent DSH file policy: danger-full-access. The DSH file sandbox does not restrict file modifications by available operations.\n\nApproval prompts are disabled in this session: actions that require approval are rejected automatically — do not request sandbox escalation (do not set `sandbox_permissions`)."}],"source":{"kind":"plugin","plugin":"@deepseek-ai/dsh-system-prompt"},"role":"user","id":"{{sessionId}}"},"surfaceOp":"append"}
{"type":"step/start","seq":4,"time":0,"data":{"turn":1,"step":1}}
{"type":"request/header","seq":5,"time":0,"data":{"header":{"config":{"provider":"deepseek-official","model":"deepseek-v4-flash"},"system":"{{system}}","tools":"{{tools}}"},"reason":"initial"}}
{"type":"request/context","seq":6,"time":0,"data":{"provider":"deepseek-official","model":"deepseek-v4-flash"}}
{"type":"assistant/chunk","seq":7,"time":0,"data":{"turn":1,"step":1,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}}}
{"type":"assistant/chunk","seq":8,"time":0,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":0,"id":"call_goal_create","name":"create_goal","argumentsDelta":"{\"objective\":\"Finish the ACP goal wrap-up snapshot proof\",\"max_goal_rounds\":2}"}}}
{"type":"assistant/chunk","seq":9,"time":0,"data":{"turn":1,"step":1,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"call_goal_create","name":"create_goal","arguments":"{\"objective\":\"Finish the ACP goal wrap-up snapshot proof\",\"max_goal_rounds\":2}"}}}}
{"type":"assistant/chunk","seq":10,"time":0,"data":{"turn":1,"step":1,"chunk":{"type":"usage","usage":{"inputTokens":20,"outputTokens":8}}}}
{"type":"assistant/chunk","seq":11,"time":0,"data":{"turn":1,"step":1,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}}
{"type":"assistant/message","seq":12,"time":0,"data":{"turn":1,"step":1,"message":{"role":"assistant","content":[{"type":"tool-call","id":"call_goal_create","name":"create_goal","arguments":"{\"objective\":\"Finish the ACP goal wrap-up snapshot proof\",\"max_goal_rounds\":2}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash"},"id":"{{sessionId}}"},"usage":{"inputTokens":20,"outputTokens":8}},"sourceEventSeqs":[7,8,9,10,11],"surfaceOp":"append"}
{"type":"tool/call","seq":13,"time":0,"data":{"turn":1,"step":1,"callId":"call_goal_create","name":"create_goal","arguments":"{\"objective\":\"Finish the ACP goal wrap-up snapshot proof\",\"max_goal_rounds\":2}"}}
{"type":"tool/result","seq":14,"time":0,"data":{"turn":1,"step":1,"message":{"source":{"kind":"tool","callId":"call_goal_create"},"content":[{"type":"tool-result","toolCallId":"call_goal_create","content":[{"type":"text","text":"{\"goal\":{\"id\":\"goal-{{sessionId}}\",\"revision\":1,\"objective\":\"Finish the ACP goal wrap-up snapshot proof\",\"phase\":\"active\",\"roundsStarted\":0,\"maxGoalRounds\":2},\"activation\":\"armed\"}"}],"isError":false}],"role":"user","id":"{{sessionId}}"}},"sourceEventSeqs":[13],"surfaceOp":"append"}
{"type":"user/message","seq":15,"time":0,"data":{"content":[{"type":"text","text":"<goal_state>{\"goal\":{\"id\":\"goal-{{sessionId}}\",\"revision\":1,\"objective\":\"Finish the ACP goal wrap-up snapshot proof\",\"phase\":\"active\",\"maxGoalRounds\":2},\"roundsStarted\":0,\"createdAt\":0,\"updatedAt\":0}</goal_state>"}],"source":{"kind":"goal","goalId":"goal-{{sessionId}}","revision":1,"round":0,"change":{"kind":"goal/change","version":1,"operation":"create","goal":{"id":"goal-{{sessionId}}","revision":1,"objective":"Finish the ACP goal wrap-up snapshot proof","phase":"active","maxGoalRounds":2},"roundsStarted":0,"createdAt":0,"updatedAt":0}},"role":"user","id":"{{sessionId}}"},"surfaceOp":"append"}
{"type":"step/end","seq":16,"time":0,"data":{"turn":1,"step":1}}
{"type":"step/start","seq":17,"time":0,"data":{"turn":1,"step":2}}
{"type":"assistant/chunk","seq":18,"time":0,"data":{"turn":1,"step":2,"chunk":{"type":"block-start","index":0,"blockType":"text"}}}
{"type":"assistant/chunk","seq":19,"time":0,"data":{"turn":1,"step":2,"chunk":{"type":"text-delta","index":0,"text":"GOAL READY"}}}
{"type":"assistant/chunk","seq":20,"time":0,"data":{"turn":1,"step":2,"chunk":{"type":"block-end","index":0,"block":{"type":"text","text":"GOAL READY"}}}}
{"type":"assistant/chunk","seq":21,"time":0,"data":{"turn":1,"step":2,"chunk":{"type":"usage","usage":{"inputTokens":28,"outputTokens":2}}}}
{"type":"assistant/chunk","seq":22,"time":0,"data":{"turn":1,"step":2,"chunk":{"type":"finish","reason":{"kind":"stop"}}}}
{"type":"assistant/message","seq":23,"time":0,"data":{"turn":1,"step":2,"message":{"role":"assistant","content":[{"type":"text","text":"GOAL READY"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash"},"id":"{{sessionId}}"},"usage":{"inputTokens":28,"outputTokens":2}},"sourceEventSeqs":[18,19,20,21,22],"surfaceOp":"append"}
{"type":"step/end","seq":24,"time":0,"data":{"turn":1,"step":2}}
{"type":"turn/end","seq":25,"time":0,"data":{"turn":1,"reason":{"kind":"completed"}}}
{"type":"turn/start","seq":26,"time":0,"data":{"turn":2,"trigger":{"kind":"message","source":{"kind":"goal","goalId":"goal-{{sessionId}}","revision":1,"round":1}}}}
{"type":"user/message","seq":27,"time":0,"data":{"content":[{"type":"text","text":"<goal_round>\nObjective: \"Finish the ACP goal wrap-up snapshot proof\"\nRound: 1/2\n\nContinue working toward the objective in this same session. Treat the current workspace, tool results, and durable session state as authoritative; inspect them instead of assuming earlier narration is still current. Make concrete progress and verify the result. Before claiming completion, gather evidence that the whole objective is achieved, read the current goal, and mark it complete. If work remains, leave the goal active for the next round. Follow the configured goal-tool policy before reporting a blocker.\n</goal_round>"}],"source":{"kind":"goal","goalId":"goal-{{sessionId}}","revision":1,"round":1},"role":"user","id":"{{sessionId}}"},"surfaceOp":"append"}
{"type":"step/start","seq":28,"time":0,"data":{"turn":2,"step":1}}
{"type":"assistant/chunk","seq":29,"time":0,"data":{"turn":2,"step":1,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}}}
{"type":"assistant/chunk","seq":30,"time":0,"data":{"turn":2,"step":1,"chunk":{"type":"tool-call-delta","index":0,"id":"call_goal_complete","name":"update_goal","argumentsDelta":"{\"goal_id\":\"goal-{{sessionId}}\",\"revision\":1,\"action\":\"complete\"}"}}}
{"type":"assistant/chunk","seq":31,"time":0,"data":{"turn":2,"step":1,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"call_goal_complete","name":"update_goal","arguments":"{\"goal_id\":\"goal-{{sessionId}}\",\"revision\":1,\"action\":\"complete\"}"}}}}
{"type":"assistant/chunk","seq":32,"time":0,"data":{"turn":2,"step":1,"chunk":{"type":"usage","usage":{"inputTokens":40,"outputTokens":9}}}}
{"type":"assistant/chunk","seq":33,"time":0,"data":{"turn":2,"step":1,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}}
{"type":"assistant/message","seq":34,"time":0,"data":{"turn":2,"step":1,"message":{"role":"assistant","content":[{"type":"tool-call","id":"call_goal_complete","name":"update_goal","arguments":"{\"goal_id\":\"goal-{{sessionId}}\",\"revision\":1,\"action\":\"complete\"}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash"},"id":"{{sessionId}}"},"usage":{"inputTokens":40,"outputTokens":9}},"sourceEventSeqs":[29,30,31,32,33],"surfaceOp":"append"}
{"type":"tool/call","seq":35,"time":0,"data":{"turn":2,"step":1,"callId":"call_goal_complete","name":"update_goal","arguments":"{\"goal_id\":\"goal-{{sessionId}}\",\"revision\":1,\"action\":\"complete\"}"}}
{"type":"tool/result","seq":36,"time":0,"data":{"turn":2,"step":1,"message":{"source":{"kind":"tool","callId":"call_goal_complete"},"content":[{"type":"tool-result","toolCallId":"call_goal_complete","content":[{"type":"text","text":"{\"goal\":{\"id\":\"goal-{{sessionId}}\",\"revision\":2,\"objective\":\"Finish the ACP goal wrap-up snapshot proof\",\"phase\":\"complete\",\"roundsStarted\":1,\"maxGoalRounds\":2},\"activation\":\"disarmed\"}"}],"isError":false}],"role":"user","id":"{{sessionId}}"}},"sourceEventSeqs":[35],"surfaceOp":"append"}
{"type":"user/message","seq":37,"time":0,"data":{"content":[{"type":"text","text":"<goal_state>{\"goal\":{\"id\":\"goal-{{sessionId}}\",\"revision\":2,\"objective\":\"Finish the ACP goal wrap-up snapshot proof\",\"phase\":\"complete\",\"maxGoalRounds\":2},\"roundsStarted\":1,\"createdAt\":0,\"updatedAt\":0}</goal_state>"}],"source":{"kind":"goal","goalId":"goal-{{sessionId}}","revision":2,"round":0,"change":{"kind":"goal/change","version":1,"operation":"complete","goal":{"id":"goal-{{sessionId}}","revision":2,"objective":"Finish the ACP goal wrap-up snapshot proof","phase":"complete","maxGoalRounds":2},"roundsStarted":1,"createdAt":0,"updatedAt":0}},"role":"user","id":"{{sessionId}}"},"surfaceOp":"append"}
{"type":"user/message","seq":38,"time":0,"data":{"content":[{"type":"text","text":"<goal_complete>\nObjective: \"Finish the ACP goal wrap-up snapshot proof\"\nThe goal is marked complete and this autonomous run is ending. Write the closing message to the user now: state the outcome, summarize what was done and how it was verified, and point to the concrete results (files, commits, or other artifacts). Report only what earlier rounds and tool results in this session actually establish; when a detail is not in the session, say so instead of inventing it. Note anything the user should review or do next. Address the user directly. Do not call any more tools.\n</goal_complete>"}],"source":{"kind":"plugin","plugin":"tool-goal"},"role":"user","id":"{{sessionId}}"},"surfaceOp":"append"}
{"type":"step/end","seq":39,"time":0,"data":{"turn":2,"step":1}}
{"type":"step/start","seq":40,"time":0,"data":{"turn":2,"step":2}}
{"type":"assistant/chunk","seq":41,"time":0,"data":{"turn":2,"step":2,"chunk":{"type":"block-start","index":0,"blockType":"text"}}}
{"type":"assistant/chunk","seq":42,"time":0,"data":{"turn":2,"step":2,"chunk":{"type":"text-delta","index":0,"text":"GOAL WRAP-UP: the snapshot objective is achieved and this closing message reaches the user."}}}
{"type":"assistant/chunk","seq":43,"time":0,"data":{"turn":2,"step":2,"chunk":{"type":"block-end","index":0,"block":{"type":"text","text":"GOAL WRAP-UP: the snapshot objective is achieved and this closing message reaches the user."}}}}
{"type":"assistant/chunk","seq":44,"time":0,"data":{"turn":2,"step":2,"chunk":{"type":"usage","usage":{"inputTokens":52,"outputTokens":14}}}}
{"type":"assistant/chunk","seq":45,"time":0,"data":{"turn":2,"step":2,"chunk":{"type":"finish","reason":{"kind":"stop"}}}}
{"type":"assistant/message","seq":46,"time":0,"data":{"turn":2,"step":2,"message":{"role":"assistant","content":[{"type":"text","text":"GOAL WRAP-UP: the snapshot objective is achieved and this closing message reaches the user."}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash"},"id":"{{sessionId}}"},"usage":{"inputTokens":52,"outputTokens":14}},"sourceEventSeqs":[41,42,43,44,45],"surfaceOp":"append"}
{"type":"step/end","seq":47,"time":0,"data":{"turn":2,"step":2}}
{"type":"turn/end","seq":48,"time":0,"data":{"turn":2,"reason":{"kind":"completed"}}}

1
examples/acp-agent/tests/goal-snapshots/goal-wrapup/session.jsonl Normal file
View File

@@ -0,0 +1 @@
{"type":"session","version":0,"id":"goal-wrapup-placeholder","createdAt":0,"cwd":"{{cwd}}"}

5
examples/acp-agent/tests/goal-snapshots/goal-wrapup/stdout.expected.jsonl Normal file
View File

@@ -0,0 +1,5 @@
{"jsonrpc":"2.0","id":1,"result":{"protocolVersion":1,"agentInfo":{"name":"deepseek-harness-acp","version":"0.0.1"},"agentCapabilities":{"promptCapabilities":{"image":false,"audio":false,"embeddedContext":false}},"authMethods":[]}}
{"jsonrpc":"2.0","id":2,"result":{"sessionId":"{{sessionId}}"}}
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":"GOAL READY"}}}}
{"jsonrpc":"2.0","id":3,"result":{"stopReason":"end_turn"}}
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":"GOAL WRAP-UP: the snapshot objective is achieved and this closing message reaches the user."}}}}

59
examples/acp-agent/tests/goal.snapshot.ts
View File

@@ -21,6 +21,7 @@ const fixtureFile = join(scenarioDir, 'session.jsonl')
const overrideFile = join(scenarioDir, 'replay.override.json')
const stdoutExpected = join(scenarioDir, 'stdout.expected.jsonl')
const sessionExpected = join(scenarioDir, 'session.expected.jsonl')
const wrapupDir = join(dirname(fileURLToPath(import.meta.url)), 'goal-snapshots/goal-wrapup')
const refreshing = process.env.DSH_SNAPSHOT === 'refresh'
const agent: AgentUnderTest = {
@@ -112,4 +113,62 @@ describe('same-session goal snapshot through the ACP automation driver', () => {
expect(stdout).toBe(await readFile(stdoutExpected, 'utf8'))
expect(session).toBe(await readFile(sessionExpected, 'utf8'))
})
it('injects the wrap-up instruction after an autonomous completion and delivers a closing message', async () => {
const input = JSON.parse(await readFile(join(wrapupDir, 'input.json'), 'utf8')) as InputScript
const result = await runScenario(input, {
agent,
mode: 'replay',
fixtureFile: join(wrapupDir, 'session.jsonl'),
overrideFile: join(wrapupDir, 'replay.override.json'),
configPath: agent.configPath,
})
expect(result.stderr).toBe('')
expect(result.sessionLogs).toHaveLength(1)
const log = result.sessionLogs[0]
if (log === undefined) throw new Error('goal wrap-up snapshot did not persist its session')
const records = parseJsonl(log.content)
const events = records.slice(1) as unknown as SessionEvent[]
const calls = events.filter(event => event.type === 'tool/call').map(event => event.data.name)
expect(calls).toEqual(['create_goal', 'update_goal'])
expect(foldGoal(events)).toMatchObject({
goal: {
objective: 'Finish the ACP goal wrap-up snapshot proof',
phase: 'complete',
revision: 2,
},
roundsStarted: 1,
})
// The wrap-up instruction is one plugin-sourced context injected after the
// terminal tool result, and the model still answers inside the same turn.
const wrapups = events.filter(event => event.type === 'user/message'
&& event.data.source.kind === 'plugin' && event.data.source.plugin === 'tool-goal')
expect(wrapups).toHaveLength(1)
const wrapupText = wrapups.map(event => event.type === 'user/message' ? event.data.content : [])[0]
expect(JSON.stringify(wrapupText)).toContain('<goal_complete>')
const closing = events.filter(event => event.type === 'assistant/message')
.flatMap(event => event.data.message.content)
.filter(block => block.type === 'text' && block.text.startsWith('GOAL WRAP-UP'))
expect(closing).toHaveLength(1)
const roundTurnEnds = events.filter(event => event.type === 'turn/end' && event.data.turn === 2)
expect(roundTurnEnds).toEqual([expect.objectContaining({ data: { turn: 2, reason: { kind: 'completed' } } })])
const context: NormalizeContext = {
sessionIds: [result.sessionId, log.id].filter((id): id is string => id !== undefined),
cwd: result.cwd,
}
const stdout = normalizeStdout(result.rawStdout, context)
const session = normalizeGoalLog(log.content, context)
const wrapupStdoutExpected = join(wrapupDir, 'stdout.expected.jsonl')
const wrapupSessionExpected = join(wrapupDir, 'session.expected.jsonl')
if (refreshing) {
await Promise.all([
writeFile(wrapupStdoutExpected, stdout),
writeFile(wrapupSessionExpected, session),
])
}
expect(stdout).toBe(await readFile(wrapupStdoutExpected, 'utf8'))
expect(session).toBe(await readFile(wrapupSessionExpected, 'utf8'))
})
})