Coder/deepseek-harness
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(user-interaction): address review feedback

Browse Source
This commit is contained in:
Yichen Jiang
2026-07-30 17:59:37 +08:00
parent 31a498b1db
commit 3ba4d40e6a
16 changed files with 119 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
apps/web/tests/question-composer.e2e.ts
View File

@@ -23,15 +23,16 @@ import { connectFreshWorkspace, newEnglishPage, saveFailureShot } from './suppor
const SNAPSHOT_DIR = fileURLToPath(new URL('./snapshots/question-composer', import.meta.url))
const FIXTURE = join(SNAPSHOT_DIR, 'session.jsonl')
const UI_EXPECTED = join(SNAPSHOT_DIR, 'ui.expected.md')
// Second golden: the answered transcript — the question resolved into its
// tool round trip and the final reply, the state the waiting golden cannot see.
const COMPOSED_EXPECTED = join(SNAPSHOT_DIR, 'composed.expected.md')
// Final golden: the answered transcript — the question resolved into its tool
// round trip and the final reply, the state the composer goldens cannot see.
const ANSWERED_EXPECTED = join(SNAPSHOT_DIR, 'answered.expected.md')
const MODE = webSnapshotMode()
// The options carry long descriptions on purpose: the squeeze assertion below
// needs option copy that WRAPS, which is the only shape that reproduces a
// collapsed row painting its copy outside its own box.
const PROMPT = 'Use the ask_user_question tool to ask me exactly one question with id "color", question "Which color do you prefer?", header "Pick one", and two options: label "Blue" with description "A cool recessive hue that reads as calm and trustworthy in long reading sessions and dense dashboards.", and label "Green" with description "A restful mid-spectrum hue with the highest perceived brightness, easiest on the eye over long sessions." After I answer, reply with the single word DONE and stop.'
const PROMPT = 'Use the ask_user_question tool to ask me exactly one multi-select question with id "color", question "Which color do you prefer?", header "Pick one", and two options: label "Blue" with description "A cool recessive hue that reads as calm and trustworthy in long reading sessions and dense dashboards.", and label "Green" with description "A restful mid-spectrum hue with the highest perceived brightness, easiest on the eye over long sessions." Set multi_select to true. After I answer, reply with the single word DONE and stop.'
describe('web e2e: resident question composer round trip', () => {
let scaffold: WebScaffold
@@ -124,9 +125,17 @@ describe('web e2e: resident question composer round trip', () => {
await page.setViewportSize(original)
}
await composer.getByRole('radio', { name: 'Blue' }).click()
// Submit: Enter on the focused option (the composer's documented submit).
await composer.getByRole('radio', { name: 'Blue' }).press('Enter')
const blue = composer.getByRole('checkbox', { name: 'Blue' })
await blue.click()
const custom = composer.getByRole('textbox')
await custom.fill('Include accessibility notes')
expect(await blue.getAttribute('aria-checked')).toBe('true')
expect(await custom.inputValue()).toBe('Include accessibility notes')
if (MODE !== 'record') {
const snapshot = await captureStableAria(page, '[data-question-key]', scaffold.workspaceCwd)
await compareOrRefreshGolden(COMPOSED_EXPECTED, snapshot, MODE)
}
await custom.press('Enter')
const sessionId = await settled
if (MODE === 'record') {
@@ -135,7 +144,14 @@ describe('web e2e: resident question composer round trip', () => {
}
// World state: the tool result carries the chosen answer, and DONE lands.
const results = sessionEvents.filter(e => e.type === 'tool/result')
expect(JSON.stringify(results.at(-1))).toContain('Blue')
const answerText = results.flatMap(event => event.data.message.content.flatMap(block =>
block.type === 'tool-result'
? block.content.filter(item => item.type === 'text').map(item => item.text)
: [],
)).at(-1)
expect(JSON.parse(answerText ?? '')).toEqual({
answers: [{ id: 'color', selected: ['Blue'], custom: 'Include accessibility notes' }],
})
await expect.poll(() => page.getByText('DONE', { exact: true }).count(), { timeout: 15_000 }).toBeGreaterThanOrEqual(1)
// Composer gone; regular input restored.
expect(await page.locator('[data-question-key]').count()).toBe(0)
@@ -149,6 +165,11 @@ describe('web e2e: resident question composer round trip', () => {
}, 200_000)
it.skipIf(MODE === 'record')('keeps the fixture inventory closed', async () => {
await assertFixtureInventory(SNAPSHOT_DIR, ['session.jsonl', 'ui.expected.md', 'answered.expected.md'])
await assertFixtureInventory(SNAPSHOT_DIR, [
'session.jsonl',
'ui.expected.md',
'composed.expected.md',
'answered.expected.md',
])
})
})

3
apps/web/tests/snapshots/question-composer/answered.expected.md
View File

@@ -4,13 +4,14 @@
- tablist:
- tab "Chat" [selected]
- tab "Trajectory"
- text: "Use the ask_user_question tool to ask me exactly one question with id \"color\", question \"Which color do you prefer?\", header \"Pick one\", and two options: label \"Blue\" with description \"A cool recessive hue that reads as calm and trustworthy in long reading sessions and dense dashboards.\", and label \"Green\" with description \"A restful mid-spectrum hue with the highest perceived brightness, easiest on the eye over long sessions.\" After I answer, reply with the single word DONE and stop. {{clock}}"
- text: "Use the ask_user_question tool to ask me exactly one multi-select question with id \"color\", question \"Which color do you prefer?\", header \"Pick one\", and two options: label \"Blue\" with description \"A cool recessive hue that reads as calm and trustworthy in long reading sessions and dense dashboards.\", and label \"Green\" with description \"A restful mid-spectrum hue with the highest perceived brightness, easiest on the eye over long sessions.\" Set multi_select to true. After I answer, reply with the single word DONE and stop. {{clock}}"
- button "复制":
- img
- button "在新对话中分支":
- img
- button "编辑":
- img
- button "▸ 上下文注入"
- button "Think The user wants me to use the ask_user_question tool with specific parameters. Let me do exactly that.":
- img
- img

17
apps/web/tests/snapshots/question-composer/composed.expected.md Normal file
View File

@@ -0,0 +1,17 @@
- region "Which color do you prefer?":
- text: Pick one
- heading "Which color do you prefer?" [level=2]
- button "Dismiss all questions":
- img
- group:
- checkbox "Blue" [checked]: Blue A cool recessive hue that reads as calm and trustworthy in long reading sessions and dense dashboards.
- checkbox "Green": Green A restful mid-spectrum hue with the highest perceived brightness, easiest on the eye over long sessions.
- textbox "Type your answer": Include accessibility notes
- button "Previous question" [disabled]:
- img
- text: 1 / 1
- button "Next question" [disabled]:
- img
- status
- button "Skip this question"
- button "Submit"

12
apps/web/tests/snapshots/question-composer/session.jsonl
View File

@@ -1,20 +1,20 @@
{"type":"session","version":0,"id":"{{sessionId}}","createdAt":1785150167878,"cwd":"{{cwd}}/workspace"}
{"type":"turn/start","seq":0,"time":1785150167924,"data":{"turn":1,"trigger":{"kind":"message","source":{"kind":"user","rpcId":"{{rpcId}}"}}}}
{"type":"user/message","seq":1,"time":1785150167925,"data":{"content":[{"type":"text","text":"Use the ask_user_question tool to ask me exactly one question with id \"color\", question \"Which color do you prefer?\", header \"Pick one\", and two options: label \"Blue\" with description \"A cool recessive hue that reads as calm and trustworthy in long reading sessions and dense dashboards.\", and label \"Green\" with description \"A restful mid-spectrum hue with the highest perceived brightness, easiest on the eye over long sessions.\" After I answer, reply with the single word DONE and stop."}],"source":{"kind":"user","rpcId":"{{rpcId}}"},"role":"user","id":"6deba879-8787-4853-a5f2-0d108a08eb2d"},"surfaceOp":"append"}
{"type":"user/message","seq":1,"time":1785150167925,"data":{"content":[{"type":"text","text":"Use the ask_user_question tool to ask me exactly one multi-select question with id \"color\", question \"Which color do you prefer?\", header \"Pick one\", and two options: label \"Blue\" with description \"A cool recessive hue that reads as calm and trustworthy in long reading sessions and dense dashboards.\", and label \"Green\" with description \"A restful mid-spectrum hue with the highest perceived brightness, easiest on the eye over long sessions.\" Set multi_select to true. After I answer, reply with the single word DONE and stop."}],"source":{"kind":"user","rpcId":"{{rpcId}}"},"role":"user","id":"6deba879-8787-4853-a5f2-0d108a08eb2d"},"surfaceOp":"append"}
{"type":"session/title","seq":2,"time":1785150167927,"data":{"title":"Use the ask_user_question tool to","messageSeqs":[1],"source":{"kind":"fallback"}}}
{"type":"step/start","seq":3,"time":1785150167928,"data":{"turn":1,"step":1}}
{"type":"request/header","seq":4,"time":1785150167929,"data":{"header":{"config":{"provider":"deepseek","model":"deepseek-v4-flash"},"system":"{{system}}","tools":"{{tools}}"},"reason":"initial"}}
{"type":"assistant/chunk","seq":5,"time":1785150168452,"data":{"turn":1,"step":1,"chunk":{"type":"block-start","index":0,"blockType":"reasoning"}}}
{"type":"reasoning-chunks","seq0":6,"time0":1785150168452,"data":{"turn":1,"step":1,"index":0,"dt":[87,26,1,0,0,0,38,0,0,0,0,1,12,27,0,27,0,0,1,25,0],"texts":["The"," user"," wants"," me"," to"," use"," the"," ask","_user","_","question"," tool"," with"," specific"," parameters","."," Let"," me"," do"," exactly"," that","."]}}
{"type":"assistant/chunk","seq":28,"time":1785150168775,"data":{"turn":1,"step":1,"chunk":{"type":"block-start","index":1,"blockType":"tool-call"}}}
{"type":"tool-call-chunks","seq0":29,"time0":1785150168776,"data":{"turn":1,"step":1,"index":1,"dt":[25,1,0,0,0,25,0,0,0,26,1,0,0,0,0,25,1,0,0,0,0,25,1,0,0,0,0,25,1,0,0,0,1,25,0,0,0,0,1,25,1,0,0,0,0,25,1,0,0,26,0,0,1,0,24,1,0,0,0,1,26,1,0,0,0,0,25,0,1,0,0,0,25,1,0,0,25,0,0,0,0,1,25,0,0,1,0,0,25,0,0,0,1,0,26,1,24],"id":"call_00_Cijldc88LYmVPCXYUsRq1617","name":"ask_user_question","args":["","{","\"","questions","\"",": ","[","{\"","id","\":"," \"","color","\","," \"","question","\":"," \"","Which"," color"," do"," you"," prefer","?\","," \"","header","\":"," \"","Pick"," one","\","," \"","options","\":"," [","{\"","label","\":"," \"","Blue","\","," \"","description","\":"," \"","A"," cool"," recessive"," hue"," that"," reads"," as"," calm"," and"," trustworthy"," in"," long"," reading"," sessions"," and"," dense"," dash","boards",".\"","},"," {\"","label","\":"," \"","Green","\","," \"","description","\":"," \"","A"," rest","ful"," mid","-spect","rum"," hue"," with"," the"," highest"," perceived"," brightness",","," easiest"," on"," the"," eye"," over"," long"," sessions",".\"","}]","}]","}"]}}
{"type":"tool-call-chunks","seq0":29,"time0":1785150168776,"data":{"turn":1,"step":1,"index":1,"dt":[25,1,0,0,0,25,0,0,0,26,1,0,0,0,0,25,1,0,0,0,0,25,1,0,0,0,0,25,1,0,0,0,1,25,0,0,0,0,1,25,1,0,0,0,0,25,1,0,0,26,0,0,1,0,24,1,0,0,0,1,26,1,0,0,0,0,25,0,1,0,0,0,25,1,0,0,25,0,0,0,0,1,25,0,0,1,0,0,25,0,0,0,1,0,26,1,24],"id":"call_00_Cijldc88LYmVPCXYUsRq1617","name":"ask_user_question","args":["","{","\"","questions","\"",": ","[","{\"","id","\":"," \"","color","\","," \"","question","\":"," \"","Which"," color"," do"," you"," prefer","?\","," \"","header","\":"," \"","Pick"," one","\", \"multi_select\": true,"," \"","options","\":"," [","{\"","label","\":"," \"","Blue","\","," \"","description","\":"," \"","A"," cool"," recessive"," hue"," that"," reads"," as"," calm"," and"," trustworthy"," in"," long"," reading"," sessions"," and"," dense"," dash","boards",".\"","},"," {\"","label","\":"," \"","Green","\","," \"","description","\":"," \"","A"," rest","ful"," mid","-spect","rum"," hue"," with"," the"," highest"," perceived"," brightness",","," easiest"," on"," the"," eye"," over"," long"," sessions",".\"","}]","}]","}"]}}
{"type":"assistant/chunk","seq":127,"time":1785150169308,"data":{"turn":1,"step":1,"chunk":{"type":"block-end","index":0,"block":{"type":"reasoning","text":"The user wants me to use the ask_user_question tool with specific parameters. Let me do exactly that."}}}}
{"type":"assistant/chunk","seq":128,"time":1785150169308,"data":{"turn":1,"step":1,"chunk":{"type":"block-end","index":1,"block":{"type":"tool-call","id":"call_00_Cijldc88LYmVPCXYUsRq1617","name":"ask_user_question","arguments":"{\"questions\": [{\"id\": \"color\", \"question\": \"Which color do you prefer?\", \"header\": \"Pick one\", \"options\": [{\"label\": \"Blue\", \"description\": \"A cool recessive hue that reads as calm and trustworthy in long reading sessions and dense dashboards.\"}, {\"label\": \"Green\", \"description\": \"A restful mid-spectrum hue with the highest perceived brightness, easiest on the eye over long sessions.\"}]}]}"}}}}
{"type":"assistant/chunk","seq":128,"time":1785150169308,"data":{"turn":1,"step":1,"chunk":{"type":"block-end","index":1,"block":{"type":"tool-call","id":"call_00_Cijldc88LYmVPCXYUsRq1617","name":"ask_user_question","arguments":"{\"questions\": [{\"id\": \"color\", \"question\": \"Which color do you prefer?\", \"header\": \"Pick one\", \"multi_select\": true, \"options\": [{\"label\": \"Blue\", \"description\": \"A cool recessive hue that reads as calm and trustworthy in long reading sessions and dense dashboards.\"}, {\"label\": \"Green\", \"description\": \"A restful mid-spectrum hue with the highest perceived brightness, easiest on the eye over long sessions.\"}]}]}"}}}}
{"type":"assistant/chunk","seq":129,"time":1785150169308,"data":{"turn":1,"step":1,"chunk":{"type":"usage","usage":{"inputTokens":113,"outputTokens":158,"cacheReadTokens":4096,"reasoningTokens":22}}}}
{"type":"assistant/chunk","seq":130,"time":1785150169308,"data":{"turn":1,"step":1,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}}
{"type":"assistant/message","seq":131,"time":1785150169311,"data":{"turn":1,"step":1,"usage":{"inputTokens":113,"outputTokens":158,"cacheReadTokens":4096,"reasoningTokens":22},"message":{"role":"assistant","content":[{"type":"reasoning","text":"The user wants me to use the ask_user_question tool with specific parameters. Let me do exactly that."},{"type":"tool-call","id":"call_00_Cijldc88LYmVPCXYUsRq1617","name":"ask_user_question","arguments":"{\"questions\": [{\"id\": \"color\", \"question\": \"Which color do you prefer?\", \"header\": \"Pick one\", \"options\": [{\"label\": \"Blue\", \"description\": \"A cool recessive hue that reads as calm and trustworthy in long reading sessions and dense dashboards.\"}, {\"label\": \"Green\", \"description\": \"A restful mid-spectrum hue with the highest perceived brightness, easiest on the eye over long sessions.\"}]}]}"}],"source":{"kind":"model","provider":"deepseek","model":"deepseek-v4-flash"},"id":"cdb1676c-e781-41ee-8f28-a3595371d729"}},"sourceEventSeqs":[5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130],"surfaceOp":"append"}
{"type":"tool/call","seq":132,"time":1785150169312,"data":{"turn":1,"step":1,"callId":"call_00_Cijldc88LYmVPCXYUsRq1617","name":"ask_user_question","arguments":"{\"questions\": [{\"id\": \"color\", \"question\": \"Which color do you prefer?\", \"header\": \"Pick one\", \"options\": [{\"label\": \"Blue\", \"description\": \"A cool recessive hue that reads as calm and trustworthy in long reading sessions and dense dashboards.\"}, {\"label\": \"Green\", \"description\": \"A restful mid-spectrum hue with the highest perceived brightness, easiest on the eye over long sessions.\"}]}]}"}}
{"type":"tool/result","seq":133,"time":1785150169787,"data":{"turn":1,"step":1,"message":{"source":{"kind":"tool","callId":"call_00_Cijldc88LYmVPCXYUsRq1617"},"content":[{"type":"tool-result","toolCallId":"call_00_Cijldc88LYmVPCXYUsRq1617","content":[{"type":"text","text":"{\"answers\":[{\"id\":\"color\",\"selected\":[\"Blue\"]}]}"}],"isError":false}],"role":"user","id":"c69ef39f-6f62-439f-b3f8-e8d10fba572f"}},"sourceEventSeqs":[132],"surfaceOp":"append"}
{"type":"assistant/message","seq":131,"time":1785150169311,"data":{"turn":1,"step":1,"usage":{"inputTokens":113,"outputTokens":158,"cacheReadTokens":4096,"reasoningTokens":22},"message":{"role":"assistant","content":[{"type":"reasoning","text":"The user wants me to use the ask_user_question tool with specific parameters. Let me do exactly that."},{"type":"tool-call","id":"call_00_Cijldc88LYmVPCXYUsRq1617","name":"ask_user_question","arguments":"{\"questions\": [{\"id\": \"color\", \"question\": \"Which color do you prefer?\", \"header\": \"Pick one\", \"multi_select\": true, \"options\": [{\"label\": \"Blue\", \"description\": \"A cool recessive hue that reads as calm and trustworthy in long reading sessions and dense dashboards.\"}, {\"label\": \"Green\", \"description\": \"A restful mid-spectrum hue with the highest perceived brightness, easiest on the eye over long sessions.\"}]}]}"}],"source":{"kind":"model","provider":"deepseek","model":"deepseek-v4-flash"},"id":"cdb1676c-e781-41ee-8f28-a3595371d729"}},"sourceEventSeqs":[5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130],"surfaceOp":"append"}
{"type":"tool/call","seq":132,"time":1785150169312,"data":{"turn":1,"step":1,"callId":"call_00_Cijldc88LYmVPCXYUsRq1617","name":"ask_user_question","arguments":"{\"questions\": [{\"id\": \"color\", \"question\": \"Which color do you prefer?\", \"header\": \"Pick one\", \"multi_select\": true, \"options\": [{\"label\": \"Blue\", \"description\": \"A cool recessive hue that reads as calm and trustworthy in long reading sessions and dense dashboards.\"}, {\"label\": \"Green\", \"description\": \"A restful mid-spectrum hue with the highest perceived brightness, easiest on the eye over long sessions.\"}]}]}"}}
{"type":"tool/result","seq":133,"time":1785150169787,"data":{"turn":1,"step":1,"message":{"source":{"kind":"tool","callId":"call_00_Cijldc88LYmVPCXYUsRq1617"},"content":[{"type":"tool-result","toolCallId":"call_00_Cijldc88LYmVPCXYUsRq1617","content":[{"type":"text","text":"{\"answers\":[{\"id\":\"color\",\"selected\":[\"Blue\"],\"custom\":\"Include accessibility notes\"}]}"}],"isError":false}],"role":"user","id":"c69ef39f-6f62-439f-b3f8-e8d10fba572f"}},"sourceEventSeqs":[132],"surfaceOp":"append"}
{"type":"step/end","seq":134,"time":1785150169790,"data":{"turn":1,"step":1}}
{"type":"step/start","seq":135,"time":1785150169790,"data":{"turn":1,"step":2}}
{"type":"assistant/chunk","seq":136,"time":1785150170605,"data":{"turn":1,"step":2,"chunk":{"type":"block-start","index":0,"blockType":"reasoning"}}}

6
apps/web/tests/snapshots/question-composer/ui.expected.md
View File

@@ -3,9 +3,9 @@
- heading "Which color do you prefer?" [level=2]
- button "Dismiss all questions":
- img
- radiogroup:
- radio "Blue": 1 Blue A cool recessive hue that reads as calm and trustworthy in long reading sessions and dense dashboards.
- radio "Green": 2 Green A restful mid-spectrum hue with the highest perceived brightness, easiest on the eye over long sessions.
- group:
- checkbox "Blue": Blue A cool recessive hue that reads as calm and trustworthy in long reading sessions and dense dashboards.
- checkbox "Green": Green A restful mid-spectrum hue with the highest perceived brightness, easiest on the eye over long sessions.
- textbox "Type your answer"
- button "Previous question" [disabled]:
- img