test(web): pin every scenario end-state with an aria golden
Every spec now commits at least one golden and the interactive ones one per distinct end-state (nine new .expected.md): - live-interactions: cancel.expected.md (frozen partial + 已停止 marker), error-auth.expected.md (the prompt bubble alone — the committed artifact of the web-error-surface gap, the diff that flips when error rendering lands), retry.expected.md (indistinguishable from a clean completion — retries are deliberately invisible in the transcript). - question-composer: answered.expected.md (the question resolved into its tool round trip plus the final reply, takeover gone) beside the existing waiting-state golden. - steering: mid-steer.expected.md pins the accepted-but-INVISIBLE state (the loop drains steering only at the step boundary, so no interjection bubble exists while the question still blocks — if the client ever renders pending steers eagerly, this golden flips first) and settled.expected.md the badged bubble plus obeying reply. - navigation-panes: waterfall.expected.md and details-open.expected.md (tool-name header, Input args, Output result) beside the trajectory one. - lifecycle-chrome: reloaded.expected.md — rendering the same settled transcript from persistence alone IS the recovery claim. Fixture inventories extended to the new closed sets; the Agent Note's expected-outputs policy updated in both languages (per-end-state goldens for interactive scenarios), pairing re-recorded.
This commit is contained in:
@@ -25,6 +25,9 @@ import { saveFailureShot } from './support.ts'
|
||||
const SNAPSHOT_DIR = fileURLToPath(new URL('./snapshots/lifecycle-chrome', import.meta.url))
|
||||
const FIXTURE = join(SNAPSHOT_DIR, 'session.jsonl')
|
||||
const HERO_EXPECTED = join(SNAPSHOT_DIR, 'hero.expected.md')
|
||||
// Post-reload golden: the same settled conversation rebuilt purely from
|
||||
// persistence + history — byte-equal rendering is exactly the recovery claim.
|
||||
const RELOADED_EXPECTED = join(SNAPSHOT_DIR, 'reloaded.expected.md')
|
||||
const MODE = webSnapshotMode()
|
||||
|
||||
const PROMPT = 'Reply with the single word LIGHTHOUSE and stop.'
|
||||
@@ -112,6 +115,10 @@ describe('web e2e: lifecycle & chrome (workspace flow / reload / dark mode)', ()
|
||||
// Expand back and confirm the tree still lists the materialized session.
|
||||
await page.getByRole('button', { name: 'Open sidebar' }).click()
|
||||
await expect.poll(() => page.locator('[role="treeitem"][aria-selected="true"]').count(), { timeout: 10_000 }).toBe(1)
|
||||
// Golden of the recovered conversation region: rebuilt from the log, it
|
||||
// must render the same settled transcript the live turn produced.
|
||||
const snapshot = await captureStableAria(page, '[class*="centerCol"]', scaffold.workspaceCwd)
|
||||
await compareOrRefreshGolden(RELOADED_EXPECTED, snapshot, MODE)
|
||||
expect(tripwire.pageErrors).toEqual([])
|
||||
}, 90_000)
|
||||
|
||||
@@ -147,6 +154,6 @@ describe('web e2e: lifecycle & chrome (workspace flow / reload / dark mode)', ()
|
||||
}, 60_000)
|
||||
|
||||
it.skipIf(MODE === 'record')('keeps the fixture inventory closed', async () => {
|
||||
await assertFixtureInventory(SNAPSHOT_DIR, ['session.jsonl', 'hero.expected.md'])
|
||||
await assertFixtureInventory(SNAPSHOT_DIR, ['session.jsonl', 'hero.expected.md', 'reloaded.expected.md'])
|
||||
})
|
||||
})
|
||||
|
||||
@@ -20,13 +20,20 @@ import { deriveReplayScript, parseSessionLog } from '@deepseek-ai/dsh-llm-replay
|
||||
import type { ReplayOverrideDoc } from '@deepseek-ai/dsh-llm-replay'
|
||||
import type { SessionEvent } from '@deepseek-ai/dsh-session'
|
||||
import {
|
||||
assertFixtureInventory, fixtureUserPrompts, launchWebScaffold, recordFixture,
|
||||
watchConsole, webSnapshotMode, type WebScaffold,
|
||||
assertFixtureInventory, captureStableAria, compareOrRefreshGolden, fixtureUserPrompts,
|
||||
launchWebScaffold, recordFixture, watchConsole, webSnapshotMode, type WebScaffold,
|
||||
} from './scaffold.ts'
|
||||
import { saveFailureShot } from './support.ts'
|
||||
|
||||
const SNAPSHOT_DIR = fileURLToPath(new URL('./snapshots/live-interactions', import.meta.url))
|
||||
const FIXTURE = join(SNAPSHOT_DIR, 'session.jsonl')
|
||||
// One golden per interactive end-state: what the user is left looking at
|
||||
// after cancel, after a non-retryable failure (pins the FIXME(web-error-surface)
|
||||
// gap as a reviewable artifact: NO error copy in the tree), and after retry
|
||||
// recovery — three genuinely different terminal surfaces of one fixture.
|
||||
const CANCEL_EXPECTED = join(SNAPSHOT_DIR, 'cancel.expected.md')
|
||||
const ERROR_EXPECTED = join(SNAPSHOT_DIR, 'error-auth.expected.md')
|
||||
const RETRY_EXPECTED = join(SNAPSHOT_DIR, 'retry.expected.md')
|
||||
const MODE = webSnapshotMode()
|
||||
|
||||
// The recorded base: one text-only turn whose derived script the sidecars
|
||||
@@ -123,6 +130,10 @@ describe('web e2e: live-turn interactions (cancel / error / retry)', () => {
|
||||
// Composer recovered; no streaming node lingers.
|
||||
await expect.poll(() => page.locator('textarea').first().isEnabled(), { timeout: 10_000 }).toBe(true)
|
||||
expect(await page.locator('[data-streaming="true"]').count()).toBe(0)
|
||||
// Golden of the aborted end-state: the prompt bubble plus the frozen
|
||||
// partial ('partial' is the hang entry's replayed prefix) and no more.
|
||||
const snapshot = await captureStableAria(page, '[class*="centerCol"]', scaffold!.workspaceCwd)
|
||||
await compareOrRefreshGolden(CANCEL_EXPECTED, snapshot, MODE)
|
||||
expect(tripwire.pageErrors).toEqual([])
|
||||
}, 120_000)
|
||||
|
||||
@@ -144,6 +155,10 @@ describe('web e2e: live-turn interactions (cancel / error / retry)', () => {
|
||||
// "no crash, composer recovers, turn logged as error".
|
||||
await expect.poll(() => page.locator('textarea').first().isEnabled(), { timeout: 10_000 }).toBe(true)
|
||||
expect(await page.locator('[data-streaming="true"]').count()).toBe(0)
|
||||
// Golden of the same gap: the prompt bubble alone, no error copy in the
|
||||
// tree — the diff that changes when web-error-surface lands.
|
||||
const snapshot = await captureStableAria(page, '[class*="centerCol"]', scaffold!.workspaceCwd)
|
||||
await compareOrRefreshGolden(ERROR_EXPECTED, snapshot, MODE)
|
||||
expect(tripwire.pageErrors).toEqual([])
|
||||
}, 120_000)
|
||||
|
||||
@@ -167,10 +182,16 @@ describe('web e2e: live-turn interactions (cancel / error / retry)', () => {
|
||||
// only on change, so attempt count is invisible there).
|
||||
expect(sessionEvents.filter(e => e.type === 'llm/retry').length).toBeGreaterThanOrEqual(1)
|
||||
await expect.poll(() => page.getByText('event sourcing', { exact: false }).count(), { timeout: 10_000 }).toBeGreaterThan(0)
|
||||
// Golden of the recovered end-state: indistinguishable from a clean
|
||||
// completion — retries are deliberately invisible in the transcript.
|
||||
const snapshot = await captureStableAria(page, '[class*="centerCol"]', scaffold!.workspaceCwd)
|
||||
await compareOrRefreshGolden(RETRY_EXPECTED, snapshot, MODE)
|
||||
expect(tripwire.pageErrors).toEqual([])
|
||||
}, 120_000)
|
||||
|
||||
it.skipIf(MODE === 'record')('keeps the fixture inventory closed', async () => {
|
||||
await assertFixtureInventory(SNAPSHOT_DIR, ['session.jsonl'])
|
||||
await assertFixtureInventory(SNAPSHOT_DIR, [
|
||||
'session.jsonl', 'cancel.expected.md', 'error-auth.expected.md', 'retry.expected.md',
|
||||
])
|
||||
})
|
||||
})
|
||||
|
||||
@@ -24,6 +24,8 @@ import { saveFailureShot } from './support.ts'
|
||||
const SNAPSHOT_DIR = fileURLToPath(new URL('./snapshots/navigation-panes', import.meta.url))
|
||||
const SEED = join(SNAPSHOT_DIR, 'seed.jsonl')
|
||||
const TRAJECTORY_EXPECTED = join(SNAPSHOT_DIR, 'trajectory.expected.md')
|
||||
const WATERFALL_EXPECTED = join(SNAPSHOT_DIR, 'waterfall.expected.md')
|
||||
const DETAILS_EXPECTED = join(SNAPSHOT_DIR, 'details-open.expected.md')
|
||||
const MODE = webSnapshotMode()
|
||||
const SEED_ID = 'navigation-panes-web-e2e'
|
||||
|
||||
@@ -148,6 +150,9 @@ describe('web e2e: navigation & panes over a rich seeded session', () => {
|
||||
for (const tag of ['turn 0', 'turn 1', 'turn 2']) {
|
||||
await expect.poll(() => page.getByText(tag, { exact: true }).count(), { timeout: 10_000 }).toBe(1)
|
||||
}
|
||||
const snapshot = (await captureStableAria(page, '[class*="viewArea"]', scaffold.workspaceCwd))
|
||||
.split(SEED_ID).join('{{seededId}}')
|
||||
await compareOrRefreshGolden(WATERFALL_EXPECTED, snapshot, MODE)
|
||||
}, 60_000)
|
||||
|
||||
it.skipIf(MODE === 'record')('opens the details column from the bash row and closes it', async () => {
|
||||
@@ -167,6 +172,10 @@ describe('web e2e: navigation & panes over a rich seeded session', () => {
|
||||
// The open panel shows the selected call's name, arguments, and durable
|
||||
// result (NAVIGATION_OK appears in the chat row too, hence >= 2 total).
|
||||
await expect.poll(() => page.getByText('NAVIGATION_OK', { exact: false }).count(), { timeout: 10_000 }).toBeGreaterThanOrEqual(2)
|
||||
// Golden of the open panel: tool name header, Input args, Output result.
|
||||
const snapshot = (await captureStableAria(page, '[class*="detailsCol"]', scaffold.workspaceCwd))
|
||||
.split(SEED_ID).join('{{seededId}}')
|
||||
await compareOrRefreshGolden(DETAILS_EXPECTED, snapshot, MODE)
|
||||
await page.getByRole('button', { name: '关闭详情' }).click()
|
||||
await expect.poll(() => frame.getAttribute('data-details-collapsed'), { timeout: 10_000 }).not.toBeNull()
|
||||
}, 60_000)
|
||||
@@ -174,6 +183,8 @@ describe('web e2e: navigation & panes over a rich seeded session', () => {
|
||||
it.skipIf(MODE === 'record')('issued zero model calls and stayed clean', async () => {
|
||||
expect(tripwire.pageErrors).toEqual([])
|
||||
expect(tripwire.warnings).toEqual([])
|
||||
await assertFixtureInventory(SNAPSHOT_DIR, ['seed.jsonl', 'trajectory.expected.md'])
|
||||
await assertFixtureInventory(SNAPSHOT_DIR, [
|
||||
'seed.jsonl', 'trajectory.expected.md', 'waterfall.expected.md', 'details-open.expected.md',
|
||||
])
|
||||
})
|
||||
})
|
||||
|
||||
@@ -23,6 +23,9 @@ import { saveFailureShot } from './support.ts'
|
||||
const SNAPSHOT_DIR = fileURLToPath(new URL('./snapshots/question-composer', import.meta.url))
|
||||
const FIXTURE = join(SNAPSHOT_DIR, 'session.jsonl')
|
||||
const UI_EXPECTED = join(SNAPSHOT_DIR, 'ui.expected.md')
|
||||
// Second golden: the answered transcript — the question resolved into its
|
||||
// tool round trip and the final reply, the state the waiting golden cannot see.
|
||||
const ANSWERED_EXPECTED = join(SNAPSHOT_DIR, 'answered.expected.md')
|
||||
const MODE = webSnapshotMode()
|
||||
|
||||
const PROMPT = 'Use the ask_user_question tool to ask me exactly one question with id "color", question "Which color do you prefer?", header "Pick one", and options labeled "Blue" and "Green". After I answer, reply with the single word DONE and stop.'
|
||||
@@ -90,10 +93,14 @@ describe('web e2e: resident question composer round trip', () => {
|
||||
// Composer gone; regular input restored.
|
||||
expect(await page.locator('[data-question-key]').count()).toBe(0)
|
||||
await expect.poll(() => page.locator('textarea').first().isEnabled(), { timeout: 10_000 }).toBe(true)
|
||||
// Golden of the answered transcript: the ask_user_question round trip
|
||||
// rendered as history (question tool row + DONE), composer takeover gone.
|
||||
const snapshot = await captureStableAria(page, '[class*="centerCol"]', scaffold.workspaceCwd)
|
||||
await compareOrRefreshGolden(ANSWERED_EXPECTED, snapshot, MODE)
|
||||
expect(tripwire.pageErrors).toEqual([])
|
||||
}, 200_000)
|
||||
|
||||
it.skipIf(MODE === 'record')('keeps the fixture inventory closed', async () => {
|
||||
await assertFixtureInventory(SNAPSHOT_DIR, ['session.jsonl', 'ui.expected.md'])
|
||||
await assertFixtureInventory(SNAPSHOT_DIR, ['session.jsonl', 'ui.expected.md', 'answered.expected.md'])
|
||||
})
|
||||
})
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
- banner:
|
||||
- navigation "Session hierarchy":
|
||||
- button "Reply with the single word" [disabled]
|
||||
- text: · 1 turns
|
||||
- tablist:
|
||||
- tab "Chat" [selected]
|
||||
- tab "Trajectory"
|
||||
- tab "Waterfall"
|
||||
- text: Reply with the single word LIGHTHOUSE and stop.
|
||||
- button "Think The user wants me to reply with a single word. Let me comply.":
|
||||
- img
|
||||
- text: Think The user wants me to reply with a single word. Let me comply.
|
||||
- paragraph: LIGHTHOUSE
|
||||
- text: cache hit 99% · 7,810 tokens · 1 turns · 1 steps
|
||||
- textbox "Message the agent"
|
||||
- button "Add attachment":
|
||||
- img
|
||||
- combobox "Plan mode":
|
||||
- option "Plan" [selected]
|
||||
- option "Agent"
|
||||
- combobox "Access mode":
|
||||
- option "Read-only" [selected]
|
||||
- option "Read-write"
|
||||
- combobox "Model":
|
||||
- option "DeepSeek-V4-Pro High" [selected]
|
||||
- option "DeepSeek-V4-Pro"
|
||||
- button "Send message" [disabled]
|
||||
@@ -0,0 +1,24 @@
|
||||
- banner:
|
||||
- navigation "Session hierarchy":
|
||||
- button "Reply with a one-sentence description" [disabled]
|
||||
- text: · 1 turns
|
||||
- tablist:
|
||||
- tab "Chat" [selected]
|
||||
- tab "Trajectory"
|
||||
- tab "Waterfall"
|
||||
- text: Reply with a one-sentence description of event sourcing, then stop.
|
||||
- paragraph: partial
|
||||
- text: 已停止 0 tokens · 1 turns · 1 steps
|
||||
- textbox "Message the agent"
|
||||
- button "Add attachment":
|
||||
- img
|
||||
- combobox "Plan mode":
|
||||
- option "Plan" [selected]
|
||||
- option "Agent"
|
||||
- combobox "Access mode":
|
||||
- option "Read-only" [selected]
|
||||
- option "Read-write"
|
||||
- combobox "Model":
|
||||
- option "DeepSeek-V4-Pro High" [selected]
|
||||
- option "DeepSeek-V4-Pro"
|
||||
- button "Send message" [disabled]
|
||||
@@ -0,0 +1,22 @@
|
||||
- banner:
|
||||
- navigation "Session hierarchy":
|
||||
- button "Reply with a one-sentence description" [disabled]
|
||||
- text: · 1 turns
|
||||
- tablist:
|
||||
- tab "Chat" [selected]
|
||||
- tab "Trajectory"
|
||||
- tab "Waterfall"
|
||||
- text: Reply with a one-sentence description of event sourcing, then stop.
|
||||
- textbox "Message the agent"
|
||||
- button "Add attachment":
|
||||
- img
|
||||
- combobox "Plan mode":
|
||||
- option "Plan" [selected]
|
||||
- option "Agent"
|
||||
- combobox "Access mode":
|
||||
- option "Read-only" [selected]
|
||||
- option "Read-write"
|
||||
- combobox "Model":
|
||||
- option "DeepSeek-V4-Pro High" [selected]
|
||||
- option "DeepSeek-V4-Pro"
|
||||
- button "Send message" [disabled]
|
||||
27
apps/web/tests/snapshots/live-interactions/retry.expected.md
Normal file
27
apps/web/tests/snapshots/live-interactions/retry.expected.md
Normal file
@@ -0,0 +1,27 @@
|
||||
- banner:
|
||||
- navigation "Session hierarchy":
|
||||
- button "Reply with a one-sentence description" [disabled]
|
||||
- text: · 1 turns
|
||||
- tablist:
|
||||
- tab "Chat" [selected]
|
||||
- tab "Trajectory"
|
||||
- tab "Waterfall"
|
||||
- text: Reply with a one-sentence description of event sourcing, then stop.
|
||||
- button "Think The user is asking for a one-sentence description of event sourcing. This is a straightforward knowledge question that doesn't require any skill loading or tool calls.":
|
||||
- img
|
||||
- text: Think The user is asking for a one-sentence description of event sourcing. This is a straightforward knowledge question that doesn't require any skill loading or tool calls.
|
||||
- paragraph: Event sourcing is a pattern where all changes to an application's state are stored as an immutable, append-only sequence of events, rather than persisting only the current state, enabling full auditability, temporal queries, and event-driven architectures.
|
||||
- text: cache hit 99% · 7,869 tokens · 1 turns · 1 steps
|
||||
- textbox "Message the agent"
|
||||
- button "Add attachment":
|
||||
- img
|
||||
- combobox "Plan mode":
|
||||
- option "Plan" [selected]
|
||||
- option "Agent"
|
||||
- combobox "Access mode":
|
||||
- option "Read-only" [selected]
|
||||
- option "Read-write"
|
||||
- combobox "Model":
|
||||
- option "DeepSeek-V4-Pro High" [selected]
|
||||
- option "DeepSeek-V4-Pro"
|
||||
- button "Send message" [disabled]
|
||||
@@ -0,0 +1,3 @@
|
||||
- text: bash
|
||||
- button "关闭详情"
|
||||
- text: "Input { \"command\": \"echo NAVIGATION_OK\", \"description\": \"Print NAVIGATION_OK\" } Output NAVIGATION_OK"
|
||||
@@ -0,0 +1 @@
|
||||
- text: 3 turns · 3 steps · 3 tool calls turn 0 turn 1 turn 2
|
||||
@@ -0,0 +1,33 @@
|
||||
- banner:
|
||||
- navigation "Session hierarchy":
|
||||
- button "Use the ask_user_question tool to" [disabled]
|
||||
- text: · 1 turns
|
||||
- tablist:
|
||||
- tab "Chat" [selected]
|
||||
- tab "Trajectory"
|
||||
- tab "Waterfall"
|
||||
- text: Use the ask_user_question tool to ask me exactly one question with id "color", question "Which color do you prefer?", header "Pick one", and options labeled "Blue" and "Green". After I answer, reply with the single word DONE and stop.
|
||||
- button "Think The user wants me to use the ask_user_question tool to ask a specific question with id \"color\", question \"Which color do you prefer?\", header \"Pick one\", and options labeled \"Blue\" and \"Green\". Let me do exactly that.":
|
||||
- img
|
||||
- text: Think The user wants me to use the ask_user_question tool to ask a specific question with id "color", question "Which color do you prefer?", header "Pick one", and options labeled "Blue" and "Green". Let me do exactly that.
|
||||
- button:
|
||||
- img
|
||||
- text: "Tool call ask_user_question · {\"questions\": [{\"id\": \"color\", \"question\": \"Which color do you prefer?\", \"header\": \"Pick one\", \"options\": [{\"label\": \"Blue\"}, {\"label\": \"Green\"}]}]}"
|
||||
- button "Think The user answered \"Blue\". I need to reply with the single word DONE and stop.":
|
||||
- img
|
||||
- text: Think The user answered "Blue". I need to reply with the single word DONE and stop.
|
||||
- paragraph: DONE
|
||||
- text: cache hit 99% · 15,978 tokens · 1 turns · 2 steps
|
||||
- textbox "Message the agent"
|
||||
- button "Add attachment":
|
||||
- img
|
||||
- combobox "Plan mode":
|
||||
- option "Plan" [selected]
|
||||
- option "Agent"
|
||||
- combobox "Access mode":
|
||||
- option "Read-only" [selected]
|
||||
- option "Read-write"
|
||||
- combobox "Model":
|
||||
- option "DeepSeek-V4-Pro High" [selected]
|
||||
- option "DeepSeek-V4-Pro"
|
||||
- button "Send message" [disabled]
|
||||
39
apps/web/tests/snapshots/steering/mid-steer.expected.md
Normal file
39
apps/web/tests/snapshots/steering/mid-steer.expected.md
Normal file
@@ -0,0 +1,39 @@
|
||||
- banner:
|
||||
- navigation "Session hierarchy":
|
||||
- button "Use the ask_user_question tool to" [disabled]
|
||||
- text: · 1 turns
|
||||
- tablist:
|
||||
- tab "Chat" [selected]
|
||||
- tab "Trajectory"
|
||||
- tab "Waterfall"
|
||||
- text: Use the ask_user_question tool to ask me exactly one question with id "checkpoint", question "Ready to continue?", header "Checkpoint", and options labeled "Yes" and "No". After I answer, reply with one short sentence acknowledging my answer and stop.
|
||||
- button "Think The user wants me to use the ask_user_question tool to ask them a specific question with the given parameters. Let me do exactly that.":
|
||||
- img
|
||||
- text: Think The user wants me to use the ask_user_question tool to ask them a specific question with the given parameters. Let me do exactly that.
|
||||
- button
|
||||
- text: "Tool call ask_user_question · {\"questions\": [{\"id\": \"checkpoint\", \"question\": \"Ready to continue?\", \"header\": \"Checkpoint\", \"options\": [{\"label\": \"Yes\"}, {\"label\": \"No\"}]}]} 等待回答(1 题)"
|
||||
- button "▸ 问题内容"
|
||||
- text: 请在原客户端处理(web 端作答后续里程碑提供) cache hit 98% · 7,946 tokens · 1 turns · 1 steps
|
||||
- region "Ready to continue?":
|
||||
- text: Checkpoint
|
||||
- heading "Ready to continue?" [level=2]
|
||||
- text: 1 / 1
|
||||
- button "上一题" [disabled]:
|
||||
- img
|
||||
- button "下一题" [disabled]:
|
||||
- img
|
||||
- button "放弃整组问题":
|
||||
- img
|
||||
- radiogroup:
|
||||
- radio "Yes":
|
||||
- text: 1 Yes
|
||||
- img
|
||||
- radio "No":
|
||||
- text: 2 No
|
||||
- img
|
||||
- button "其他,请填写自定义答案":
|
||||
- img
|
||||
- text: 其他,请填写自定义答案
|
||||
- status
|
||||
- button "跳过本题"
|
||||
- button "提交" [disabled]
|
||||
33
apps/web/tests/snapshots/steering/settled.expected.md
Normal file
33
apps/web/tests/snapshots/steering/settled.expected.md
Normal file
@@ -0,0 +1,33 @@
|
||||
- banner:
|
||||
- navigation "Session hierarchy":
|
||||
- button "Use the ask_user_question tool to" [disabled]
|
||||
- text: · 1 turns
|
||||
- tablist:
|
||||
- tab "Chat" [selected]
|
||||
- tab "Trajectory"
|
||||
- tab "Waterfall"
|
||||
- text: Use the ask_user_question tool to ask me exactly one question with id "checkpoint", question "Ready to continue?", header "Checkpoint", and options labeled "Yes" and "No". After I answer, reply with one short sentence acknowledging my answer and stop.
|
||||
- button "Think The user wants me to use the ask_user_question tool to ask them a specific question with the given parameters. Let me do exactly that.":
|
||||
- img
|
||||
- text: Think The user wants me to use the ask_user_question tool to ask them a specific question with the given parameters. Let me do exactly that.
|
||||
- button:
|
||||
- img
|
||||
- text: "Tool call ask_user_question · {\"questions\": [{\"id\": \"checkpoint\", \"question\": \"Ready to continue?\", \"header\": \"Checkpoint\", \"options\": [{\"label\": \"Yes\"}, {\"label\": \"No\"}]}]} 插话 Interjection: include the word BANANA in your final reply."
|
||||
- button "Think The user selected \"Yes\" and wants me to include the word \"BANANA\" in my final reply. Let me acknowledge their answer.":
|
||||
- img
|
||||
- text: Think The user selected "Yes" and wants me to include the word "BANANA" in my final reply. Let me acknowledge their answer.
|
||||
- paragraph: Great, let's move forward. BANANA!
|
||||
- text: cache hit 98% · 15,967 tokens · 1 turns · 2 steps
|
||||
- textbox "Message the agent"
|
||||
- button "Add attachment":
|
||||
- img
|
||||
- combobox "Plan mode":
|
||||
- option "Plan" [selected]
|
||||
- option "Agent"
|
||||
- combobox "Access mode":
|
||||
- option "Read-only" [selected]
|
||||
- option "Read-write"
|
||||
- combobox "Model":
|
||||
- option "DeepSeek-V4-Pro High" [selected]
|
||||
- option "DeepSeek-V4-Pro"
|
||||
- button "Send message" [disabled]
|
||||
@@ -20,13 +20,22 @@ import { afterAll, beforeAll, describe, expect, it, onTestFailed } from 'vitest'
|
||||
import { parseSessionLog } from '@deepseek-ai/dsh-llm-replay'
|
||||
import type { SessionEvent } from '@deepseek-ai/dsh-session'
|
||||
import {
|
||||
assertFixtureInventory, fixtureUserPrompts, launchWebScaffold, recordFixture,
|
||||
watchConsole, webSnapshotMode, type WebScaffold,
|
||||
assertFixtureInventory, captureStableAria, compareOrRefreshGolden, fixtureUserPrompts,
|
||||
launchWebScaffold, recordFixture, watchConsole, webSnapshotMode, type WebScaffold,
|
||||
} from './scaffold.ts'
|
||||
import { saveFailureShot } from './support.ts'
|
||||
|
||||
const SNAPSHOT_DIR = fileURLToPath(new URL('./snapshots/steering', import.meta.url))
|
||||
const FIXTURE = join(SNAPSHOT_DIR, 'session.jsonl')
|
||||
// Two goldens for the two distinct states this interaction produces: the
|
||||
// mid-turn moment (steer ACCEPTED but deliberately invisible — the loop
|
||||
// drains steering at the step boundary, so no interjection bubble exists
|
||||
// while the question still blocks the step) and the settled transcript
|
||||
// (badged bubble in place, final reply obeying it). The pair pins the
|
||||
// timing semantics visually: if the client ever starts rendering pending
|
||||
// steers eagerly, the mid-steer golden flips first.
|
||||
const MID_EXPECTED = join(SNAPSHOT_DIR, 'mid-steer.expected.md')
|
||||
const SETTLED_EXPECTED = join(SNAPSHOT_DIR, 'settled.expected.md')
|
||||
const MODE = webSnapshotMode()
|
||||
|
||||
const PROMPT = 'Use the ask_user_question tool to ask me exactly one question with id "checkpoint", question "Ready to continue?", header "Checkpoint", and options labeled "Yes" and "No". After I answer, reply with one short sentence acknowledging my answer and stop.'
|
||||
@@ -104,6 +113,17 @@ describe('web e2e: mid-turn steering lands durably and visibly', () => {
|
||||
}, { sessionId: liveSessionId!, text: STEER })
|
||||
expect(reply.result?.ok).toBe(true)
|
||||
|
||||
if (MODE !== 'record') {
|
||||
// Mid-turn golden: the ACCEPTED steer is durable in the inbox but the
|
||||
// loop drains steering only at the step boundary, so no steering/message
|
||||
// exists yet and no interjection bubble renders — the composer still
|
||||
// blocks, alone. The DOM is stable here (no further SSE frames can
|
||||
// arrive until the question is answered), making this state capturable.
|
||||
expect(await page.getByText('插话').count()).toBe(0)
|
||||
const snapshot = await captureStableAria(page, '[class*="centerCol"]', scaffold.workspaceCwd)
|
||||
await compareOrRefreshGolden(MID_EXPECTED, snapshot, MODE)
|
||||
}
|
||||
|
||||
// Answer the composer; the tool result closes the step, the loop drains
|
||||
// the steer as steering/message, and the steered continuation runs the
|
||||
// final model call.
|
||||
@@ -137,10 +157,14 @@ describe('web e2e: mid-turn steering lands durably and visibly', () => {
|
||||
await expect.poll(() => page.getByText('Interjection:', { exact: false }).count(), { timeout: 10_000 }).toBe(1)
|
||||
await expect.poll(() => page.getByText('BANANA', { exact: false }).count(), { timeout: 10_000 }).toBeGreaterThanOrEqual(2)
|
||||
expect(await page.locator('[data-question-key]').count()).toBe(0)
|
||||
// Settled golden: badge + interjection between the question round trip
|
||||
// and the obeying reply, composer takeover gone.
|
||||
const snapshot = await captureStableAria(page, '[class*="centerCol"]', scaffold.workspaceCwd)
|
||||
await compareOrRefreshGolden(SETTLED_EXPECTED, snapshot, MODE)
|
||||
expect(tripwire.pageErrors).toEqual([])
|
||||
}, 200_000)
|
||||
|
||||
it.skipIf(MODE === 'record')('keeps the fixture inventory closed', async () => {
|
||||
await assertFixtureInventory(SNAPSHOT_DIR, ['session.jsonl'])
|
||||
await assertFixtureInventory(SNAPSHOT_DIR, ['session.jsonl', 'mid-steer.expected.md', 'settled.expected.md'])
|
||||
})
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user