From ea1d8d06b36788be8407ec439325d525ac30041e Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Sun, 26 Jul 2026 12:45:44 +0800 Subject: [PATCH] test(web): pin every scenario end-state with an aria golden MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every spec now commits at least one golden and the interactive ones one per distinct end-state (nine new .expected.md): - live-interactions: cancel.expected.md (frozen partial + 已停止 marker), error-auth.expected.md (the prompt bubble alone — the committed artifact of the web-error-surface gap, the diff that flips when error rendering lands), retry.expected.md (indistinguishable from a clean completion — retries are deliberately invisible in the transcript). - question-composer: answered.expected.md (the question resolved into its tool round trip plus the final reply, takeover gone) beside the existing waiting-state golden. - steering: mid-steer.expected.md pins the accepted-but-INVISIBLE state (the loop drains steering only at the step boundary, so no interjection bubble exists while the question still blocks — if the client ever renders pending steers eagerly, this golden flips first) and settled.expected.md the badged bubble plus obeying reply. - navigation-panes: waterfall.expected.md and details-open.expected.md (tool-name header, Input args, Output result) beside the trajectory one. - lifecycle-chrome: reloaded.expected.md — rendering the same settled transcript from persistence alone IS the recovery claim. Fixture inventories extended to the new closed sets; the Agent Note's expected-outputs policy updated in both languages (per-end-state goldens for interactive scenarios), pairing re-recorded. --- ...6-07-24-web-gui-browser-e2e-lane.i18n.yaml | 4 +- .../2026-07-24-web-gui-browser-e2e-lane.md | 14 +++---- .../2026-07-24-web-gui-browser-e2e-lane.zh.md | 14 +++---- apps/web/tests/lifecycle-chrome.e2e.ts | 9 ++++- apps/web/tests/live-interactions.e2e.ts | 27 +++++++++++-- apps/web/tests/navigation-panes.e2e.ts | 13 ++++++- apps/web/tests/question-composer.e2e.ts | 9 ++++- .../lifecycle-chrome/reloaded.expected.md | 27 +++++++++++++ .../live-interactions/cancel.expected.md | 24 ++++++++++++ .../live-interactions/error-auth.expected.md | 22 +++++++++++ .../live-interactions/retry.expected.md | 27 +++++++++++++ .../navigation-panes/details-open.expected.md | 3 ++ .../navigation-panes/waterfall.expected.md | 1 + .../question-composer/answered.expected.md | 33 ++++++++++++++++ .../snapshots/steering/mid-steer.expected.md | 39 +++++++++++++++++++ .../snapshots/steering/settled.expected.md | 33 ++++++++++++++++ apps/web/tests/steering.e2e.ts | 30 ++++++++++++-- 17 files changed, 304 insertions(+), 25 deletions(-) create mode 100644 apps/web/tests/snapshots/lifecycle-chrome/reloaded.expected.md create mode 100644 apps/web/tests/snapshots/live-interactions/cancel.expected.md create mode 100644 apps/web/tests/snapshots/live-interactions/error-auth.expected.md create mode 100644 apps/web/tests/snapshots/live-interactions/retry.expected.md create mode 100644 apps/web/tests/snapshots/navigation-panes/details-open.expected.md create mode 100644 apps/web/tests/snapshots/navigation-panes/waterfall.expected.md create mode 100644 apps/web/tests/snapshots/question-composer/answered.expected.md create mode 100644 apps/web/tests/snapshots/steering/mid-steer.expected.md create mode 100644 apps/web/tests/snapshots/steering/settled.expected.md diff --git a/.agents/notes/implemented/testing/2026-07-24-web-gui-browser-e2e-lane.i18n.yaml b/.agents/notes/implemented/testing/2026-07-24-web-gui-browser-e2e-lane.i18n.yaml index bf6ca9d0d7..3745347bea 100644 --- a/.agents/notes/implemented/testing/2026-07-24-web-gui-browser-e2e-lane.i18n.yaml +++ b/.agents/notes/implemented/testing/2026-07-24-web-gui-browser-e2e-lane.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write -2026-07-24-web-gui-browser-e2e-lane.md: 88730cdecf527ece8033ddab1151afcbc6edd83f -2026-07-24-web-gui-browser-e2e-lane.zh.md: 9850023a49a860a8f4bbdacc8c48fc389ec77210 +2026-07-24-web-gui-browser-e2e-lane.md: cc9b1606a62cfbb2322a4c4647d809dfd809b117 +2026-07-24-web-gui-browser-e2e-lane.zh.md: 3ab0f3716affef6f1446e50d237d74486161afb1 diff --git a/.agents/notes/implemented/testing/2026-07-24-web-gui-browser-e2e-lane.md b/.agents/notes/implemented/testing/2026-07-24-web-gui-browser-e2e-lane.md index 88730cdecf..cc9b1606a6 100644 --- a/.agents/notes/implemented/testing/2026-07-24-web-gui-browser-e2e-lane.md +++ b/.agents/notes/implemented/testing/2026-07-24-web-gui-browser-e2e-lane.md @@ -32,23 +32,23 @@ Every scenario fails on any pageerror and on the client's connection-loss/gap-re ### Expected outputs -One committed golden per scenario: a normalized `ariaSnapshot()` of the conversation region (`ui.expected.md`) — uuid/cwd/workspace-basename/duration tokens normalized, captured poll-until-equal at the settled milestone — plus a few role/text anchor assertions that stay green under a semantics-preserving component rewrite while the golden churns reviewably. The aria tree is the mechanization of the client rule "assert what the user would see, never class names". World-state assertions ride root-context session events inline (which tool call produced which durable result, whether `turn/end` completed) instead of a second committed log golden: the persisted-log surface is pinned by the ACP/headless/TUI suites through the same loop and persistence, and re-pinning it here would double refresh cost against the tier discipline. `refresh` is the sole golden writer — a missing golden in replay mode fails with the healing command rather than self-bootstrapping. +At least one committed golden per scenario, and one per DISTINCT end-state for the interactive scenarios (cancel/error/retry, waiting/answered, mid-steer/settled, panel-open, post-reload): a normalized `ariaSnapshot()` of the scenario's owning region — uuid/cwd/workspace-basename/duration tokens normalized, captured poll-until-equal at the settled milestone — plus a few role/text anchor assertions that stay green under a semantics-preserving component rewrite while the golden churns reviewably. The aria tree is the mechanization of the client rule "assert what the user would see, never class names". World-state assertions ride root-context session events inline (which tool call produced which durable result, whether `turn/end` completed) instead of a second committed log golden: the persisted-log surface is pinned by the ACP/headless/TUI suites through the same loop and persistence, and re-pinning it here would double refresh cost against the tier discipline. `refresh` is the sole golden writer — a missing golden in replay mode fails with the healing command rather than self-bootstrapping. The typecheck plane split is structural: the three files that boot the host spine (`scaffold`, `replay-round-trip.e2e`, and `seeded-history.e2e`) are excluded from the client-registered `apps/web` project. Those files and their shared `support.ts` are included file-by-file in `tsconfig.host.json` — one program cannot hold both sides of the cordis `Context` merges. ### Modes and fixtures -`DSH_SNAPSHOT` selects replay (default, keyless), record (with key), or refresh (keyless) as inline spec branches — the TUI shape, not a suite factory: at two scenarios the acp-snapshot factory machinery has no owner, and the genuinely shared parts are already exported (`scrubRequestHeaders`, `parseSessionLog`, `installLlmReplay`). Each spec splits into drive steps (type, send, `whenTurnSettled` — run in all modes, never waiting on model-content selectors, so record cannot hang on a live model answering differently) and assertion steps (replay/refresh only). Record = drive live through the real composer + harvest the in-memory `session.header`/`session.events` (the TUI `rawSessionLog` shape — no file decompression) + `scrubRequestHeaders` + `{{sessionId}}`/`{{cwd}}`/`{{rpcId}}` tokenization; a follow-up keyless refresh regenerates `ui.expected.md`. Every prompting scenario's fixture was recorded against this assembly through this flow. A drift guard ties each spec's drive prompt to the fixture's recorded `user/message`. A fixture-inventory guard holds each scenario directory closed (exact file set, every JSONL a scrub fixed-point with no run-local `rpcId`). Web fixtures scrub headers everywhere and pin no header class, following the TUI precedent over the strict [pinned-header](2026-07-06-pin-request-header-content-in-one-scenario.md) reading — see Deferred. +`DSH_SNAPSHOT` selects replay (default, keyless), record (with key), or refresh (keyless) as inline spec branches — the TUI shape, not a suite factory: at two scenarios the acp-snapshot factory machinery has no owner, and the genuinely shared parts are already exported (`scrubRequestHeaders`, `parseSessionLog`, `installLlmReplay`). Each spec splits into drive steps (type, send, `whenTurnSettled` — run in all modes, never waiting on model-content selectors, so record cannot hang on a live model answering differently) and assertion steps (replay/refresh only). Record = drive live through the real composer + harvest the in-memory `session.header`/`session.events` (the TUI `rawSessionLog` shape — no file decompression) + `scrubRequestHeaders` + `{{sessionId}}`/`{{cwd}}`/`{{rpcId}}` tokenization; a follow-up keyless refresh regenerates the aria goldens. Every prompting scenario's fixture was recorded against this assembly through this flow. A drift guard ties each spec's drive prompt to the fixture's recorded `user/message`. A fixture-inventory guard holds each scenario directory closed (exact file set, every JSONL a scrub fixed-point with no run-local `rpcId`). Web fixtures scrub headers everywhere and pin no header class, following the TUI precedent over the strict [pinned-header](2026-07-06-pin-request-header-content-in-one-scenario.md) reading — see Deferred. ### Scenarios 1. **`replay-round-trip`** — new session, prompt through the real composer, replay streams reasoning + a `bash` tool call that really executes in the temp workspace + final text (paced 15ms). Asserts settled markdown, the aria golden, and inline world state (the bash call's durable result is exactly `WEB_E2E_OK\n`, completed `turn/end`, >10 chunk events). 2. **`seeded-history`** — a recorded session seeded cold; the sidebar lists it (group row → session row, collapsed by default), opening renders tool cards and text purely from the log through the implicit cold-resume attach inside `session.history` — zero model calls in replay, so no binding constraints; record mode drives the same turn live (real `read` tool against seeded workspace files) to produce the seed. -3. **`live-interactions`** — one tool-free recorded turn serves three replay-only scenarios through override sidecars whose CONTENT is authored in the spec and minted as a per-run file in a spec-owned temp dir (the derived success entry for the retry append is re-derived from the fixture via `deriveReplayScript`, never copied into a committed sidecar). Cancel: a `{ patches }` `hang` with a `readyFile` marker — the marker's existence proves the stream is parked mid-turn before the test clicks Stop, making mid-stream cancellation deterministic by construction (`turn/end` reason `aborted`, composer re-enabled). AUTH error: a pre-chunk `throw` outside llm-retry's retryable set (`turn/end` reason `error`, zero `llm/retry` events, composer recovers). SERVER retry: `throw` at call 0 + the fixture's own success appended at 1, proving llm-retry end-to-end in the browser via the durable `llm/retry` record (`request/header` logs only on change, so attempt count is invisible there). -4. **`question-composer`** — the shipped composition's resident `ask_user_question` takeover: a recorded turn blocks mid-step on the real userInteraction seam, the composer (`[data-question-key]`) renders in the browser, the test answers through it (the ONE sanctioned place a drive step reacts to model content: the turn cannot complete without the answer, in record and replay alike), and the tool result carries the chosen label. Golden: the composer's stable waiting state. -5. **`steering`** — mid-turn steer while the question composer blocks the step (the deterministic mid-turn window; no timing dependence). The composer locks while running, so the steer POSTs `session.prompt` `mode:'steer'` from the page over the same same-origin `/api` wire the client uses (`TODO(web-steer-composer)`: drive a composer gesture once one exists); everything downstream is product — gateway → `Agent.steer` → step-boundary drain → durable `steering/message` → SSE → badged interjection bubble. Record-mode fixture honesty: the recording is rejected unless the live model's final reply obeys an instruction only the steering message carries. -6. **`navigation-panes`** — one rich two-turn seed (turn 1: bash + two parallel reads in one assistant message; turn 2: a markdown-heavy reply) rendered cold through the seeded-history pattern (zero model calls), serving four surfaces: sidebar search (client-side title filter — asserted only after the durable title lands with the attach baseline, because a cold `SessionSummary` carries no title and search matches the `displayTitle` the user sees; negative query empties the tree, positive narrows, clear restores), the Trajectory tab (turn sections + the step group's tool mix plus the view-area aria golden), the Waterfall tab (span stats + one lane per span — the P-I fold counts a turn-0 prologue span because only assistant/steering nodes carry a turn number, pinned as-is), and the details column (the bash toolview row routes click to openDetails; open/closed is asserted on the frame's `data-details-collapsed` attribute because close collapses the grid column to width 0 without unmounting the subtree). -7. **`lifecycle-chrome`** — one tiny recorded text turn drives three whole-page concerns. Workspace flow over the real wire: the empty-state hero's first send materializes a real Workspace + Session (the jsdom `workspace-flow.snapshot.ts` suite pins this state machine over the fixture client; this scenario pins it through HTTP RPC + SSE + the gateway), proven durably by the session header's cwd being the create-by-name target `/workspace`, plus the hero waiting-state aria golden. Reload recovery: collapse the sidebar (persisted `dsh.layout.panels`), `page.reload`, and the surface comes back whole from persistence alone — layout collapsed, selection restored (`dsh.sessions.current`), the recorded turn re-rendered from `session.history` with zero model calls (the drained replay cursor makes any stray request fail loud at close). Dark mode: no product control flips the theme yet, so the scenario drives the ThemeService's entire DOM contract — the `body[data-ds-dark-theme]` attribute — and pins the shipped cascade: the alias token flips, a painted surface repaints, and removing the attribute restores the light sample exactly (`TODO(web-theme-gesture)` upgrades to a real settings control); per the scope ruling there is no theme/layout golden (aria is color-blind). +3. **`live-interactions`** — one tool-free recorded turn serves three replay-only scenarios through override sidecars whose CONTENT is authored in the spec and minted as a per-run file in a spec-owned temp dir (the derived success entry for the retry append is re-derived from the fixture via `deriveReplayScript`, never copied into a committed sidecar). Cancel: a `{ patches }` `hang` with a `readyFile` marker — the marker's existence proves the stream is parked mid-turn before the test clicks Stop, making mid-stream cancellation deterministic by construction (`turn/end` reason `aborted`, composer re-enabled). AUTH error: a pre-chunk `throw` outside llm-retry's retryable set (`turn/end` reason `error`, zero `llm/retry` events, composer recovers). SERVER retry: `throw` at call 0 + the fixture's own success appended at 1, proving llm-retry end-to-end in the browser via the durable `llm/retry` record (`request/header` logs only on change, so attempt count is invisible there). Each scenario pins its terminal surface as a golden: `cancel.expected.md` (frozen `partial`, 已停止 marker), `error-auth.expected.md` (the prompt bubble alone — the committed artifact of the web-error-surface gap, the diff that flips when error rendering lands), `retry.expected.md` (indistinguishable from a clean completion — retries are deliberately invisible in the transcript). +4. **`question-composer`** — the shipped composition's resident `ask_user_question` takeover: a recorded turn blocks mid-step on the real userInteraction seam, the composer (`[data-question-key]`) renders in the browser, the test answers through it (the ONE sanctioned place a drive step reacts to model content: the turn cannot complete without the answer, in record and replay alike), and the tool result carries the chosen label. Goldens: the composer's stable waiting state (`ui.expected.md`) and the answered transcript (`answered.expected.md` — the question resolved into its tool round trip plus the final reply, takeover gone). +5. **`steering`** — mid-turn steer while the question composer blocks the step (the deterministic mid-turn window; no timing dependence). The composer locks while running, so the steer POSTs `session.prompt` `mode:'steer'` from the page over the same same-origin `/api` wire the client uses (`TODO(web-steer-composer)`: drive a composer gesture once one exists); everything downstream is product — gateway → `Agent.steer` → step-boundary drain → durable `steering/message` → SSE → badged interjection bubble. Record-mode fixture honesty: the recording is rejected unless the live model's final reply obeys an instruction only the steering message carries. Goldens pin the timing semantics visually: `mid-steer.expected.md` captures the accepted-but-invisible state (the loop drains steering only at the step boundary, so no interjection bubble exists while the question still blocks — if the client ever renders pending steers eagerly, this golden flips first) and `settled.expected.md` the badged bubble plus obeying reply. +6. **`navigation-panes`** — one rich two-turn seed (turn 1: bash + two parallel reads in one assistant message; turn 2: a markdown-heavy reply) rendered cold through the seeded-history pattern (zero model calls), serving four surfaces: sidebar search (client-side title filter — asserted only after the durable title lands with the attach baseline, because a cold `SessionSummary` carries no title and search matches the `displayTitle` the user sees; negative query empties the tree, positive narrows, clear restores), the Trajectory tab (turn sections + the step group's tool mix plus the view-area aria golden), the Waterfall tab (span stats + one lane per span — the P-I fold counts a turn-0 prologue span because only assistant/steering nodes carry a turn number, pinned as-is), and the details column (the bash toolview row routes click to openDetails; open/closed is asserted on the frame's `data-details-collapsed` attribute because close collapses the grid column to width 0 without unmounting the subtree). Goldens: `trajectory.expected.md` and `waterfall.expected.md` (each tab's view area) plus `details-open.expected.md` (the open panel: tool-name header, Input args, Output result). +7. **`lifecycle-chrome`** — one tiny recorded text turn drives three whole-page concerns. Workspace flow over the real wire: the empty-state hero's first send materializes a real Workspace + Session (the jsdom `workspace-flow.snapshot.ts` suite pins this state machine over the fixture client; this scenario pins it through HTTP RPC + SSE + the gateway), proven durably by the session header's cwd being the create-by-name target `/workspace`, plus the hero waiting-state aria golden. Reload recovery: collapse the sidebar (persisted `dsh.layout.panels`), `page.reload`, and the surface comes back whole from persistence alone — layout collapsed, selection restored (`dsh.sessions.current`), the recorded turn re-rendered from `session.history` with zero model calls (the drained replay cursor makes any stray request fail loud at close), and `reloaded.expected.md` pins the rebuilt conversation region — rendering the same settled transcript from persistence alone IS the recovery claim. Dark mode: no product control flips the theme yet, so the scenario drives the ThemeService's entire DOM contract — the `body[data-ds-dark-theme]` attribute — and pins the shipped cascade: the alias token flips, a painted surface repaints, and removing the attribute restores the light sample exactly (`TODO(web-theme-gesture)` upgrades to a real settings control); per the scope ruling there is no theme/layout golden (aria is color-blind). ### CI stance diff --git a/.agents/notes/implemented/testing/2026-07-24-web-gui-browser-e2e-lane.zh.md b/.agents/notes/implemented/testing/2026-07-24-web-gui-browser-e2e-lane.zh.md index 9850023a49..3ab0f3716a 100644 --- a/.agents/notes/implemented/testing/2026-07-24-web-gui-browser-e2e-lane.zh.md +++ b/.agents/notes/implemented/testing/2026-07-24-web-gui-browser-e2e-lane.zh.md @@ -32,23 +32,23 @@ Web GUI 以一条真实组装链交付——chromium 页面 → client 插件 bu ### 预期输出 -每场景一份提交的预期输出:会话区规范化 `ariaSnapshot()`(`ui.expected.md`)——uuid/cwd/工作区目录名/时长归一为稳定 token,在安定里程碑处轮询至两次相等再采集——外加几条 role/文本锚断言,让保语义的组件重写在预期输出可评审地变动时仍保持绿色锚点。aria 树是 client 规则「断言用户所见,绝不断言类名」的机械化。世界状态断言内联在根上下文的会话事件上(哪次工具调用产生了哪项已持久化的工具结果、`turn/end` 是否完成)而不是第二份提交的日志预期输出:持久化日志表面已由 ACP/headless/TUI 套件经同一循环和持久化钉住,在此重复钉住会违背分层纪律、翻倍刷新成本。`refresh` 是预期输出的唯一写入者——回放模式下预期输出缺失会连同修复命令一起报错,而不是静默自举。 +每场景至少一份提交的预期输出,交互类场景则每个不同终态各一份(取消/错误/重试、等待/已作答、steer 中途/安定、面板打开、重新加载后):该场景所属区域的规范化 `ariaSnapshot()`——uuid/cwd/工作区目录名/时长归一为稳定 token,在安定里程碑处轮询至两次相等再采集——外加几条 role/文本锚断言,让保语义的组件重写在预期输出可评审地变动时仍保持绿色锚点。aria 树是 client 规则「断言用户所见,绝不断言类名」的机械化。世界状态断言内联在根上下文的会话事件上(哪次工具调用产生了哪项已持久化的工具结果、`turn/end` 是否完成)而不是第二份提交的日志预期输出:持久化日志表面已由 ACP/headless/TUI 套件经同一循环和持久化钉住,在此重复钉住会违背分层纪律、翻倍刷新成本。`refresh` 是预期输出的唯一写入者——回放模式下预期输出缺失会连同修复命令一起报错,而不是静默自举。 类型检查平面切分是结构性的:启动 host 主干的三个文件(`scaffold`、`replay-round-trip.e2e` 和 `seeded-history.e2e`)被排除出注册在 client 侧的 `apps/web` 工程。这三个文件及其共享的 `support.ts` 逐文件纳入 `tsconfig.host.json`——一个程序不能同时持有 cordis `Context` 合并的两侧。 ### 模式与 fixture -`DSH_SNAPSHOT` 以内联 spec 分支选择 replay(默认,无密钥)、record(带密钥)或 refresh(无密钥)——TUI 的形态,不是套件工厂:两个场景撑不起 acp-snapshot 工厂机制,且真正共享的部分已被导出(`scrubRequestHeaders`、`parseSessionLog`、`installLlmReplay`)。每个 spec 切分为驱动步骤(输入、发送、`whenTurnSettled`——所有模式都执行,绝不等待模型内容选择器,因此 record 不会因真实模型答法不同而挂起)与断言步骤(仅 replay/refresh)。Record = 经真实输入框实时驱动 + 采收内存中的 `session.header`/`session.events`(TUI 的 `rawSessionLog` 形态——无需文件解压)+ `scrubRequestHeaders` + `{{sessionId}}`/`{{cwd}}`/`{{rpcId}}` token 化;随后一次无密钥 refresh 重新生成 `ui.expected.md`。每个发起提示的场景,其 fixture 都经此流程对本组装录制。一条漂移防线把每个 spec 的驱动提示词与 fixture 录制的 `user/message` 绑定。fixture 清单防线保持每个场景目录封闭(精确文件集合,每个 JSONL 都是脱敏不动点,不含当次运行的 `rpcId`)。Web fixture 全部脱敏请求头且不钉任何头类别,沿用 TUI 先例而非[钉住请求头](2026-07-06-pin-request-header-content-in-one-scenario.md)的严格读法——见「暂缓」。 +`DSH_SNAPSHOT` 以内联 spec 分支选择 replay(默认,无密钥)、record(带密钥)或 refresh(无密钥)——TUI 的形态,不是套件工厂:两个场景撑不起 acp-snapshot 工厂机制,且真正共享的部分已被导出(`scrubRequestHeaders`、`parseSessionLog`、`installLlmReplay`)。每个 spec 切分为驱动步骤(输入、发送、`whenTurnSettled`——所有模式都执行,绝不等待模型内容选择器,因此 record 不会因真实模型答法不同而挂起)与断言步骤(仅 replay/refresh)。Record = 经真实输入框实时驱动 + 采收内存中的 `session.header`/`session.events`(TUI 的 `rawSessionLog` 形态——无需文件解压)+ `scrubRequestHeaders` + `{{sessionId}}`/`{{cwd}}`/`{{rpcId}}` token 化;随后一次无密钥 refresh 重新生成各份 aria 预期输出。每个发起提示的场景,其 fixture 都经此流程对本组装录制。一条漂移防线把每个 spec 的驱动提示词与 fixture 录制的 `user/message` 绑定。fixture 清单防线保持每个场景目录封闭(精确文件集合,每个 JSONL 都是脱敏不动点,不含当次运行的 `rpcId`)。Web fixture 全部脱敏请求头且不钉任何头类别,沿用 TUI 先例而非[钉住请求头](2026-07-06-pin-request-header-content-in-one-scenario.md)的严格读法——见「暂缓」。 ### 场景 1. **`replay-round-trip`**——新会话,经真实输入框发送提示词,回放流式输出推理(reasoning)+ 一次在临时工作区真实执行的 `bash` 工具调用 + 最终文本(15ms 节奏)。断言安定后的 markdown、aria 预期输出与内联世界状态(这次 bash 调用的已持久化工具结果严格等于 `WEB_E2E_OK\n`、完成的 `turn/end`、>10 个分片事件)。 2. **`seeded-history`**——冷播种一份已录会话;侧栏列出它(分组行 → 会话行,默认折叠),打开后纯凭日志经 `session.history` 内的隐式冷恢复挂载渲染工具卡片与文本——replay 下零模型调用,因此没有任何绑定约束;record 模式实时驱动同一轮(真实 `read` 工具读取播种的工作区文件)来产出种子。 -3. **`live-interactions`**——一段不含工具调用的已录轮次经覆写 sidecar 承载三个仅回放的场景:sidecar 的内容本身写在 spec 里,每次运行时在 spec 自有的临时目录中生成文件(重试追加所用的派生成功条目经 `deriveReplayScript` 从 fixture 重新派生,绝不复制进已提交的 sidecar)。取消:一个带 `readyFile` 标记的 `{ patches }` `hang`——标记文件的存在证明流在测试点击 Stop 之前已停驻在轮次中途,使流中取消按构造即确定(`turn/end` 原因为 `aborted`,输入框重新启用)。AUTH 错误:一次落在 llm-retry 可重试集合之外的分片前 `throw`(`turn/end` 原因为 `error`,零条 `llm/retry` 事件,输入框恢复可用)。SERVER 重试:第 0 次调用 `throw` + 在第 1 次调用处追加 fixture 自身的成功条目,凭持久的 `llm/retry` 记录在浏览器中端到端证明 llm-retry(`request/header` 仅在变化时记录,因此尝试次数在那里不可见)。 -4. **`question-composer`**——已交付组合中常驻的 `ask_user_question` 接管:一段已录轮次在真实的 userInteraction seam 上阻塞于步骤中途,提问输入框(`[data-question-key]`)在浏览器中渲染,测试经它作答(这是驱动步骤对模型内容作出反应的唯一获准之处:没有这个回答,轮次无法完成,record 与 replay 皆然),工具结果携带所选的 label。预期输出:提问输入框稳定的等待态。 -5. **`steering`**——在提问输入框阻塞该步骤时做轮次中途 steering(中途引导),此即确定性的轮次中途窗口,不依赖任何时序。输入框在运行期间锁定,因此这一 steer 由页面经客户端所用的同一条同源 `/api` wire POST `session.prompt` `mode:'steer'`(`TODO(web-steer-composer)`:待有输入框手势后改为驱动它);下游的一切都是产品路径——gateway → `Agent.steer` → 步骤边界排空 → 持久的 `steering/message` → SSE → 带徽标的插话气泡。record 模式的 fixture 诚实性:除非真实模型的最终回复遵循了一条只有 steering 消息才携带的指令,否则该次录制被拒绝。 -6. **`navigation-panes`**——一份内容丰富的双轮次种子(轮次 1:同一条 assistant 消息内的 bash + 两次并行 read;轮次 2:一段 markdown 密集的回复)经 seeded-history 模式冷渲染(零模型调用),承载四个表面:侧栏搜索(客户端标题过滤;仅在持久的标题随 attach 基线一同到达后才断言,因为冷的 `SessionSummary` 不携带标题,而搜索匹配的是用户所见的 `displayTitle`;反例查询清空整棵树,正例查询收窄,清除后复原)、Trajectory 标签页(轮次分节 + 步骤组的工具构成,外加视图区 aria 预期输出)、Waterfall 标签页(span 统计 + 每个 span 一条泳道;只有 assistant/steering 节点携带轮次编号,因此 P-I 折叠会将一个轮次 0 的序幕 span 计入,按原样钉住)与详情列(bash 工具视图行把点击路由到 openDetails;打开/关闭状态断言在 frame 的 `data-details-collapsed` 属性上,因为关闭把网格列收缩到宽度 0 而不卸载子树)。 -7. **`lifecycle-chrome`**——一段极小的已录纯文本轮次驱动三个整页关注点。真实 wire 上的 Workspace 动线:空态 hero 的首次发送物化出真实的 Workspace + Session(jsdom 的 `workspace-flow.snapshot.ts` 套件基于 fixture 客户端钉住这一状态机;本场景则经 HTTP RPC + SSE + gateway 钉住它),其持久证据是会话头部的 cwd 恰为按名创建的目标 `/workspace`,外加 hero 等待态的 aria 预期输出。重新加载恢复:折叠侧栏(持久化于 `dsh.layout.panels`),`page.reload`,整个表面纯凭持久化完整归来——布局保持折叠,选中项恢复(`dsh.sessions.current`),已录轮次从 `session.history` 重新渲染且零模型调用(已耗尽的回放游标使任何离群请求都在 close 时大声失败)。暗色模式:产品尚无任何控件能切换主题,因此本场景驱动 ThemeService 的整个 DOM 契约(即 `body[data-ds-dark-theme]` 属性),并钉住已交付的级联:alias token 翻转,某个实际绘制的表面重绘,移除该属性则精确还原亮色采样值(`TODO(web-theme-gesture)`:待有真实设置控件后升级为驱动它);按范围裁定,主题/布局不设预期输出(aria 感知不到颜色)。 +3. **`live-interactions`**——一段不含工具调用的已录轮次经覆写 sidecar 承载三个仅回放的场景:sidecar 的内容本身写在 spec 里,每次运行时在 spec 自有的临时目录中生成文件(重试追加所用的派生成功条目经 `deriveReplayScript` 从 fixture 重新派生,绝不复制进已提交的 sidecar)。取消:一个带 `readyFile` 标记的 `{ patches }` `hang`——标记文件的存在证明流在测试点击 Stop 之前已停驻在轮次中途,使流中取消按构造即确定(`turn/end` 原因为 `aborted`,输入框重新启用)。AUTH 错误:一次落在 llm-retry 可重试集合之外的分片前 `throw`(`turn/end` 原因为 `error`,零条 `llm/retry` 事件,输入框恢复可用)。SERVER 重试:第 0 次调用 `throw` + 在第 1 次调用处追加 fixture 自身的成功条目,凭持久的 `llm/retry` 记录在浏览器中端到端证明 llm-retry(`request/header` 仅在变化时记录,因此尝试次数在那里不可见)。每个场景都把各自的终态表面钉为一份预期输出:`cancel.expected.md`(冻结的 `partial`、「已停止」标记)、`error-auth.expected.md`(仅有提示词气泡——web-error-surface 缺口的已提交产物,错误渲染落地时翻转的那份 diff)、`retry.expected.md`(与一次干净完成无从区分——重试在文本记录中刻意不可见)。 +4. **`question-composer`**——已交付组合中常驻的 `ask_user_question` 接管:一段已录轮次在真实的 userInteraction seam 上阻塞于步骤中途,提问输入框(`[data-question-key]`)在浏览器中渲染,测试经它作答(这是驱动步骤对模型内容作出反应的唯一获准之处:没有这个回答,轮次无法完成,record 与 replay 皆然),工具结果携带所选的 label。预期输出:提问输入框稳定的等待态(`ui.expected.md`)与已作答的文本记录(`answered.expected.md`——提问已落定为其工具往返加最终回复,接管消失)。 +5. **`steering`**——在提问输入框阻塞该步骤时做轮次中途 steering(中途引导),此即确定性的轮次中途窗口,不依赖任何时序。输入框在运行期间锁定,因此这一 steer 由页面经客户端所用的同一条同源 `/api` wire POST `session.prompt` `mode:'steer'`(`TODO(web-steer-composer)`:待有输入框手势后改为驱动它);下游的一切都是产品路径——gateway → `Agent.steer` → 步骤边界排空 → 持久的 `steering/message` → SSE → 带徽标的插话气泡。record 模式的 fixture 诚实性:除非真实模型的最终回复遵循了一条只有 steering 消息才携带的指令,否则该次录制被拒绝。预期输出以可视方式钉住这一时序语义:`mid-steer.expected.md` 捕捉「已接受但不可见」的状态(循环仅在步骤边界才排空 steering,因此提问仍在阻塞时不存在插话气泡——若 client 日后提前渲染待处理的 steer,这份预期输出会最先翻转),`settled.expected.md` 则捕捉带徽标的气泡加遵循指令的回复。 +6. **`navigation-panes`**——一份内容丰富的双轮次种子(轮次 1:同一条 assistant 消息内的 bash + 两次并行 read;轮次 2:一段 markdown 密集的回复)经 seeded-history 模式冷渲染(零模型调用),承载四个表面:侧栏搜索(客户端标题过滤;仅在持久的标题随 attach 基线一同到达后才断言,因为冷的 `SessionSummary` 不携带标题,而搜索匹配的是用户所见的 `displayTitle`;反例查询清空整棵树,正例查询收窄,清除后复原)、Trajectory 标签页(轮次分节 + 步骤组的工具构成,外加视图区 aria 预期输出)、Waterfall 标签页(span 统计 + 每个 span 一条泳道;只有 assistant/steering 节点携带轮次编号,因此 P-I 折叠会将一个轮次 0 的序幕 span 计入,按原样钉住)与详情列(bash 工具视图行把点击路由到 openDetails;打开/关闭状态断言在 frame 的 `data-details-collapsed` 属性上,因为关闭把网格列收缩到宽度 0 而不卸载子树)。预期输出:`trajectory.expected.md` 与 `waterfall.expected.md`(各自标签页的视图区),外加 `details-open.expected.md`(打开的面板:工具名标题、Input 参数、Output 结果)。 +7. **`lifecycle-chrome`**——一段极小的已录纯文本轮次驱动三个整页关注点。真实 wire 上的 Workspace 动线:空态 hero 的首次发送物化出真实的 Workspace + Session(jsdom 的 `workspace-flow.snapshot.ts` 套件基于 fixture 客户端钉住这一状态机;本场景则经 HTTP RPC + SSE + gateway 钉住它),其持久证据是会话头部的 cwd 恰为按名创建的目标 `/workspace`,外加 hero 等待态的 aria 预期输出。重新加载恢复:折叠侧栏(持久化于 `dsh.layout.panels`),`page.reload`,整个表面纯凭持久化完整归来——布局保持折叠,选中项恢复(`dsh.sessions.current`),已录轮次从 `session.history` 重新渲染且零模型调用(已耗尽的回放游标使任何离群请求都在 close 时大声失败),且 `reloaded.expected.md` 钉住重建后的会话区——纯凭持久化渲染出同一份安定的文本记录,这本身就是恢复主张。暗色模式:产品尚无任何控件能切换主题,因此本场景驱动 ThemeService 的整个 DOM 契约(即 `body[data-ds-dark-theme]` 属性),并钉住已交付的级联:alias token 翻转,某个实际绘制的表面重绘,移除该属性则精确还原亮色采样值(`TODO(web-theme-gesture)`:待有真实设置控件后升级为驱动它);按范围裁定,主题/布局不设预期输出(aria 感知不到颜色)。 ### CI 立场 diff --git a/apps/web/tests/lifecycle-chrome.e2e.ts b/apps/web/tests/lifecycle-chrome.e2e.ts index 2ab4f5aeea..d91704f585 100644 --- a/apps/web/tests/lifecycle-chrome.e2e.ts +++ b/apps/web/tests/lifecycle-chrome.e2e.ts @@ -25,6 +25,9 @@ import { saveFailureShot } from './support.ts' const SNAPSHOT_DIR = fileURLToPath(new URL('./snapshots/lifecycle-chrome', import.meta.url)) const FIXTURE = join(SNAPSHOT_DIR, 'session.jsonl') const HERO_EXPECTED = join(SNAPSHOT_DIR, 'hero.expected.md') +// Post-reload golden: the same settled conversation rebuilt purely from +// persistence + history — byte-equal rendering is exactly the recovery claim. +const RELOADED_EXPECTED = join(SNAPSHOT_DIR, 'reloaded.expected.md') const MODE = webSnapshotMode() const PROMPT = 'Reply with the single word LIGHTHOUSE and stop.' @@ -112,6 +115,10 @@ describe('web e2e: lifecycle & chrome (workspace flow / reload / dark mode)', () // Expand back and confirm the tree still lists the materialized session. await page.getByRole('button', { name: 'Open sidebar' }).click() await expect.poll(() => page.locator('[role="treeitem"][aria-selected="true"]').count(), { timeout: 10_000 }).toBe(1) + // Golden of the recovered conversation region: rebuilt from the log, it + // must render the same settled transcript the live turn produced. + const snapshot = await captureStableAria(page, '[class*="centerCol"]', scaffold.workspaceCwd) + await compareOrRefreshGolden(RELOADED_EXPECTED, snapshot, MODE) expect(tripwire.pageErrors).toEqual([]) }, 90_000) @@ -147,6 +154,6 @@ describe('web e2e: lifecycle & chrome (workspace flow / reload / dark mode)', () }, 60_000) it.skipIf(MODE === 'record')('keeps the fixture inventory closed', async () => { - await assertFixtureInventory(SNAPSHOT_DIR, ['session.jsonl', 'hero.expected.md']) + await assertFixtureInventory(SNAPSHOT_DIR, ['session.jsonl', 'hero.expected.md', 'reloaded.expected.md']) }) }) diff --git a/apps/web/tests/live-interactions.e2e.ts b/apps/web/tests/live-interactions.e2e.ts index 632dc79085..60dec690c7 100644 --- a/apps/web/tests/live-interactions.e2e.ts +++ b/apps/web/tests/live-interactions.e2e.ts @@ -20,13 +20,20 @@ import { deriveReplayScript, parseSessionLog } from '@deepseek-ai/dsh-llm-replay import type { ReplayOverrideDoc } from '@deepseek-ai/dsh-llm-replay' import type { SessionEvent } from '@deepseek-ai/dsh-session' import { - assertFixtureInventory, fixtureUserPrompts, launchWebScaffold, recordFixture, - watchConsole, webSnapshotMode, type WebScaffold, + assertFixtureInventory, captureStableAria, compareOrRefreshGolden, fixtureUserPrompts, + launchWebScaffold, recordFixture, watchConsole, webSnapshotMode, type WebScaffold, } from './scaffold.ts' import { saveFailureShot } from './support.ts' const SNAPSHOT_DIR = fileURLToPath(new URL('./snapshots/live-interactions', import.meta.url)) const FIXTURE = join(SNAPSHOT_DIR, 'session.jsonl') +// One golden per interactive end-state: what the user is left looking at +// after cancel, after a non-retryable failure (pins the FIXME(web-error-surface) +// gap as a reviewable artifact: NO error copy in the tree), and after retry +// recovery — three genuinely different terminal surfaces of one fixture. +const CANCEL_EXPECTED = join(SNAPSHOT_DIR, 'cancel.expected.md') +const ERROR_EXPECTED = join(SNAPSHOT_DIR, 'error-auth.expected.md') +const RETRY_EXPECTED = join(SNAPSHOT_DIR, 'retry.expected.md') const MODE = webSnapshotMode() // The recorded base: one text-only turn whose derived script the sidecars @@ -123,6 +130,10 @@ describe('web e2e: live-turn interactions (cancel / error / retry)', () => { // Composer recovered; no streaming node lingers. await expect.poll(() => page.locator('textarea').first().isEnabled(), { timeout: 10_000 }).toBe(true) expect(await page.locator('[data-streaming="true"]').count()).toBe(0) + // Golden of the aborted end-state: the prompt bubble plus the frozen + // partial ('partial' is the hang entry's replayed prefix) and no more. + const snapshot = await captureStableAria(page, '[class*="centerCol"]', scaffold!.workspaceCwd) + await compareOrRefreshGolden(CANCEL_EXPECTED, snapshot, MODE) expect(tripwire.pageErrors).toEqual([]) }, 120_000) @@ -144,6 +155,10 @@ describe('web e2e: live-turn interactions (cancel / error / retry)', () => { // "no crash, composer recovers, turn logged as error". await expect.poll(() => page.locator('textarea').first().isEnabled(), { timeout: 10_000 }).toBe(true) expect(await page.locator('[data-streaming="true"]').count()).toBe(0) + // Golden of the same gap: the prompt bubble alone, no error copy in the + // tree — the diff that changes when web-error-surface lands. + const snapshot = await captureStableAria(page, '[class*="centerCol"]', scaffold!.workspaceCwd) + await compareOrRefreshGolden(ERROR_EXPECTED, snapshot, MODE) expect(tripwire.pageErrors).toEqual([]) }, 120_000) @@ -167,10 +182,16 @@ describe('web e2e: live-turn interactions (cancel / error / retry)', () => { // only on change, so attempt count is invisible there). expect(sessionEvents.filter(e => e.type === 'llm/retry').length).toBeGreaterThanOrEqual(1) await expect.poll(() => page.getByText('event sourcing', { exact: false }).count(), { timeout: 10_000 }).toBeGreaterThan(0) + // Golden of the recovered end-state: indistinguishable from a clean + // completion — retries are deliberately invisible in the transcript. + const snapshot = await captureStableAria(page, '[class*="centerCol"]', scaffold!.workspaceCwd) + await compareOrRefreshGolden(RETRY_EXPECTED, snapshot, MODE) expect(tripwire.pageErrors).toEqual([]) }, 120_000) it.skipIf(MODE === 'record')('keeps the fixture inventory closed', async () => { - await assertFixtureInventory(SNAPSHOT_DIR, ['session.jsonl']) + await assertFixtureInventory(SNAPSHOT_DIR, [ + 'session.jsonl', 'cancel.expected.md', 'error-auth.expected.md', 'retry.expected.md', + ]) }) }) diff --git a/apps/web/tests/navigation-panes.e2e.ts b/apps/web/tests/navigation-panes.e2e.ts index 2147ef9cdd..bbae7363df 100644 --- a/apps/web/tests/navigation-panes.e2e.ts +++ b/apps/web/tests/navigation-panes.e2e.ts @@ -24,6 +24,8 @@ import { saveFailureShot } from './support.ts' const SNAPSHOT_DIR = fileURLToPath(new URL('./snapshots/navigation-panes', import.meta.url)) const SEED = join(SNAPSHOT_DIR, 'seed.jsonl') const TRAJECTORY_EXPECTED = join(SNAPSHOT_DIR, 'trajectory.expected.md') +const WATERFALL_EXPECTED = join(SNAPSHOT_DIR, 'waterfall.expected.md') +const DETAILS_EXPECTED = join(SNAPSHOT_DIR, 'details-open.expected.md') const MODE = webSnapshotMode() const SEED_ID = 'navigation-panes-web-e2e' @@ -148,6 +150,9 @@ describe('web e2e: navigation & panes over a rich seeded session', () => { for (const tag of ['turn 0', 'turn 1', 'turn 2']) { await expect.poll(() => page.getByText(tag, { exact: true }).count(), { timeout: 10_000 }).toBe(1) } + const snapshot = (await captureStableAria(page, '[class*="viewArea"]', scaffold.workspaceCwd)) + .split(SEED_ID).join('{{seededId}}') + await compareOrRefreshGolden(WATERFALL_EXPECTED, snapshot, MODE) }, 60_000) it.skipIf(MODE === 'record')('opens the details column from the bash row and closes it', async () => { @@ -167,6 +172,10 @@ describe('web e2e: navigation & panes over a rich seeded session', () => { // The open panel shows the selected call's name, arguments, and durable // result (NAVIGATION_OK appears in the chat row too, hence >= 2 total). await expect.poll(() => page.getByText('NAVIGATION_OK', { exact: false }).count(), { timeout: 10_000 }).toBeGreaterThanOrEqual(2) + // Golden of the open panel: tool name header, Input args, Output result. + const snapshot = (await captureStableAria(page, '[class*="detailsCol"]', scaffold.workspaceCwd)) + .split(SEED_ID).join('{{seededId}}') + await compareOrRefreshGolden(DETAILS_EXPECTED, snapshot, MODE) await page.getByRole('button', { name: '关闭详情' }).click() await expect.poll(() => frame.getAttribute('data-details-collapsed'), { timeout: 10_000 }).not.toBeNull() }, 60_000) @@ -174,6 +183,8 @@ describe('web e2e: navigation & panes over a rich seeded session', () => { it.skipIf(MODE === 'record')('issued zero model calls and stayed clean', async () => { expect(tripwire.pageErrors).toEqual([]) expect(tripwire.warnings).toEqual([]) - await assertFixtureInventory(SNAPSHOT_DIR, ['seed.jsonl', 'trajectory.expected.md']) + await assertFixtureInventory(SNAPSHOT_DIR, [ + 'seed.jsonl', 'trajectory.expected.md', 'waterfall.expected.md', 'details-open.expected.md', + ]) }) }) diff --git a/apps/web/tests/question-composer.e2e.ts b/apps/web/tests/question-composer.e2e.ts index 9678a7a648..361cd72be6 100644 --- a/apps/web/tests/question-composer.e2e.ts +++ b/apps/web/tests/question-composer.e2e.ts @@ -23,6 +23,9 @@ import { saveFailureShot } from './support.ts' const SNAPSHOT_DIR = fileURLToPath(new URL('./snapshots/question-composer', import.meta.url)) const FIXTURE = join(SNAPSHOT_DIR, 'session.jsonl') const UI_EXPECTED = join(SNAPSHOT_DIR, 'ui.expected.md') +// Second golden: the answered transcript — the question resolved into its +// tool round trip and the final reply, the state the waiting golden cannot see. +const ANSWERED_EXPECTED = join(SNAPSHOT_DIR, 'answered.expected.md') const MODE = webSnapshotMode() const PROMPT = 'Use the ask_user_question tool to ask me exactly one question with id "color", question "Which color do you prefer?", header "Pick one", and options labeled "Blue" and "Green". After I answer, reply with the single word DONE and stop.' @@ -90,10 +93,14 @@ describe('web e2e: resident question composer round trip', () => { // Composer gone; regular input restored. expect(await page.locator('[data-question-key]').count()).toBe(0) await expect.poll(() => page.locator('textarea').first().isEnabled(), { timeout: 10_000 }).toBe(true) + // Golden of the answered transcript: the ask_user_question round trip + // rendered as history (question tool row + DONE), composer takeover gone. + const snapshot = await captureStableAria(page, '[class*="centerCol"]', scaffold.workspaceCwd) + await compareOrRefreshGolden(ANSWERED_EXPECTED, snapshot, MODE) expect(tripwire.pageErrors).toEqual([]) }, 200_000) it.skipIf(MODE === 'record')('keeps the fixture inventory closed', async () => { - await assertFixtureInventory(SNAPSHOT_DIR, ['session.jsonl', 'ui.expected.md']) + await assertFixtureInventory(SNAPSHOT_DIR, ['session.jsonl', 'ui.expected.md', 'answered.expected.md']) }) }) diff --git a/apps/web/tests/snapshots/lifecycle-chrome/reloaded.expected.md b/apps/web/tests/snapshots/lifecycle-chrome/reloaded.expected.md new file mode 100644 index 0000000000..6c0b20cc22 --- /dev/null +++ b/apps/web/tests/snapshots/lifecycle-chrome/reloaded.expected.md @@ -0,0 +1,27 @@ +- banner: + - navigation "Session hierarchy": + - button "Reply with the single word" [disabled] + - text: · 1 turns + - tablist: + - tab "Chat" [selected] + - tab "Trajectory" + - tab "Waterfall" +- text: Reply with the single word LIGHTHOUSE and stop. +- button "Think The user wants me to reply with a single word. Let me comply.": + - img + - text: Think The user wants me to reply with a single word. Let me comply. +- paragraph: LIGHTHOUSE +- text: cache hit 99% · 7,810 tokens · 1 turns · 1 steps +- textbox "Message the agent" +- button "Add attachment": + - img +- combobox "Plan mode": + - option "Plan" [selected] + - option "Agent" +- combobox "Access mode": + - option "Read-only" [selected] + - option "Read-write" +- combobox "Model": + - option "DeepSeek-V4-Pro High" [selected] + - option "DeepSeek-V4-Pro" +- button "Send message" [disabled] diff --git a/apps/web/tests/snapshots/live-interactions/cancel.expected.md b/apps/web/tests/snapshots/live-interactions/cancel.expected.md new file mode 100644 index 0000000000..1c0807b33b --- /dev/null +++ b/apps/web/tests/snapshots/live-interactions/cancel.expected.md @@ -0,0 +1,24 @@ +- banner: + - navigation "Session hierarchy": + - button "Reply with a one-sentence description" [disabled] + - text: · 1 turns + - tablist: + - tab "Chat" [selected] + - tab "Trajectory" + - tab "Waterfall" +- text: Reply with a one-sentence description of event sourcing, then stop. +- paragraph: partial +- text: 已停止 0 tokens · 1 turns · 1 steps +- textbox "Message the agent" +- button "Add attachment": + - img +- combobox "Plan mode": + - option "Plan" [selected] + - option "Agent" +- combobox "Access mode": + - option "Read-only" [selected] + - option "Read-write" +- combobox "Model": + - option "DeepSeek-V4-Pro High" [selected] + - option "DeepSeek-V4-Pro" +- button "Send message" [disabled] diff --git a/apps/web/tests/snapshots/live-interactions/error-auth.expected.md b/apps/web/tests/snapshots/live-interactions/error-auth.expected.md new file mode 100644 index 0000000000..5862e97ab6 --- /dev/null +++ b/apps/web/tests/snapshots/live-interactions/error-auth.expected.md @@ -0,0 +1,22 @@ +- banner: + - navigation "Session hierarchy": + - button "Reply with a one-sentence description" [disabled] + - text: · 1 turns + - tablist: + - tab "Chat" [selected] + - tab "Trajectory" + - tab "Waterfall" +- text: Reply with a one-sentence description of event sourcing, then stop. +- textbox "Message the agent" +- button "Add attachment": + - img +- combobox "Plan mode": + - option "Plan" [selected] + - option "Agent" +- combobox "Access mode": + - option "Read-only" [selected] + - option "Read-write" +- combobox "Model": + - option "DeepSeek-V4-Pro High" [selected] + - option "DeepSeek-V4-Pro" +- button "Send message" [disabled] diff --git a/apps/web/tests/snapshots/live-interactions/retry.expected.md b/apps/web/tests/snapshots/live-interactions/retry.expected.md new file mode 100644 index 0000000000..ed77fac08b --- /dev/null +++ b/apps/web/tests/snapshots/live-interactions/retry.expected.md @@ -0,0 +1,27 @@ +- banner: + - navigation "Session hierarchy": + - button "Reply with a one-sentence description" [disabled] + - text: · 1 turns + - tablist: + - tab "Chat" [selected] + - tab "Trajectory" + - tab "Waterfall" +- text: Reply with a one-sentence description of event sourcing, then stop. +- button "Think The user is asking for a one-sentence description of event sourcing. This is a straightforward knowledge question that doesn't require any skill loading or tool calls.": + - img + - text: Think The user is asking for a one-sentence description of event sourcing. This is a straightforward knowledge question that doesn't require any skill loading or tool calls. +- paragraph: Event sourcing is a pattern where all changes to an application's state are stored as an immutable, append-only sequence of events, rather than persisting only the current state, enabling full auditability, temporal queries, and event-driven architectures. +- text: cache hit 99% · 7,869 tokens · 1 turns · 1 steps +- textbox "Message the agent" +- button "Add attachment": + - img +- combobox "Plan mode": + - option "Plan" [selected] + - option "Agent" +- combobox "Access mode": + - option "Read-only" [selected] + - option "Read-write" +- combobox "Model": + - option "DeepSeek-V4-Pro High" [selected] + - option "DeepSeek-V4-Pro" +- button "Send message" [disabled] diff --git a/apps/web/tests/snapshots/navigation-panes/details-open.expected.md b/apps/web/tests/snapshots/navigation-panes/details-open.expected.md new file mode 100644 index 0000000000..39bf528542 --- /dev/null +++ b/apps/web/tests/snapshots/navigation-panes/details-open.expected.md @@ -0,0 +1,3 @@ +- text: bash +- button "关闭详情" +- text: "Input { \"command\": \"echo NAVIGATION_OK\", \"description\": \"Print NAVIGATION_OK\" } Output NAVIGATION_OK" diff --git a/apps/web/tests/snapshots/navigation-panes/waterfall.expected.md b/apps/web/tests/snapshots/navigation-panes/waterfall.expected.md new file mode 100644 index 0000000000..6c5ab1a046 --- /dev/null +++ b/apps/web/tests/snapshots/navigation-panes/waterfall.expected.md @@ -0,0 +1 @@ +- text: 3 turns · 3 steps · 3 tool calls turn 0 turn 1 turn 2 diff --git a/apps/web/tests/snapshots/question-composer/answered.expected.md b/apps/web/tests/snapshots/question-composer/answered.expected.md new file mode 100644 index 0000000000..c0e64f7bf3 --- /dev/null +++ b/apps/web/tests/snapshots/question-composer/answered.expected.md @@ -0,0 +1,33 @@ +- banner: + - navigation "Session hierarchy": + - button "Use the ask_user_question tool to" [disabled] + - text: · 1 turns + - tablist: + - tab "Chat" [selected] + - tab "Trajectory" + - tab "Waterfall" +- text: Use the ask_user_question tool to ask me exactly one question with id "color", question "Which color do you prefer?", header "Pick one", and options labeled "Blue" and "Green". After I answer, reply with the single word DONE and stop. +- button "Think The user wants me to use the ask_user_question tool to ask a specific question with id \"color\", question \"Which color do you prefer?\", header \"Pick one\", and options labeled \"Blue\" and \"Green\". Let me do exactly that.": + - img + - text: Think The user wants me to use the ask_user_question tool to ask a specific question with id "color", question "Which color do you prefer?", header "Pick one", and options labeled "Blue" and "Green". Let me do exactly that. +- button: + - img +- text: "Tool call ask_user_question · {\"questions\": [{\"id\": \"color\", \"question\": \"Which color do you prefer?\", \"header\": \"Pick one\", \"options\": [{\"label\": \"Blue\"}, {\"label\": \"Green\"}]}]}" +- button "Think The user answered \"Blue\". I need to reply with the single word DONE and stop.": + - img + - text: Think The user answered "Blue". I need to reply with the single word DONE and stop. +- paragraph: DONE +- text: cache hit 99% · 15,978 tokens · 1 turns · 2 steps +- textbox "Message the agent" +- button "Add attachment": + - img +- combobox "Plan mode": + - option "Plan" [selected] + - option "Agent" +- combobox "Access mode": + - option "Read-only" [selected] + - option "Read-write" +- combobox "Model": + - option "DeepSeek-V4-Pro High" [selected] + - option "DeepSeek-V4-Pro" +- button "Send message" [disabled] diff --git a/apps/web/tests/snapshots/steering/mid-steer.expected.md b/apps/web/tests/snapshots/steering/mid-steer.expected.md new file mode 100644 index 0000000000..a26bbb7bd8 --- /dev/null +++ b/apps/web/tests/snapshots/steering/mid-steer.expected.md @@ -0,0 +1,39 @@ +- banner: + - navigation "Session hierarchy": + - button "Use the ask_user_question tool to" [disabled] + - text: · 1 turns + - tablist: + - tab "Chat" [selected] + - tab "Trajectory" + - tab "Waterfall" +- text: Use the ask_user_question tool to ask me exactly one question with id "checkpoint", question "Ready to continue?", header "Checkpoint", and options labeled "Yes" and "No". After I answer, reply with one short sentence acknowledging my answer and stop. +- button "Think The user wants me to use the ask_user_question tool to ask them a specific question with the given parameters. Let me do exactly that.": + - img + - text: Think The user wants me to use the ask_user_question tool to ask them a specific question with the given parameters. Let me do exactly that. +- button +- text: "Tool call ask_user_question · {\"questions\": [{\"id\": \"checkpoint\", \"question\": \"Ready to continue?\", \"header\": \"Checkpoint\", \"options\": [{\"label\": \"Yes\"}, {\"label\": \"No\"}]}]} 等待回答(1 题)" +- button "▸ 问题内容" +- text: 请在原客户端处理(web 端作答后续里程碑提供) cache hit 98% · 7,946 tokens · 1 turns · 1 steps +- region "Ready to continue?": + - text: Checkpoint + - heading "Ready to continue?" [level=2] + - text: 1 / 1 + - button "上一题" [disabled]: + - img + - button "下一题" [disabled]: + - img + - button "放弃整组问题": + - img + - radiogroup: + - radio "Yes": + - text: 1 Yes + - img + - radio "No": + - text: 2 No + - img + - button "其他,请填写自定义答案": + - img + - text: 其他,请填写自定义答案 + - status + - button "跳过本题" + - button "提交" [disabled] diff --git a/apps/web/tests/snapshots/steering/settled.expected.md b/apps/web/tests/snapshots/steering/settled.expected.md new file mode 100644 index 0000000000..6faa2f01a3 --- /dev/null +++ b/apps/web/tests/snapshots/steering/settled.expected.md @@ -0,0 +1,33 @@ +- banner: + - navigation "Session hierarchy": + - button "Use the ask_user_question tool to" [disabled] + - text: · 1 turns + - tablist: + - tab "Chat" [selected] + - tab "Trajectory" + - tab "Waterfall" +- text: Use the ask_user_question tool to ask me exactly one question with id "checkpoint", question "Ready to continue?", header "Checkpoint", and options labeled "Yes" and "No". After I answer, reply with one short sentence acknowledging my answer and stop. +- button "Think The user wants me to use the ask_user_question tool to ask them a specific question with the given parameters. Let me do exactly that.": + - img + - text: Think The user wants me to use the ask_user_question tool to ask them a specific question with the given parameters. Let me do exactly that. +- button: + - img +- text: "Tool call ask_user_question · {\"questions\": [{\"id\": \"checkpoint\", \"question\": \"Ready to continue?\", \"header\": \"Checkpoint\", \"options\": [{\"label\": \"Yes\"}, {\"label\": \"No\"}]}]} 插话 Interjection: include the word BANANA in your final reply." +- button "Think The user selected \"Yes\" and wants me to include the word \"BANANA\" in my final reply. Let me acknowledge their answer.": + - img + - text: Think The user selected "Yes" and wants me to include the word "BANANA" in my final reply. Let me acknowledge their answer. +- paragraph: Great, let's move forward. BANANA! +- text: cache hit 98% · 15,967 tokens · 1 turns · 2 steps +- textbox "Message the agent" +- button "Add attachment": + - img +- combobox "Plan mode": + - option "Plan" [selected] + - option "Agent" +- combobox "Access mode": + - option "Read-only" [selected] + - option "Read-write" +- combobox "Model": + - option "DeepSeek-V4-Pro High" [selected] + - option "DeepSeek-V4-Pro" +- button "Send message" [disabled] diff --git a/apps/web/tests/steering.e2e.ts b/apps/web/tests/steering.e2e.ts index e4617e2a04..e3ed1ddac8 100644 --- a/apps/web/tests/steering.e2e.ts +++ b/apps/web/tests/steering.e2e.ts @@ -20,13 +20,22 @@ import { afterAll, beforeAll, describe, expect, it, onTestFailed } from 'vitest' import { parseSessionLog } from '@deepseek-ai/dsh-llm-replay' import type { SessionEvent } from '@deepseek-ai/dsh-session' import { - assertFixtureInventory, fixtureUserPrompts, launchWebScaffold, recordFixture, - watchConsole, webSnapshotMode, type WebScaffold, + assertFixtureInventory, captureStableAria, compareOrRefreshGolden, fixtureUserPrompts, + launchWebScaffold, recordFixture, watchConsole, webSnapshotMode, type WebScaffold, } from './scaffold.ts' import { saveFailureShot } from './support.ts' const SNAPSHOT_DIR = fileURLToPath(new URL('./snapshots/steering', import.meta.url)) const FIXTURE = join(SNAPSHOT_DIR, 'session.jsonl') +// Two goldens for the two distinct states this interaction produces: the +// mid-turn moment (steer ACCEPTED but deliberately invisible — the loop +// drains steering at the step boundary, so no interjection bubble exists +// while the question still blocks the step) and the settled transcript +// (badged bubble in place, final reply obeying it). The pair pins the +// timing semantics visually: if the client ever starts rendering pending +// steers eagerly, the mid-steer golden flips first. +const MID_EXPECTED = join(SNAPSHOT_DIR, 'mid-steer.expected.md') +const SETTLED_EXPECTED = join(SNAPSHOT_DIR, 'settled.expected.md') const MODE = webSnapshotMode() const PROMPT = 'Use the ask_user_question tool to ask me exactly one question with id "checkpoint", question "Ready to continue?", header "Checkpoint", and options labeled "Yes" and "No". After I answer, reply with one short sentence acknowledging my answer and stop.' @@ -104,6 +113,17 @@ describe('web e2e: mid-turn steering lands durably and visibly', () => { }, { sessionId: liveSessionId!, text: STEER }) expect(reply.result?.ok).toBe(true) + if (MODE !== 'record') { + // Mid-turn golden: the ACCEPTED steer is durable in the inbox but the + // loop drains steering only at the step boundary, so no steering/message + // exists yet and no interjection bubble renders — the composer still + // blocks, alone. The DOM is stable here (no further SSE frames can + // arrive until the question is answered), making this state capturable. + expect(await page.getByText('插话').count()).toBe(0) + const snapshot = await captureStableAria(page, '[class*="centerCol"]', scaffold.workspaceCwd) + await compareOrRefreshGolden(MID_EXPECTED, snapshot, MODE) + } + // Answer the composer; the tool result closes the step, the loop drains // the steer as steering/message, and the steered continuation runs the // final model call. @@ -137,10 +157,14 @@ describe('web e2e: mid-turn steering lands durably and visibly', () => { await expect.poll(() => page.getByText('Interjection:', { exact: false }).count(), { timeout: 10_000 }).toBe(1) await expect.poll(() => page.getByText('BANANA', { exact: false }).count(), { timeout: 10_000 }).toBeGreaterThanOrEqual(2) expect(await page.locator('[data-question-key]').count()).toBe(0) + // Settled golden: badge + interjection between the question round trip + // and the obeying reply, composer takeover gone. + const snapshot = await captureStableAria(page, '[class*="centerCol"]', scaffold.workspaceCwd) + await compareOrRefreshGolden(SETTLED_EXPECTED, snapshot, MODE) expect(tripwire.pageErrors).toEqual([]) }, 200_000) it.skipIf(MODE === 'record')('keeps the fixture inventory closed', async () => { - await assertFixtureInventory(SNAPSHOT_DIR, ['session.jsonl']) + await assertFixtureInventory(SNAPSHOT_DIR, ['session.jsonl', 'mid-steer.expected.md', 'settled.expected.md']) }) })