test(web): live-turn interaction scenarios — cancel, error, retry, question composer, steering

Five browser e2e scenarios over the existing keyless lane, one recorded base fixture per spec family: - live-interactions: one tool-free recorded turn + per-run override sidecars authored in the spec (content single-sourced from the fixture via deriveReplayScript, minted into a spec-owned temp dir). Cancel uses a hang patch with a readyFile marker — the marker proves the stream is parked mid-turn before the Stop click, so mid-stream cancellation is deterministic by construction (turn/end 'aborted', composer re-enabled). AUTH pins the non-retryable path: turn/end 'error', zero llm/retry events, composer recovers; FIXME(web-error-surface) marks the found product gap (no error copy renders — the client consumes no agent/error frames and a pre-chunk failure freezes no partial). SERVER retry appends the fixture's own success after an injected throw and proves llm-retry end-to-end in the browser via the durable llm/retry record. - question-composer: the shipped ask_user_question takeover blocks the turn mid-step on the real userInteraction seam; the test answers through the composer (the one sanctioned model-content-reactive drive step: the turn cannot complete without it) and the tool result carries the answer. Adds the composer waiting-state aria golden. - steering: steers mid-turn while the composer blocks the step (the deterministic mid-turn window). The steer rides the real wire (session.prompt mode:'steer' POSTed from the page; the locked composer has no steering gesture yet — TODO(web-steer-composer)); downstream is all product: gateway -> Agent.steer -> step-boundary drain -> durable steering/message -> SSE -> badged interjection bubble. Record mode rejects a fixture whose live reply ignored the steer. Scaffold gains the replayOverride passthrough; specs register in both tsconfig planes (client exclude, host include).
2026-07-26 03:32:17 +08:00
parent 90d91c3cf9
commit 04b7f517ae
10 changed files with 841 additions and 0 deletions
--- a/apps/web/tests/steering.e2e.ts
+++ b/apps/web/tests/steering.e2e.ts
@@ -0,0 +1,146 @@
+// Web e2e scenario: mid-turn steering, end to end. The composer locks while a
+// turn runs, so the product UI has no steering gesture yet — the steer is
+// POSTed from the page itself over the same same-origin /api transport the
+// client uses (TODO(web-steer-composer): drive this through a composer
+// gesture once one exists). Everything downstream is product: the gateway
+// routes mode:'steer' to Agent.steer, the loop drains it at the step
+// boundary into a durable steering/message event, the SSE mux pushes it, and
+// the transcript renders the badged interjection bubble. The question
+// composer supplies the deterministic mid-turn window: while ask_user_question
+// blocks, the turn is provably running, so record and replay perform the
+// identical steer-then-answer sequence with zero timing dependence — and the
+// recorded final reply proves the steer reached the MODEL (it obeys an
+// instruction that only the steering message carries).
+import { readFile } from 'node:fs/promises'
+import { fileURLToPath } from 'node:url'
+import { join } from 'node:path'
+import type { Browser, Page } from 'playwright'
+import { chromium } from 'playwright'
+import { afterAll, beforeAll, describe, expect, it, onTestFailed } from 'vitest'
+import { parseSessionLog } from '@deepseek-ai/dsh-llm-replay'
+import type { SessionEvent } from '@deepseek-ai/dsh-session'
+import {
+  assertFixtureInventory, fixtureUserPrompts, launchWebScaffold, recordFixture,
+  watchConsole, webSnapshotMode, type WebScaffold,
+} from './scaffold.ts'
+import { saveFailureShot } from './support.ts'
+
+const SNAPSHOT_DIR = fileURLToPath(new URL('./snapshots/steering', import.meta.url))
+const FIXTURE = join(SNAPSHOT_DIR, 'session.jsonl')
+const MODE = webSnapshotMode()
+
+const PROMPT = 'Use the ask_user_question tool to ask me exactly one question with id "checkpoint", question "Ready to continue?", header "Checkpoint", and options labeled "Yes" and "No". After I answer, reply with one short sentence acknowledging my answer and stop.'
+const STEER = 'Interjection: include the word BANANA in your final reply.'
+
+/** Concatenated assistant text deltas — the model-visible reply body. */
+function assistantText(events: SessionEvent[]): string {
+  return events
+    .filter(e => e.type === 'assistant/chunk')
+    .map((e) => {
+      const chunk = (e as SessionEvent & { data: { chunk: { type: string; text?: string } } }).data.chunk
+      return chunk.type === 'text-delta' ? chunk.text ?? '' : ''
+    })
+    .join('')
+}
+
+describe('web e2e: mid-turn steering lands durably and visibly', () => {
+  let scaffold: WebScaffold
+  let browser: Browser
+  let page: Page
+  let tripwire: ReturnType<typeof watchConsole>
+  let liveSessionId: string | undefined
+  const sessionEvents: SessionEvent[] = []
+
+  beforeAll(async () => {
+    scaffold = await launchWebScaffold(MODE === 'record' ? {} : { replayFixture: FIXTURE, paceMs: 15 })
+    scaffold.ctx.on('session/event', (session, event) => {
+      liveSessionId ??= session.id
+      sessionEvents.push(event)
+    })
+    browser = await chromium.launch()
+    page = await browser.newPage({ viewport: { width: 1680, height: 1000 } })
+    tripwire = watchConsole(page)
+    await page.goto(scaffold.baseUrl, { waitUntil: 'load' })
+    await page.waitForSelector('[class*="frame"]', { timeout: 30_000 })
+  }, 120_000)
+
+  afterAll(async () => {
+    await browser?.close()
+    await scaffold?.close()
+  })
+
+  it('steers during the blocked step; the interjection is logged, rendered, and obeyed', async () => {
+    onTestFailed(() => saveFailureShot(page, 'web-e2e-steering'))
+    if (MODE !== 'record') {
+      // The steer must NOT be a user/message — it lands as steering/message.
+      expect(fixtureUserPrompts(await readFile(FIXTURE, 'utf8'))).toEqual([PROMPT])
+    }
+    const input = page.locator('textarea').first()
+    await input.waitFor({ timeout: 10_000 })
+    const settled = scaffold.whenTurnSettled(MODE === 'record' ? 180_000 : 30_000)
+    await input.fill(PROMPT)
+    await input.press('Enter')
+
+    // The blocked composer is the mid-turn barrier: its presence proves the
+    // ask_user_question step is executing, i.e. the turn is running NOW.
+    const composer = page.locator('[data-question-key]')
+    await composer.waitFor({ timeout: MODE === 'record' ? 120_000 : 30_000 })
+
+    // Steer through the real wire from the page (same envelope + endpoint the
+    // web client's session.prompt uses). accepted:true is the transport proof.
+    expect(liveSessionId).toBeDefined()
+    const reply = await page.evaluate(async ({ sessionId, text }) => {
+      const response = await fetch('/api/session.prompt', {
+        method: 'POST',
+        headers: { 'content-type': 'application/json' },
+        body: JSON.stringify({
+          type: 'client-request',
+          rpcId: crypto.randomUUID(),
+          method: 'session.prompt',
+          payload: { sessionId, mode: 'steer', content: [{ type: 'text', text }] },
+        }),
+      })
+      return await response.json() as { result?: { ok?: boolean } }
+    }, { sessionId: liveSessionId!, text: STEER })
+    expect(reply.result?.ok).toBe(true)
+
+    // Answer the composer; the tool result closes the step, the loop drains
+    // the steer as steering/message, and the steered continuation runs the
+    // final model call.
+    await composer.getByRole('radio', { name: 'Yes' }).click()
+    await composer.getByRole('radio', { name: 'Yes' }).press('Enter')
+    await settled
+
+    if (MODE === 'record') {
+      const sessionId = await settled
+      await recordFixture(scaffold, sessionId, FIXTURE)
+      // Fixture honesty: a recording where the live model ignored the steer
+      // would replay as a vacuous scenario — reject it and re-record instead.
+      const recorded = parseSessionLog(await readFile(FIXTURE, 'utf8'))
+      expect(recorded.filter(e => e.type === 'steering/message')).toHaveLength(1)
+      expect(assistantText(recorded)).toContain('BANANA')
+      return
+    }
+
+    // Durable: exactly one steering/message, inside turn 1, carrying the text.
+    const steerEvents = sessionEvents.filter(e => e.type === 'steering/message')
+    expect(steerEvents).toHaveLength(1)
+    expect((steerEvents[0] as SessionEvent & { data: { turn: number } }).data.turn).toBe(1)
+    expect(JSON.stringify(steerEvents[0])).toContain('BANANA')
+    const turnEnds = sessionEvents.filter(e => e.type === 'turn/end')
+    expect(turnEnds).toHaveLength(1)
+    expect((turnEnds[0] as SessionEvent & { data: { reason: { kind: string } } }).data.reason.kind).toBe('completed')
+
+    // Visible: the badged interjection bubble plus the reply that obeys it
+    // (steer text + final reply each contain the marker word).
+    await expect.poll(() => page.getByText('插话').count(), { timeout: 15_000 }).toBe(1)
+    await expect.poll(() => page.getByText('Interjection:', { exact: false }).count(), { timeout: 10_000 }).toBe(1)
+    await expect.poll(() => page.getByText('BANANA', { exact: false }).count(), { timeout: 10_000 }).toBeGreaterThanOrEqual(2)
+    expect(await page.locator('[data-question-key]').count()).toBe(0)
+    expect(tripwire.pageErrors).toEqual([])
+  }, 200_000)
+
+  it.skipIf(MODE === 'record')('keeps the fixture inventory closed', async () => {
+    await assertFixtureInventory(SNAPSHOT_DIR, ['session.jsonl'])
+  })
+})