From 04b7f517aebc6526d697a0c9b5b625bac73f2472 Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Sun, 26 Jul 2026 03:32:17 +0800 Subject: [PATCH] =?UTF-8?q?test(web):=20live-turn=20interaction=20scenario?= =?UTF-8?q?s=20=E2=80=94=20cancel,=20error,=20retry,=20question=20composer?= =?UTF-8?q?,=20steering?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Five browser e2e scenarios over the existing keyless lane, one recorded base fixture per spec family: - live-interactions: one tool-free recorded turn + per-run override sidecars authored in the spec (content single-sourced from the fixture via deriveReplayScript, minted into a spec-owned temp dir). Cancel uses a hang patch with a readyFile marker — the marker proves the stream is parked mid-turn before the Stop click, so mid-stream cancellation is deterministic by construction (turn/end 'aborted', composer re-enabled). AUTH pins the non-retryable path: turn/end 'error', zero llm/retry events, composer recovers; FIXME(web-error-surface) marks the found product gap (no error copy renders — the client consumes no agent/error frames and a pre-chunk failure freezes no partial). SERVER retry appends the fixture's own success after an injected throw and proves llm-retry end-to-end in the browser via the durable llm/retry record. - question-composer: the shipped ask_user_question takeover blocks the turn mid-step on the real userInteraction seam; the test answers through the composer (the one sanctioned model-content-reactive drive step: the turn cannot complete without it) and the tool result carries the answer. Adds the composer waiting-state aria golden. - steering: steers mid-turn while the composer blocks the step (the deterministic mid-turn window). The steer rides the real wire (session.prompt mode:'steer' POSTed from the page; the locked composer has no steering gesture yet — TODO(web-steer-composer)); downstream is all product: gateway -> Agent.steer -> step-boundary drain -> durable steering/message -> SSE -> badged interjection bubble. Record mode rejects a fixture whose live reply ignored the steer. Scaffold gains the replayOverride passthrough; specs register in both tsconfig planes (client exclude, host include). --- apps/web/tests/live-interactions.e2e.ts | 176 ++++++++++++++++++ apps/web/tests/question-composer.e2e.ts | 99 ++++++++++ apps/web/tests/scaffold.ts | 7 + .../snapshots/live-interactions/session.jsonl | 93 +++++++++ .../snapshots/question-composer/session.jsonl | 147 +++++++++++++++ .../question-composer/ui.expected.md | 23 +++ .../tests/snapshots/steering/session.jsonl | 144 ++++++++++++++ apps/web/tests/steering.e2e.ts | 146 +++++++++++++++ apps/web/tsconfig.json | 3 + tsconfig.host.json | 3 + 10 files changed, 841 insertions(+) create mode 100644 apps/web/tests/live-interactions.e2e.ts create mode 100644 apps/web/tests/question-composer.e2e.ts create mode 100644 apps/web/tests/snapshots/live-interactions/session.jsonl create mode 100644 apps/web/tests/snapshots/question-composer/session.jsonl create mode 100644 apps/web/tests/snapshots/question-composer/ui.expected.md create mode 100644 apps/web/tests/snapshots/steering/session.jsonl create mode 100644 apps/web/tests/steering.e2e.ts diff --git a/apps/web/tests/live-interactions.e2e.ts b/apps/web/tests/live-interactions.e2e.ts new file mode 100644 index 0000000000..632dc79085 --- /dev/null +++ b/apps/web/tests/live-interactions.e2e.ts @@ -0,0 +1,176 @@ +// Web e2e scenarios: live-turn interactions — cancellation, error surfacing, +// and transient-retry recovery, all through the real composition and wire. +// The model seam is dsh-llm-replay with override sidecars: `hang` (+ a +// readyFile marker) makes mid-stream cancel deterministic by construction, +// `throw` entries express provider failures by stable code, and `{ patches }` +// augmentation injects a transient throw before the recorded success so +// llm-retry's recovery is proven end-to-end in the browser. Sidecar CONTENT +// is authored here (single-sourced against the fixture via deriveReplayScript +// — no committed copy of recorded chunks); the file is a per-run artifact in +// the temp workspace. One recorded base fixture serves all three scenarios. +import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises' +import { existsSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { fileURLToPath } from 'node:url' +import { join } from 'node:path' +import type { Browser, Page } from 'playwright' +import { chromium } from 'playwright' +import { afterEach, describe, expect, it, onTestFailed } from 'vitest' +import { deriveReplayScript, parseSessionLog } from '@deepseek-ai/dsh-llm-replay' +import type { ReplayOverrideDoc } from '@deepseek-ai/dsh-llm-replay' +import type { SessionEvent } from '@deepseek-ai/dsh-session' +import { + assertFixtureInventory, fixtureUserPrompts, launchWebScaffold, recordFixture, + watchConsole, webSnapshotMode, type WebScaffold, +} from './scaffold.ts' +import { saveFailureShot } from './support.ts' + +const SNAPSHOT_DIR = fileURLToPath(new URL('./snapshots/live-interactions', import.meta.url)) +const FIXTURE = join(SNAPSHOT_DIR, 'session.jsonl') +const MODE = webSnapshotMode() + +// The recorded base: one text-only turn whose derived script the sidecars +// patch. Kept deliberately tool-free so the derived script is exactly one +// model call. +const PROMPT = 'Reply with a one-sentence description of event sourcing, then stop.' + +/** turn/end reasons observed, in order. */ +function turnEndReasons(events: SessionEvent[]): string[] { + return events + .filter(e => e.type === 'turn/end') + .map(e => (e as SessionEvent & { data: { reason: { kind: string } } }).data.reason.kind) +} + +describe('web e2e: live-turn interactions (cancel / error / retry)', () => { + let scaffold: WebScaffold | undefined + let browser: Browser | undefined + let page: Page + let tripwire: ReturnType + let sessionEvents: SessionEvent[] + let sidecarDir: string | undefined + + afterEach(async () => { + await browser?.close().catch(() => undefined) + browser = undefined + await scaffold?.close().catch(() => undefined) + scaffold = undefined + if (sidecarDir !== undefined) await rm(sidecarDir, { recursive: true, force: true }).catch(() => undefined) + sidecarDir = undefined + }) + + /** Boot scaffold + page with an optional override doc materialized per run. */ + async function launch(buildOverride?: (sidecarHome: string) => ReplayOverrideDoc): Promise { + sessionEvents = [] + let overridePath: string | undefined + if (buildOverride !== undefined) { + // The sidecar CONTENT is authored in this spec; the file is a per-run + // artifact minted in a spec-owned temp dir. It must exist BEFORE the + // scaffold boots — installLlmReplay resolves the script at install. + sidecarDir = await mkdtemp(join(tmpdir(), 'dsh-web-e2e-sidecar-')) + overridePath = join(sidecarDir, 'replay.override.json') + await writeFile(overridePath, JSON.stringify(buildOverride(sidecarDir))) + } + scaffold = await launchWebScaffold({ + replayFixture: FIXTURE, + ...(overridePath === undefined ? {} : { replayOverride: overridePath }), + }) + scaffold.ctx.on('session/event', (_session, event: SessionEvent) => { sessionEvents.push(event) }) + browser = await chromium.launch() + page = await browser.newPage({ viewport: { width: 1680, height: 1000 } }) + tripwire = watchConsole(page) + await page.goto(scaffold.baseUrl, { waitUntil: 'load' }) + await page.waitForSelector('[class*="frame"]', { timeout: 30_000 }) + } + + /** + * Type the recorded prompt and send, with the settled barrier pre-armed. + * Returned WRAPPED ({ settled }) — a bare returned promise would be + * flattened by the caller's await, blocking on turn/end before the caller + * can act mid-turn (the cancel scenario's whole point). + */ + async function sendPrompt(timeoutMs?: number): Promise<{ settled: ReturnType }> { + const input = page.locator('textarea').first() + await input.waitFor({ timeout: 10_000 }) + const settled = scaffold!.whenTurnSettled(timeoutMs) + await input.fill(PROMPT) + await input.press('Enter') + return { settled } + } + + it.skipIf(MODE !== 'record')('records the base fixture live through the composer', async () => { + await launch() + onTestFailed(() => saveFailureShot(page, 'web-e2e-interactions-record')) + const { settled } = await sendPrompt(180_000) + const sessionId = await settled + await recordFixture(scaffold!, sessionId, FIXTURE) + }, 200_000) + + it.skipIf(MODE === 'record')('cancels a hung stream deterministically via the readyFile marker', async () => { + expect(fixtureUserPrompts(await readFile(FIXTURE, 'utf8'))).toEqual([PROMPT]) + let marker = '' + await launch((sidecarHome) => { + marker = join(sidecarHome, '.hang-ready') + return { patches: [{ at: 0, entry: { kind: 'hang', readyFile: marker } }] } + }) + onTestFailed(() => saveFailureShot(page, 'web-e2e-cancel')) + const { settled } = await sendPrompt() + // The marker IS the synchronization: the stream is provably parked in the + // hang (prefix chunks delivered to the loop) before the stop click. + await expect.poll(() => existsSync(marker), { timeout: 15_000 }).toBe(true) + await page.getByRole('button', { name: 'Stop generating' }).click() + await settled + expect(turnEndReasons(sessionEvents).at(-1)).toBe('aborted') + // Composer recovered; no streaming node lingers. + await expect.poll(() => page.locator('textarea').first().isEnabled(), { timeout: 10_000 }).toBe(true) + expect(await page.locator('[data-streaming="true"]').count()).toBe(0) + expect(tripwire.pageErrors).toEqual([]) + }, 120_000) + + it.skipIf(MODE === 'record')('surfaces a non-retryable AUTH failure without retrying', async () => { + await launch(() => ({ + patches: [{ at: 0, entry: { kind: 'throw', chunks: [], message: 'invalid api key', code: 'AUTH' } }], + })) + onTestFailed(() => saveFailureShot(page, 'web-e2e-error-auth')) + const { settled } = await sendPrompt() + await settled + expect(turnEndReasons(sessionEvents).at(-1)).toBe('error') + // AUTH is outside llm-retry's retryable set: no retry record. + expect(sessionEvents.filter(e => e.type === 'llm/retry').length).toBe(0) + // Product gap found by this lane, pinned as-is: the client consumes no + // agent/error frames and a pre-chunk failure freezes no partial, so THIS + // failure renders no error copy anywhere — the user sees the send simply + // stop. FIXME(web-error-surface): assert visible error text here once the + // web UI grows an error rendering; until then the pinned contract is + // "no crash, composer recovers, turn logged as error". + await expect.poll(() => page.locator('textarea').first().isEnabled(), { timeout: 10_000 }).toBe(true) + expect(await page.locator('[data-streaming="true"]').count()).toBe(0) + expect(tripwire.pageErrors).toEqual([]) + }, 120_000) + + it.skipIf(MODE === 'record')('recovers a transient SERVER failure through llm-retry and completes', async () => { + const derived = deriveReplayScript(parseSessionLog(await readFile(FIXTURE, 'utf8'))) + expect(derived).toHaveLength(1) + await launch(() => ({ + patches: [ + { at: 0, entry: { kind: 'throw', chunks: [], message: 'upstream 503', code: 'SERVER' } }, + // Append the fixture's own success as the retry attempt — single- + // sourced from the recording, never copied into a committed sidecar. + { at: 1, entry: derived[0]! }, + ], + })) + onTestFailed(() => saveFailureShot(page, 'web-e2e-retry')) + // llm-retry backs off ~500ms before the second attempt. + const { settled } = await sendPrompt(60_000) + await settled + expect(turnEndReasons(sessionEvents).at(-1)).toBe('completed') + // The durable retry record proves the second attempt (request/header logs + // only on change, so attempt count is invisible there). + expect(sessionEvents.filter(e => e.type === 'llm/retry').length).toBeGreaterThanOrEqual(1) + await expect.poll(() => page.getByText('event sourcing', { exact: false }).count(), { timeout: 10_000 }).toBeGreaterThan(0) + expect(tripwire.pageErrors).toEqual([]) + }, 120_000) + + it.skipIf(MODE === 'record')('keeps the fixture inventory closed', async () => { + await assertFixtureInventory(SNAPSHOT_DIR, ['session.jsonl']) + }) +}) diff --git a/apps/web/tests/question-composer.e2e.ts b/apps/web/tests/question-composer.e2e.ts new file mode 100644 index 0000000000..9678a7a648 --- /dev/null +++ b/apps/web/tests/question-composer.e2e.ts @@ -0,0 +1,99 @@ +// Web e2e scenario: the resident question composer. The shipped composition +// already exposes ask_user_question (the ui-question row's node half mounts +// the tool), so a recorded turn where the model asks blocks mid-turn on the +// real userInteraction seam: the composer renders in the browser, the test +// answers through it, and the turn completes with the answer in the log. +// Replay is fully deterministic — the question content arrives from replayed +// chunks, the composer wait is real, and the answer click is the test's own +// gesture (the ONE place a drive step legitimately reacts to model content: +// the turn cannot complete without it, in record and replay alike). +import { readFile } from 'node:fs/promises' +import { fileURLToPath } from 'node:url' +import { join } from 'node:path' +import type { Browser, Page } from 'playwright' +import { chromium } from 'playwright' +import { afterAll, beforeAll, describe, expect, it, onTestFailed } from 'vitest' +import type { SessionEvent } from '@deepseek-ai/dsh-session' +import { + assertFixtureInventory, captureStableAria, compareOrRefreshGolden, fixtureUserPrompts, + launchWebScaffold, recordFixture, watchConsole, webSnapshotMode, type WebScaffold, +} from './scaffold.ts' +import { saveFailureShot } from './support.ts' + +const SNAPSHOT_DIR = fileURLToPath(new URL('./snapshots/question-composer', import.meta.url)) +const FIXTURE = join(SNAPSHOT_DIR, 'session.jsonl') +const UI_EXPECTED = join(SNAPSHOT_DIR, 'ui.expected.md') +const MODE = webSnapshotMode() + +const PROMPT = 'Use the ask_user_question tool to ask me exactly one question with id "color", question "Which color do you prefer?", header "Pick one", and options labeled "Blue" and "Green". After I answer, reply with the single word DONE and stop.' + +describe('web e2e: resident question composer round trip', () => { + let scaffold: WebScaffold + let browser: Browser + let page: Page + let tripwire: ReturnType + const sessionEvents: SessionEvent[] = [] + + beforeAll(async () => { + scaffold = await launchWebScaffold(MODE === 'record' ? {} : { replayFixture: FIXTURE, paceMs: 15 }) + scaffold.ctx.on('session/event', (_session, event: SessionEvent) => { sessionEvents.push(event) }) + browser = await chromium.launch() + page = await browser.newPage({ viewport: { width: 1680, height: 1000 } }) + tripwire = watchConsole(page) + await page.goto(scaffold.baseUrl, { waitUntil: 'load' }) + await page.waitForSelector('[class*="frame"]', { timeout: 30_000 }) + }, 120_000) + + afterAll(async () => { + await browser?.close() + await scaffold?.close() + }) + + it('asks through the composer, answers, and completes with the answer logged', async () => { + onTestFailed(() => saveFailureShot(page, 'web-e2e-question')) + if (MODE !== 'record') { + expect(fixtureUserPrompts(await readFile(FIXTURE, 'utf8'))).toEqual([PROMPT]) + } + const input = page.locator('textarea').first() + await input.waitFor({ timeout: 10_000 }) + const settled = scaffold.whenTurnSettled(MODE === 'record' ? 180_000 : 30_000) + await input.fill(PROMPT) + await input.press('Enter') + + // The composer takes over the input area while the tool blocks. Its + // presence is a STABLE waiting state (not a transient): it stays until + // answered, so a plain waitFor is race-free. + const composer = page.locator('[data-question-key]') + await composer.waitFor({ timeout: MODE === 'record' ? 120_000 : 30_000 }) + await expect.poll(() => composer.getByText('Which color do you prefer?').count(), { timeout: 10_000 }).toBeGreaterThan(0) + + if (MODE !== 'record') { + // Golden of the composer's waiting state (the transcript region golden + // is #612's job; this pins the question surface). + const snapshot = await captureStableAria(page, '[data-question-key]', scaffold.workspaceCwd) + await compareOrRefreshGolden(UI_EXPECTED, snapshot, MODE) + } + + await composer.getByRole('radio', { name: 'Blue' }).click() + // Submit: Enter on the focused option (the composer's documented submit). + await composer.getByRole('radio', { name: 'Blue' }).press('Enter') + + const sessionId = await settled + if (MODE === 'record') { + await recordFixture(scaffold, sessionId, FIXTURE) + return + } + // World state: the tool result carries the chosen answer, and DONE lands. + const results = sessionEvents.filter(e => e.type === 'tool/result') + expect(JSON.stringify(results.at(-1))).toContain('Blue') + await expect.poll(() => page.getByText('DONE', { exact: true }).count(), { timeout: 15_000 }).toBeGreaterThanOrEqual(1) + // Composer gone; regular input restored. + expect(await page.locator('[data-question-key]').count()).toBe(0) + await expect.poll(() => page.locator('textarea').first().isEnabled(), { timeout: 10_000 }).toBe(true) + expect(tripwire.pageErrors).toEqual([]) + }, 200_000) + + it.skipIf(MODE === 'record')('keeps the fixture inventory closed', async () => { + await assertFixtureInventory(SNAPSHOT_DIR, ['session.jsonl', 'ui.expected.md']) + }) +}) diff --git a/apps/web/tests/scaffold.ts b/apps/web/tests/scaffold.ts index d858e0f7ad..98fe05f0ca 100644 --- a/apps/web/tests/scaffold.ts +++ b/apps/web/tests/scaffold.ts @@ -101,6 +101,12 @@ export interface LaunchOptions { * mounts). */ replayFixture?: string + /** + * Optional replay.override.json sidecar (whole-script replacement or + * `{ patches }` augmentation) for throw/hang scenarios not expressible as + * recorded chunks; replay/refresh only. + */ + replayOverride?: string /** Per-chunk replay pacing (ms) so the browser observes genuinely incremental SSE; replay/refresh only. */ paceMs?: number } @@ -179,6 +185,7 @@ export async function launchWebScaffold(options: LaunchOptions = {}): Promise e.type === 'assistant/chunk') + .map((e) => { + const chunk = (e as SessionEvent & { data: { chunk: { type: string; text?: string } } }).data.chunk + return chunk.type === 'text-delta' ? chunk.text ?? '' : '' + }) + .join('') +} + +describe('web e2e: mid-turn steering lands durably and visibly', () => { + let scaffold: WebScaffold + let browser: Browser + let page: Page + let tripwire: ReturnType + let liveSessionId: string | undefined + const sessionEvents: SessionEvent[] = [] + + beforeAll(async () => { + scaffold = await launchWebScaffold(MODE === 'record' ? {} : { replayFixture: FIXTURE, paceMs: 15 }) + scaffold.ctx.on('session/event', (session, event) => { + liveSessionId ??= session.id + sessionEvents.push(event) + }) + browser = await chromium.launch() + page = await browser.newPage({ viewport: { width: 1680, height: 1000 } }) + tripwire = watchConsole(page) + await page.goto(scaffold.baseUrl, { waitUntil: 'load' }) + await page.waitForSelector('[class*="frame"]', { timeout: 30_000 }) + }, 120_000) + + afterAll(async () => { + await browser?.close() + await scaffold?.close() + }) + + it('steers during the blocked step; the interjection is logged, rendered, and obeyed', async () => { + onTestFailed(() => saveFailureShot(page, 'web-e2e-steering')) + if (MODE !== 'record') { + // The steer must NOT be a user/message — it lands as steering/message. + expect(fixtureUserPrompts(await readFile(FIXTURE, 'utf8'))).toEqual([PROMPT]) + } + const input = page.locator('textarea').first() + await input.waitFor({ timeout: 10_000 }) + const settled = scaffold.whenTurnSettled(MODE === 'record' ? 180_000 : 30_000) + await input.fill(PROMPT) + await input.press('Enter') + + // The blocked composer is the mid-turn barrier: its presence proves the + // ask_user_question step is executing, i.e. the turn is running NOW. + const composer = page.locator('[data-question-key]') + await composer.waitFor({ timeout: MODE === 'record' ? 120_000 : 30_000 }) + + // Steer through the real wire from the page (same envelope + endpoint the + // web client's session.prompt uses). accepted:true is the transport proof. + expect(liveSessionId).toBeDefined() + const reply = await page.evaluate(async ({ sessionId, text }) => { + const response = await fetch('/api/session.prompt', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ + type: 'client-request', + rpcId: crypto.randomUUID(), + method: 'session.prompt', + payload: { sessionId, mode: 'steer', content: [{ type: 'text', text }] }, + }), + }) + return await response.json() as { result?: { ok?: boolean } } + }, { sessionId: liveSessionId!, text: STEER }) + expect(reply.result?.ok).toBe(true) + + // Answer the composer; the tool result closes the step, the loop drains + // the steer as steering/message, and the steered continuation runs the + // final model call. + await composer.getByRole('radio', { name: 'Yes' }).click() + await composer.getByRole('radio', { name: 'Yes' }).press('Enter') + await settled + + if (MODE === 'record') { + const sessionId = await settled + await recordFixture(scaffold, sessionId, FIXTURE) + // Fixture honesty: a recording where the live model ignored the steer + // would replay as a vacuous scenario — reject it and re-record instead. + const recorded = parseSessionLog(await readFile(FIXTURE, 'utf8')) + expect(recorded.filter(e => e.type === 'steering/message')).toHaveLength(1) + expect(assistantText(recorded)).toContain('BANANA') + return + } + + // Durable: exactly one steering/message, inside turn 1, carrying the text. + const steerEvents = sessionEvents.filter(e => e.type === 'steering/message') + expect(steerEvents).toHaveLength(1) + expect((steerEvents[0] as SessionEvent & { data: { turn: number } }).data.turn).toBe(1) + expect(JSON.stringify(steerEvents[0])).toContain('BANANA') + const turnEnds = sessionEvents.filter(e => e.type === 'turn/end') + expect(turnEnds).toHaveLength(1) + expect((turnEnds[0] as SessionEvent & { data: { reason: { kind: string } } }).data.reason.kind).toBe('completed') + + // Visible: the badged interjection bubble plus the reply that obeys it + // (steer text + final reply each contain the marker word). + await expect.poll(() => page.getByText('插话').count(), { timeout: 15_000 }).toBe(1) + await expect.poll(() => page.getByText('Interjection:', { exact: false }).count(), { timeout: 10_000 }).toBe(1) + await expect.poll(() => page.getByText('BANANA', { exact: false }).count(), { timeout: 10_000 }).toBeGreaterThanOrEqual(2) + expect(await page.locator('[data-question-key]').count()).toBe(0) + expect(tripwire.pageErrors).toEqual([]) + }, 200_000) + + it.skipIf(MODE === 'record')('keeps the fixture inventory closed', async () => { + await assertFixtureInventory(SNAPSHOT_DIR, ['session.jsonl']) + }) +}) diff --git a/apps/web/tsconfig.json b/apps/web/tsconfig.json index 54c5673451..fa92bde8ea 100644 --- a/apps/web/tsconfig.json +++ b/apps/web/tsconfig.json @@ -23,6 +23,9 @@ // cannot see both sides of the cordis Context merges). "exclude": [ "tests/scaffold.ts", + "tests/live-interactions.e2e.ts", + "tests/question-composer.e2e.ts", + "tests/steering.e2e.ts", "tests/replay-round-trip.e2e.ts", "tests/seeded-history.e2e.ts" ], diff --git a/tsconfig.host.json b/tsconfig.host.json index 7386119274..a6e24f2a52 100644 --- a/tsconfig.host.json +++ b/tsconfig.host.json @@ -10,6 +10,9 @@ "include": [ "apps/web/tests/scaffold.ts", "apps/web/tests/support.ts", + "apps/web/tests/live-interactions.e2e.ts", + "apps/web/tests/question-composer.e2e.ts", + "apps/web/tests/steering.e2e.ts", "apps/web/tests/replay-round-trip.e2e.ts", "apps/web/tests/seeded-history.e2e.ts", "examples/*/src/**/*.ts",