fix(review): harden web replay verification

Validate replay sidecars and cross-copy failure facts, make browser console tripwires and macOS temp paths deterministic, and wait for asynchronous TUI resume details. Keep the owning docs, translations, and generated catalog aligned.
This commit is contained in:
Tianyi Cui
2026-07-26 22:38:33 +08:00
parent b67ecc45fc
commit e2882f486b
20 changed files with 310 additions and 132 deletions

View File

@@ -17,7 +17,7 @@ import { chromium } from 'playwright'
import { afterAll, beforeAll, describe, expect, it, onTestFailed } from 'vitest'
import type { SessionEvent } from '@deepseek-ai/dsh-session'
import {
assertFixtureInventory, captureStableAria, compareOrRefreshGolden, fixtureUserPrompts,
acknowledgeReloadConnectionLoss, assertFixtureInventory, captureStableAria, compareOrRefreshGolden, fixtureUserPrompts,
launchWebScaffold, recordFixture, watchConsole, webSnapshotMode, type WebScaffold,
} from './scaffold.ts'
import { saveFailureShot } from './support.ts'
@@ -103,8 +103,10 @@ describe('web e2e: lifecycle & chrome (workspace flow / reload / dark mode)', ()
// (persisted under dsh.layout.panels) before reloading.
await page.getByRole('button', { name: 'Collapse sidebar' }).click()
await expect.poll(() => page.getByRole('button', { name: 'Open sidebar' }).count(), { timeout: 10_000 }).toBe(1)
const warningStart = tripwire.warnings.length
await page.reload({ waitUntil: 'load' })
await page.waitForSelector('[class*="frame"]', { timeout: 30_000 })
acknowledgeReloadConnectionLoss(tripwire, warningStart)
// Layout persisted: the sidebar comes back collapsed.
await expect.poll(() => page.getByRole('button', { name: 'Open sidebar' }).count(), { timeout: 10_000 }).toBe(1)
// Selection persisted (dsh.sessions.current) and history replayed: the
@@ -155,6 +157,7 @@ describe('web e2e: lifecycle & chrome (workspace flow / reload / dark mode)', ()
}, 60_000)
it.skipIf(MODE === 'record')('keeps the fixture inventory closed', async () => {
expect(tripwire.warnings).toEqual([])
await assertFixtureInventory(SNAPSHOT_DIR, ['session.jsonl', 'hero.expected.md', 'reloaded.expected.md'])
})
})

View File

@@ -142,6 +142,7 @@ describe('web e2e: live-turn interactions (cancel / error / retry)', () => {
const snapshot = await captureStableAria(page, '[class*="centerCol"]', scaffold!.workspaceCwd)
await compareOrRefreshGolden(CANCEL_EXPECTED, snapshot, MODE)
expect(tripwire.pageErrors).toEqual([])
expect(tripwire.warnings).toEqual([])
}, 120_000)
it.skipIf(MODE === 'record')('surfaces a non-retryable AUTH failure without retrying', async () => {
@@ -167,6 +168,7 @@ describe('web e2e: live-turn interactions (cancel / error / retry)', () => {
const snapshot = await captureStableAria(page, '[class*="centerCol"]', scaffold!.workspaceCwd)
await compareOrRefreshGolden(ERROR_EXPECTED, snapshot, MODE)
expect(tripwire.pageErrors).toEqual([])
expect(tripwire.warnings).toEqual([])
}, 120_000)
it.skipIf(MODE === 'record')('recovers a transient SERVER failure through llm-retry and completes', async () => {
@@ -194,6 +196,7 @@ describe('web e2e: live-turn interactions (cancel / error / retry)', () => {
const snapshot = await captureStableAria(page, '[class*="centerCol"]', scaffold!.workspaceCwd)
await compareOrRefreshGolden(RETRY_EXPECTED, snapshot, MODE)
expect(tripwire.pageErrors).toEqual([])
expect(tripwire.warnings).toEqual([])
}, 120_000)
it.skipIf(MODE === 'record')('keeps the fixture inventory closed', async () => {

View File

@@ -71,8 +71,8 @@ describe('web e2e: resident question composer round trip', () => {
await expect.poll(() => composer.getByText('Which color do you prefer?').count(), { timeout: 10_000 }).toBeGreaterThan(0)
if (MODE !== 'record') {
// Golden of the composer's waiting state (the transcript region golden
// is #612's job; this pins the question surface).
// This golden owns the stable question surface; the answered-state
// golden below owns the resulting transcript.
const snapshot = await captureStableAria(page, '[data-question-key]', scaffold.workspaceCwd)
await compareOrRefreshGolden(UI_EXPECTED, snapshot, MODE)
}
@@ -98,6 +98,7 @@ describe('web e2e: resident question composer round trip', () => {
const snapshot = await captureStableAria(page, '[class*="centerCol"]', scaffold.workspaceCwd)
await compareOrRefreshGolden(ANSWERED_EXPECTED, snapshot, MODE)
expect(tripwire.pageErrors).toEqual([])
expect(tripwire.warnings).toEqual([])
}, 200_000)
it.skipIf(MODE === 'record')('keeps the fixture inventory closed', async () => {

View File

@@ -18,7 +18,7 @@
// (the plugin-row path discards the ReplayHandle; the direct install keeps
// assertConsumed for the teardown fixture-consumption check).
import { existsSync, readFileSync } from 'node:fs'
import { mkdtemp, readFile, readdir, rm, utimes, writeFile } from 'node:fs/promises'
import { mkdtemp, readFile, readdir, realpath, rm, utimes, writeFile } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join, resolve } from 'node:path'
import { pathToFileURL } from 'node:url'
@@ -141,7 +141,7 @@ export async function launchWebScaffold(options: LaunchOptions = {}): Promise<We
throw new Error('web e2e record mode needs DEEPSEEK_API_KEY (env or repo-root .env)')
}
}
const workspaceCwd = await mkdtemp(join(tmpdir(), 'dsh-web-e2e-ws-'))
const workspaceCwd = await realpath(await mkdtemp(join(tmpdir(), 'dsh-web-e2e-ws-')))
let persistenceRoot: string
try {
persistenceRoot = await mkdtemp(join(tmpdir(), 'dsh-web-e2e-sessions-'))
@@ -453,3 +453,17 @@ export function watchConsole(page: Page): { warnings: string[]; pageErrors: stri
page.on('pageerror', (error) => { pageErrors.push(String(error)) })
return { warnings, pageErrors }
}
/**
* Remove only connection-loss warnings emitted after an intentional reload.
* Earlier warnings and all gap-repair/discontinuity warnings remain fatal.
* @param tripwire - the live console-warning collector.
* @param warningStart - warning count captured immediately before reloading.
*/
export function acknowledgeReloadConnectionLoss(
tripwire: ReturnType<typeof watchConsole>,
warningStart: number,
): void {
const reloadWarnings = tripwire.warnings.splice(warningStart)
tripwire.warnings.push(...reloadWarnings.filter(text => !/connection lost/i.test(text)))
}

View File

@@ -12,7 +12,7 @@ import { chromium } from 'playwright'
import { afterAll, beforeAll, describe, expect, it, onTestFailed } from 'vitest'
import { join } from 'node:path'
import {
assertFixtureInventory, captureStableAria, compareOrRefreshGolden,
acknowledgeReloadConnectionLoss, assertFixtureInventory, captureStableAria, compareOrRefreshGolden,
launchWebScaffold, watchConsole, webSnapshotMode, type WebScaffold,
} from './scaffold.ts'
import { saveFailureShot } from './support.ts'
@@ -36,17 +36,6 @@ describe('web e2e: settings modal, appearance gesture, language switch', () => {
await page.waitForSelector('[class*="frame"]', { timeout: 30_000 })
}, 120_000)
/**
* An INTENTIONAL reload tears the SSE stream mid-flight, so the dying
* page's reconnect note is expected — drain exactly those entries so the
* tripwire still fails the spec on any UNEXPECTED connection loss.
*/
const drainReloadWarnings = (): void => {
const kept = tripwire.warnings.filter(text => !/connection lost/i.test(text))
tripwire.warnings.length = 0
tripwire.warnings.push(...kept)
}
afterAll(async () => {
await browser?.close()
await scaffold?.close()
@@ -114,9 +103,10 @@ describe('web e2e: settings modal, appearance gesture, language switch', () => {
await page.keyboard.press('Escape')
// Reload: the preference survives boot (restore + presenter initial apply).
const warningStart = tripwire.warnings.length
await page.reload({ waitUntil: 'load' })
await page.waitForSelector('[class*="frame"]', { timeout: 30_000 })
drainReloadWarnings()
acknowledgeReloadConnectionLoss(tripwire, warningStart)
await page.emulateMedia({ colorScheme: 'light' })
const reloaded = await readState()
expect(reloaded.attr).toBe(true)
@@ -158,9 +148,10 @@ describe('web e2e: settings modal, appearance gesture, language switch', () => {
expect(await page.evaluate(() => localStorage.getItem('dsh.locale'))).toBe('en')
// Reload keeps English; then restore zh so shared page state (and the
// other specs' 设置-anchored selectors + goldens) see the default again.
const warningStart = tripwire.warnings.length
await page.reload({ waitUntil: 'load' })
await page.waitForSelector('[class*="frame"]', { timeout: 30_000 })
drainReloadWarnings()
acknowledgeReloadConnectionLoss(tripwire, warningStart)
const enTrigger = page.getByRole('button', { name: 'Settings' })
await enTrigger.waitFor({ timeout: 10_000 })
await enTrigger.click()

View File

@@ -162,6 +162,7 @@ describe('web e2e: mid-turn steering lands durably and visibly', () => {
const snapshot = await captureStableAria(page, '[class*="centerCol"]', scaffold.workspaceCwd)
await compareOrRefreshGolden(SETTLED_EXPECTED, snapshot, MODE)
expect(tripwire.pageErrors).toEqual([])
expect(tripwire.warnings).toEqual([])
}, 200_000)
it.skipIf(MODE === 'record')('keeps the fixture inventory closed', async () => {

View File

@@ -12,7 +12,7 @@ import type { Browser, Page } from 'playwright'
import { chromium } from 'playwright'
import { afterAll, beforeAll, describe, expect, it, onTestFailed } from 'vitest'
import {
assertFixtureInventory, launchWebScaffold, seedSession, watchConsole,
acknowledgeReloadConnectionLoss, assertFixtureInventory, launchWebScaffold, seedSession, watchConsole,
webSnapshotMode, type WebScaffold,
} from './scaffold.ts'
import { saveFailureShot } from './support.ts'
@@ -45,17 +45,6 @@ describe('web e2e: workspace management (create / rename / flat view / hover car
await page.waitForSelector('[class*="frame"]', { timeout: 30_000 })
}, 120_000)
/**
* An INTENTIONAL reload tears the SSE stream mid-flight, so the dying
* page's reconnect note is expected — drain exactly those entries so the
* tripwire still fails the spec on any UNEXPECTED connection loss.
*/
const drainReloadWarnings = (): void => {
const kept = tripwire.warnings.filter(text => !/connection lost/i.test(text))
tripwire.warnings.length = 0
tripwire.warnings.push(...kept)
}
afterAll(async () => {
await browser?.close()
await scaffold?.close()
@@ -108,9 +97,10 @@ describe('web e2e: workspace management (create / rename / flat view / hover car
expect(await page.getByText('alpha-ws', { exact: true }).count()).toBe(0)
// Host durability, then reload: the projection is rebuilt from the wire.
expect(scaffold.ctx.workspace.list().map(workspace => workspace.title)).toContain('gamma-ws')
const warningStart = tripwire.warnings.length
await page.reload({ waitUntil: 'load' })
await page.waitForSelector('[class*="frame"]', { timeout: 30_000 })
drainReloadWarnings()
acknowledgeReloadConnectionLoss(tripwire, warningStart)
await expect.poll(() => page.getByText('gamma-ws', { exact: true }).count(), { timeout: 15_000 }).toBeGreaterThanOrEqual(1)
expect(tripwire.pageErrors).toEqual([])
}, 90_000)
@@ -129,9 +119,10 @@ describe('web e2e: workspace management (create / rename / flat view / hover car
await expect.poll(() => page.locator('[role="treeitem"]').count(), { timeout: 10_000 }).toBeGreaterThanOrEqual(1)
expect(await page.evaluate(() => localStorage.getItem('dsh.workspace.view'))).toContain('flat')
// Persisted across reload; then restore grouped for inter-spec hygiene.
const warningStart = tripwire.warnings.length
await page.reload({ waitUntil: 'load' })
await page.waitForSelector('[class*="frame"]', { timeout: 30_000 })
drainReloadWarnings()
acknowledgeReloadConnectionLoss(tripwire, warningStart)
await expect.poll(() => page.getByText('Ungrouped', { exact: true }).count(), { timeout: 15_000 }).toBe(0)
await page.getByRole('button', { name: 'Group by' }).click()
await page.getByRole('menuitem', { name: 'WorkSpace' }).click()