Coder/deepseek-harness
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(web-e2e): rename harness -> scaffold; add interaction coverage

Browse Source 
The shared test module was named harness.ts inside a repo whose product
IS a harness — hopelessly ambiguous. Renamed to scaffold.ts with
launchWebScaffold/WebScaffold; tsconfig plane-split entries, the seam
JSDoc/README mentions, and both Agent Note languages updated.

Both scenarios gain a Playwright interaction step over the settled
transcript (after the golden capture, so committed aria surfaces stay
untouched): replay-round-trip clicks the reasoning fold open/closed over
wire-delivered state; seeded-history expands a read tool row rebuilt from
the cold log and asserts the recorded result text appears (read rows are
expand-in-place — rowExpands routes the click to the inline fold, not the
details column). test:web 30 passed | 1 skipped.
This commit is contained in:
Tianyi Cui
2026-07-24 22:18:19 +08:00
parent 8f97f95d7b
commit 6d3c25f494
9 changed files with 91 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
apps/web/tests/replay-round-trip.e2e.ts
View File

@@ -16,8 +16,8 @@ import { afterAll, beforeAll, describe, expect, it, onTestFailed } from 'vitest'
import type { SessionEvent } from '@deepseek-ai/dsh-session'
import {
assertFixtureInventory, captureStableAria, compareOrRefreshGolden, fixtureUserPrompts,
launchWebHarness, recordFixture, watchConsole, webSnapshotMode, type WebHarness,
} from './harness.ts'
launchWebScaffold, recordFixture, watchConsole, webSnapshotMode, type WebScaffold,
} from './scaffold.ts'
import { saveFailureShot } from './support.ts'
const SNAPSHOT_DIR = fileURLToPath(new URL('./snapshots/fresh-round-trip', import.meta.url))
@@ -31,27 +31,27 @@ const MODE = webSnapshotMode()
const PROMPT = 'Use the bash tool to run exactly: echo WEB_E2E_OK. Then reply with the single word DONE and stop.'
describe('web e2e: fresh round trip through the real assembly', () => {
let harness: WebHarness
let scaffold: WebScaffold
let browser: Browser
let page: Page
let tripwire: ReturnType<typeof watchConsole>
const sessionEvents: SessionEvent[] = []
beforeAll(async () => {
harness = await launchWebHarness({
scaffold = await launchWebScaffold({
...(MODE === 'record' ? {} : { replayFixture: FIXTURE, paceMs: 15 }),
})
harness.host.ctx.on('session/event', (_session, event: SessionEvent) => { sessionEvents.push(event) })
scaffold.host.ctx.on('session/event', (_session, event: SessionEvent) => { sessionEvents.push(event) })
browser = await chromium.launch()
page = await browser.newPage({ viewport: { width: 1680, height: 1000 } })
tripwire = watchConsole(page)
await page.goto(harness.baseUrl, { waitUntil: 'load' })
await page.goto(scaffold.baseUrl, { waitUntil: 'load' })
await page.waitForSelector('[class*="frame"]', { timeout: 30_000 })
}, 120_000)
afterAll(async () => {
await browser?.close()
await harness?.close()
await scaffold?.close()
})
it('drives the recorded prompt to a settled turn (all modes)', async () => {
@@ -63,12 +63,12 @@ describe('web e2e: fresh round trip through the real assembly', () => {
const input = page.locator('textarea').first()
await input.waitFor({ timeout: 10_000 })
// Arm the host-side settled barrier BEFORE the send click.
const settled = harness.whenTurnSettled()
const settled = scaffold.whenTurnSettled()
await input.fill(PROMPT)
await input.press('Enter')
const sessionId = await settled
if (MODE === 'record') {
await recordFixture(harness, sessionId, FIXTURE)
await recordFixture(scaffold, sessionId, FIXTURE)
}
}, 200_000)
@@ -96,14 +96,28 @@ describe('web e2e: fresh round trip through the real assembly', () => {
// while the whole-region golden churns.
await expect(page.getByRole('textbox').first().isVisible()).resolves.toBe(true)
expect(await page.getByText('WEB_E2E_OK', { exact: false }).count()).toBeGreaterThanOrEqual(1)
const snapshot = await captureStableAria(page, '[class*="centerCol"]', harness.workspaceCwd)
const snapshot = await captureStableAria(page, '[class*="centerCol"]', scaffold.workspaceCwd)
await compareOrRefreshGolden(UI_EXPECTED, snapshot, MODE)
})
it.skipIf(MODE === 'record')('expands and collapses the reasoning fold from its click target', async () => {
onTestFailed(() => saveFailureShot(page, 'web-e2e-round-trip-think'))
// Interaction over the REAL wire-delivered transcript (the fixture-client
// tier pins the same gesture against FixtureApiClient; this one runs on
// mux-frame-fed state). Runs after the golden capture so the committed
// aria surface stays the untouched settled state.
const think = page.getByRole('button', { name: /^Think/ }).first()
expect(await think.getAttribute('aria-expanded')).toBe('false')
await think.click()
await expect.poll(() => think.getAttribute('aria-expanded'), { timeout: 5_000 }).toBe('true')
await think.click()
await expect.poll(() => think.getAttribute('aria-expanded'), { timeout: 5_000 }).toBe('false')
})
it.skipIf(MODE === 'record')('stayed clean: no pageerrors, no reconnect self-healing, no server errors', async () => {
expect(tripwire.pageErrors).toEqual([])
expect(tripwire.warnings).toEqual([])
expect(harness.serverErrors).toEqual([])
expect(scaffold.serverErrors).toEqual([])
await assertFixtureInventory(SNAPSHOT_DIR, ['session.jsonl', 'ui.expected.md'])
})
})

40
apps/web/tests/harness.ts → apps/web/tests/scaffold.ts
View File

@@ -1,4 +1,4 @@
// Shared harness for the keyless browser e2e lane (Agent Note:
// Shared scaffold for the keyless browser e2e lane (Agent Note:
// .agents/notes/implemented/testing/2026-07-24-web-gui-browser-e2e-lane.md).
// Boots the REAL web assembly in-process from the exported production
// functions — startHost (bootHost spine) + mountWebPlugins + registry +
@@ -78,9 +78,9 @@ function loadRootEnv(): void {
}
}
/** A booted web harness: real assembly, mode-selected model backend, temp world. */
export interface WebHarness {
/** The active snapshot mode this harness booted under. */
/** A booted web scaffold: real assembly, mode-selected model backend, temp world. */
export interface WebScaffold {
/** The active snapshot mode this scaffold booted under. */
mode: WebSnapshotMode
/** Browser-facing origin (http://127.0.0.1:<bound port>). */
baseUrl: string
@@ -98,7 +98,7 @@ export interface WebHarness {
close(): Promise<void>
}
/** Options for {@link launchWebHarness}. */
/** Options for {@link launchWebScaffold}. */
export interface LaunchOptions {
/**
* Replay fixture (session.jsonl) served by dsh-llm-replay in replay/refresh
@@ -114,9 +114,9 @@ export interface LaunchOptions {
/**
* Boot the real web assembly under the current snapshot mode.
* @param options - replay fixture selection and pacing.
* @returns the running harness.
* @returns the running scaffold.
*/
export async function launchWebHarness(options: LaunchOptions = {}): Promise<WebHarness> {
export async function launchWebScaffold(options: LaunchOptions = {}): Promise<WebScaffold> {
requireDist()
const mode = webSnapshotMode()
if (mode === 'record') {
@@ -221,7 +221,7 @@ export async function launchWebHarness(options: LaunchOptions = {}): Promise<Web
await runningHost.dispose().catch((e: unknown) => failures.push(e))
await rm(workspaceCwd, { recursive: true, force: true }).catch((e: unknown) => failures.push(e))
await rm(persistenceRoot, { recursive: true, force: true }).catch((e: unknown) => failures.push(e))
if (failures.length > 0) throw new AggregateError(failures, 'web harness teardown failed')
if (failures.length > 0) throw new AggregateError(failures, 'web scaffold teardown failed')
},
}
}
@@ -246,16 +246,16 @@ function rawSessionLog(session: Session): string {
* work), tokenize the run-local session id and cwd ({{sessionId}}/{{cwd}},
* the committed ACP fixture convention — re-records then diff only on real
* content), and write the committed fixture.
* @param harness - the record-mode harness.
* @param scaffold - the record-mode scaffold.
* @param sessionId - the driven session.
* @param fixturePath - the committed session.jsonl / seed.jsonl target.
*/
export async function recordFixture(harness: WebHarness, sessionId: SessionId, fixturePath: string): Promise<void> {
const agent = harness.host.ctx.agents.get(sessionId)
export async function recordFixture(scaffold: WebScaffold, sessionId: SessionId, fixturePath: string): Promise<void> {
const agent = scaffold.host.ctx.agents.get(sessionId)
if (agent === undefined) throw new Error(`record harvest: no live agent for ${sessionId}`)
const tokenized = scrubRequestHeaders(rawSessionLog(agent.session))
.split(sessionId).join('{{sessionId}}')
.split(harness.workspaceCwd).join('{{cwd}}')
.split(scaffold.workspaceCwd).join('{{cwd}}')
await writeFile(fixturePath, tokenized)
}
@@ -274,27 +274,27 @@ export function fixtureUserPrompts(fixtureText: string): string[] {
}
/**
* Seed a recorded session fixture into the harness's persistence root through
* Seed a recorded session fixture into the scaffold's persistence root through
* the REAL backend API (throwaway Context + SessionStore + JSONL plugin — the
* semantic-checkpoint precedent), never raw file writes: no knowledge of
* bucket hashing, filename encoding, or compression, and malformed shapes
* fail loud at seed time. The fixture's recorded cwd is rewritten to the
* harness workspace so header/path identity and event payload paths agree.
* @param harness - the target harness.
* scaffold workspace so header/path identity and event payload paths agree.
* @param scaffold - the target scaffold.
* @param fixtureText - raw recorded session.jsonl contents.
* @param id - the seeded session id (stable for deterministic goldens).
* @returns the seeded id.
*/
export async function seedSession(harness: WebHarness, fixtureText: string, id: string): Promise<SessionId> {
export async function seedSession(scaffold: WebScaffold, fixtureText: string, id: string): Promise<SessionId> {
// Committed fixtures tokenize run-local identity ({{sessionId}}/{{cwd}},
// written by recordFixture); realize both for this world before parsing.
const realized = fixtureText
.split('{{sessionId}}').join(id)
.split('{{cwd}}').join(harness.workspaceCwd)
.split('{{cwd}}').join(scaffold.workspaceCwd)
const fixtureCwd = (JSON.parse(realized.split('\n', 1)[0]!) as { cwd?: string }).cwd
const rewritten = fixtureCwd === undefined
? realized
: realized.split(fixtureCwd).join(harness.workspaceCwd)
: realized.split(fixtureCwd).join(scaffold.workspaceCwd)
const events = parseSessionLog(rewritten)
if (events.length === 0) throw new Error('seed fixture has no events')
const last = events[events.length - 1]!
@@ -305,7 +305,7 @@ export async function seedSession(harness: WebHarness, fixtureText: string, id:
version: SESSION_FORMAT_VERSION,
id: SessionId(id),
createdAt: Date.now() - 60_000,
cwd: harness.workspaceCwd,
cwd: scaffold.workspaceCwd,
delegationDepth: 0,
}
const ctx = new Context()
@@ -313,7 +313,7 @@ export async function seedSession(harness: WebHarness, fixtureText: string, id:
await ctx.plugin(SessionStore)
// Same root as the host with the plugin's own default compression, so the
// host's directory-scan list() sees one consistent encoding.
await ctx.plugin(SessionPersistenceJsonl, { root: harness.persistenceRoot })
await ctx.plugin(SessionPersistenceJsonl, { root: scaffold.persistenceRoot })
await ctx.sessionPersistence.create(meta)
await ctx.sessionPersistence.append(meta.id, events)
// Deterministic sidebar order: cold summaries take updatedAt from mtime.

45
apps/web/tests/seeded-history.e2e.ts
View File

@@ -15,8 +15,8 @@ import { afterAll, beforeAll, describe, expect, it, onTestFailed } from 'vitest'
import { join } from 'node:path'
import {
assertFixtureInventory, captureStableAria, compareOrRefreshGolden, fixtureUserPrompts,
launchWebHarness, recordFixture, seedSession, watchConsole, webSnapshotMode, type WebHarness,
} from './harness.ts'
launchWebScaffold, recordFixture, seedSession, watchConsole, webSnapshotMode, type WebScaffold,
} from './scaffold.ts'
import { saveFailureShot } from './support.ts'
const SNAPSHOT_DIR = fileURLToPath(new URL('./snapshots/seeded-history', import.meta.url))
@@ -28,44 +28,44 @@ const SEED_ID = 'seeded-history-web-e2e'
const PROMPT = 'Use the read tool twice in one assistant message: read a.txt and b.txt. Then reply with the single word DONE and stop.'
describe('web e2e: seeded history renders through cold resume', () => {
let harness: WebHarness
let scaffold: WebScaffold
let browser: Browser
let page: Page
let tripwire: ReturnType<typeof watchConsole>
beforeAll(async () => {
harness = await launchWebHarness({})
scaffold = await launchWebScaffold({})
// The read-tool targets exist in both modes: record needs them for the
// live turn; replay's seeded log carries their recorded contents but the
// workspace stays consistent for any user poking the harness.
await writeFile(join(harness.workspaceCwd, 'a.txt'), 'alpha\n')
await writeFile(join(harness.workspaceCwd, 'b.txt'), 'beta\n')
// workspace stays consistent for any user poking the scaffold.
await writeFile(join(scaffold.workspaceCwd, 'a.txt'), 'alpha\n')
await writeFile(join(scaffold.workspaceCwd, 'b.txt'), 'beta\n')
if (MODE !== 'record') {
const raw = await readFile(SEED, 'utf8')
expect(fixtureUserPrompts(raw), 'seed fixture must carry exactly the drive prompt').toEqual([PROMPT])
await seedSession(harness, raw, SEED_ID)
await seedSession(scaffold, raw, SEED_ID)
}
browser = await chromium.launch()
page = await browser.newPage({ viewport: { width: 1680, height: 1000 } })
tripwire = watchConsole(page)
await page.goto(harness.baseUrl, { waitUntil: 'load' })
await page.goto(scaffold.baseUrl, { waitUntil: 'load' })
await page.waitForSelector('[class*="frame"]', { timeout: 30_000 })
}, 120_000)
afterAll(async () => {
await browser?.close()
await harness?.close()
await scaffold?.close()
})
it.skipIf(MODE !== 'record')('records the seed turn live through the composer', async () => {
onTestFailed(() => saveFailureShot(page, 'web-e2e-seeded-record'))
const input = page.locator('textarea').first()
await input.waitFor({ timeout: 10_000 })
const settled = harness.whenTurnSettled()
const settled = scaffold.whenTurnSettled()
await input.fill(PROMPT)
await input.press('Enter')
const sessionId = await settled
await recordFixture(harness, sessionId, SEED)
await recordFixture(scaffold, sessionId, SEED)
}, 200_000)
it.skipIf(MODE === 'record')('lists the seeded session cold and renders its history from the log', async () => {
@@ -89,17 +89,34 @@ describe('web e2e: seeded history renders through cold resume', () => {
it.skipIf(MODE === 'record')('matches the historical conversation aria golden', async () => {
onTestFailed(() => saveFailureShot(page, 'web-e2e-seeded-aria'))
const snapshot = (await captureStableAria(page, '[class*="centerCol"]', harness.workspaceCwd))
const snapshot = (await captureStableAria(page, '[class*="centerCol"]', scaffold.workspaceCwd))
.split(SEED_ID).join('{{seededId}}')
await compareOrRefreshGolden(UI_EXPECTED, snapshot, MODE)
})
it.skipIf(MODE === 'record')('expands and collapses a tool row rebuilt from the cold log', async () => {
onTestFailed(() => saveFailureShot(page, 'web-e2e-seeded-toolrow'))
// Interaction over cold-resumed history: read rows are expand-in-place
// rows (rowExpands routes the click to toggleExpand, not openDetails), so
// the gesture under test is the inline fold over log-rebuilt content.
// Runs after the golden capture; still zero model calls.
const row = page.locator('[data-variant] [data-clickable][role="button"]').first()
await row.waitFor({ timeout: 10_000 })
expect(await row.getAttribute('aria-expanded')).toBe('false')
await row.click()
await expect.poll(() => row.getAttribute('aria-expanded'), { timeout: 5_000 }).toBe('true')
// The expanded body renders the recorded tool result (a.txt's contents).
await expect.poll(() => page.getByText('alpha', { exact: false }).count(), { timeout: 5_000 }).toBeGreaterThan(0)
await row.click()
await expect.poll(() => row.getAttribute('aria-expanded'), { timeout: 5_000 }).toBe('false')
})
it.skipIf(MODE === 'record')('issued zero model calls and stayed clean', async () => {
// No replay fixture was installed and the llm seam is open — any stray
// stream would have failed the turn loudly. Cleanliness pins the wire.
expect(tripwire.pageErrors).toEqual([])
expect(tripwire.warnings).toEqual([])
expect(harness.serverErrors).toEqual([])
expect(scaffold.serverErrors).toEqual([])
await assertFixtureInventory(SNAPSHOT_DIR, ['seed.jsonl', 'ui.expected.md'])
})
})

4
apps/web/tsconfig.json
View File

@@ -17,12 +17,12 @@
"src",
"tests"
],
// The web e2e lane (harness + replay specs) boots the host spine and reads
// The web e2e lane (scaffold + replay specs) boots the host spine and reads
// its Context merges — host-plane programs, checked in tsconfig.host.json;
// this client-registered project must not also hold them (one program
// cannot see both sides of the cordis Context merges).
"exclude": [
"tests/harness.ts",
"tests/scaffold.ts",
"tests/replay-round-trip.e2e.ts",
"tests/seeded-history.e2e.ts"
],