test(web): make real title smoke deterministic
This commit is contained in:
@@ -77,6 +77,55 @@ async function rpc<T>(baseUrl: string, method: string, payload: unknown): Promis
|
|||||||
return body.result.value
|
return body.result.value
|
||||||
}
|
}
|
||||||
|
|
||||||
|
interface HistoryPage {
|
||||||
|
events: { event: { type: string; data: unknown } }[]
|
||||||
|
hasMore: boolean
|
||||||
|
}
|
||||||
|
|
||||||
|
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||||
|
return typeof value === 'object' && value !== null
|
||||||
|
}
|
||||||
|
|
||||||
|
function providerTitle(page: HistoryPage): string | undefined {
|
||||||
|
for (let index = page.events.length - 1; index >= 0; index--) {
|
||||||
|
const event = page.events[index]!.event
|
||||||
|
if (event.type !== 'session/title' || !isRecord(event.data)) continue
|
||||||
|
const source = event.data.source
|
||||||
|
if (typeof event.data.title === 'string' && isRecord(source) && source.kind === 'provider') {
|
||||||
|
return event.data.title
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return undefined
|
||||||
|
}
|
||||||
|
|
||||||
|
function hasAssistantMarker(page: HistoryPage, marker: string): boolean {
|
||||||
|
return page.events.some(({ event }) => {
|
||||||
|
if (event.type !== 'assistant/message' || !isRecord(event.data) || !Array.isArray(event.data.content)) return false
|
||||||
|
return event.data.content.some(block =>
|
||||||
|
isRecord(block) && block.type === 'text' && typeof block.text === 'string' && block.text.includes(marker))
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
async function history(baseUrl: string, sessionId: string): Promise<HistoryPage> {
|
||||||
|
return rpc<HistoryPage>(baseUrl, 'session.history', { sessionId, maxMessages: 10 })
|
||||||
|
}
|
||||||
|
|
||||||
|
async function waitForProviderTitle(baseUrl: string, sessionId: string): Promise<string> {
|
||||||
|
let observed: string | undefined
|
||||||
|
await expect.poll(async () => {
|
||||||
|
observed = providerTitle(await history(baseUrl, sessionId))
|
||||||
|
return observed
|
||||||
|
}, { timeout: 90_000 }).toEqual(expect.any(String))
|
||||||
|
if (observed === undefined) throw new Error('provider-backed session title was not observed')
|
||||||
|
return observed
|
||||||
|
}
|
||||||
|
|
||||||
|
async function waitForAssistantMarker(baseUrl: string, sessionId: string, marker: string): Promise<void> {
|
||||||
|
await expect.poll(async () => hasAssistantMarker(await history(baseUrl, sessionId), marker), {
|
||||||
|
timeout: 120_000,
|
||||||
|
}).toBe(true)
|
||||||
|
}
|
||||||
|
|
||||||
/** W5 screenshot: evidence for the figma comparison, not a failure artifact. */
|
/** W5 screenshot: evidence for the figma comparison, not a failure artifact. */
|
||||||
async function screen(page: Page, name: string): Promise<void> {
|
async function screen(page: Page, name: string): Promise<void> {
|
||||||
await page.screenshot({ path: join(REPO_ROOT, '.artifacts', `w5-${name}.png`) })
|
await page.screenshot({ path: join(REPO_ROOT, '.artifacts', `w5-${name}.png`) })
|
||||||
@@ -282,7 +331,6 @@ describe.skipIf(!process.env.DEEPSEEK_API_KEY || notReady.length > 0)('web smoke
|
|||||||
await input.waitFor({ timeout: 10_000 })
|
await input.waitFor({ timeout: 10_000 })
|
||||||
await screen(page, '02-empty-state')
|
await screen(page, '02-empty-state')
|
||||||
const prompt = `Please answer this request carefully: explain event sourcing in two sentences, ending with exactly ${ROUND_DONE_MARKER}.`
|
const prompt = `Please answer this request carefully: explain event sourcing in two sentences, ending with exactly ${ROUND_DONE_MARKER}.`
|
||||||
const fallbackTitle = 'Please answer this request carefully:'
|
|
||||||
await input.fill(prompt)
|
await input.fill(prompt)
|
||||||
await input.press('Enter')
|
await input.press('Enter')
|
||||||
// startSession chain: session mounts, composer moves to the bottom.
|
// startSession chain: session mounts, composer moves to the bottom.
|
||||||
@@ -296,14 +344,18 @@ describe.skipIf(!process.env.DEEPSEEK_API_KEY || notReady.length > 0)('web smoke
|
|||||||
undefined,
|
undefined,
|
||||||
{ timeout: 15_000 },
|
{ timeout: 15_000 },
|
||||||
)
|
)
|
||||||
|
await expect.poll(async () => (await rpc<{ items: { sessionId: string }[] }>(baseUrl, 'session.list', {})).items.length, {
|
||||||
|
timeout: 15_000,
|
||||||
|
}).toBe(1)
|
||||||
|
const sessions = await rpc<{ items: { sessionId: string }[] }>(baseUrl, 'session.list', {})
|
||||||
|
const sessionId = sessions.items[0]?.sessionId
|
||||||
|
if (sessionId === undefined) throw new Error('created Web session was not listed')
|
||||||
|
const durableTitle = await waitForProviderTitle(baseUrl, sessionId)
|
||||||
await page.waitForFunction(
|
await page.waitForFunction(
|
||||||
expected => document.title !== `${expected} — DeepSeek Harness`
|
expected => document.title === `${expected} — DeepSeek Harness`,
|
||||||
&& document.title.endsWith(' — DeepSeek Harness'),
|
durableTitle,
|
||||||
fallbackTitle,
|
{ timeout: 15_000 },
|
||||||
{ timeout: 90_000 },
|
|
||||||
)
|
)
|
||||||
const durableTitle = (await page.title()).replace(/ — DeepSeek Harness$/, '')
|
|
||||||
expect(durableTitle).not.toBe(fallbackTitle)
|
|
||||||
const sessionTree = page.getByRole('tree', { name: 'Sessions' })
|
const sessionTree = page.getByRole('tree', { name: 'Sessions' })
|
||||||
const projectRow = sessionTree.getByRole('treeitem').first()
|
const projectRow = sessionTree.getByRole('treeitem').first()
|
||||||
if (await projectRow.getAttribute('aria-expanded') === 'false') await projectRow.click()
|
if (await projectRow.getAttribute('aria-expanded') === 'false') await projectRow.click()
|
||||||
@@ -311,7 +363,8 @@ describe.skipIf(!process.env.DEEPSEEK_API_KEY || notReady.length > 0)('web smoke
|
|||||||
sessionTree.getByText(durableTitle, { exact: true }).waitFor({ timeout: 10_000 }),
|
sessionTree.getByText(durableTitle, { exact: true }).waitFor({ timeout: 10_000 }),
|
||||||
page.getByRole('navigation').getByText(durableTitle, { exact: true }).waitFor({ timeout: 10_000 }),
|
page.getByRole('navigation').getByText(durableTitle, { exact: true }).waitFor({ timeout: 10_000 }),
|
||||||
])
|
])
|
||||||
await page.waitForFunction(marker => document.body.innerText.includes(marker), ROUND_DONE_MARKER, { timeout: 120_000 })
|
await waitForAssistantMarker(baseUrl, sessionId, ROUND_DONE_MARKER)
|
||||||
|
await page.locator('p').filter({ hasText: ROUND_DONE_MARKER }).waitFor({ timeout: 10_000 })
|
||||||
await screen(page, '04-round-complete')
|
await screen(page, '04-round-complete')
|
||||||
}, 150_000)
|
}, 150_000)
|
||||||
|
|
||||||
@@ -386,7 +439,7 @@ describe.skipIf(!process.env.DEEPSEEK_API_KEY || notReady.length > 0)('web smoke
|
|||||||
onTestFailed(() => saveFailureShot(page, 'w5-reload'))
|
onTestFailed(() => saveFailureShot(page, 'w5-reload'))
|
||||||
await page.reload({ waitUntil: 'load' })
|
await page.reload({ waitUntil: 'load' })
|
||||||
await page.waitForSelector('[class*="frame"]', { timeout: 30_000 })
|
await page.waitForSelector('[class*="frame"]', { timeout: 30_000 })
|
||||||
await page.waitForFunction(marker => document.body.innerText.includes(marker), ROUND_DONE_MARKER, { timeout: 30_000 })
|
await page.locator('p').filter({ hasText: ROUND_DONE_MARKER }).waitFor({ timeout: 30_000 })
|
||||||
await screen(page, '12-reload-recovery')
|
await screen(page, '12-reload-recovery')
|
||||||
})
|
})
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user