|
|
|
|
@@ -1,15 +1,7 @@
|
|
|
|
|
// Web e2e scenario: mid-turn steering over the host wire. The Web UI has no
|
|
|
|
|
// steer entry, so the steer is POSTed from the page over the same
|
|
|
|
|
// same-origin /api transport the client uses. Everything downstream is
|
|
|
|
|
// product: the gateway routes mode:'steer' to Agent.steer, the loop drains
|
|
|
|
|
// it at the step boundary into a durable steering/message event, the SSE mux
|
|
|
|
|
// pushes it, and the transcript shows the text as a plain bubble (no
|
|
|
|
|
// interjection chrome). The question composer supplies the deterministic
|
|
|
|
|
// mid-turn window: while ask_user_question blocks, the turn is provably
|
|
|
|
|
// running, so record and replay perform the identical steer-then-answer
|
|
|
|
|
// sequence with zero timing dependence — and the recorded final reply proves
|
|
|
|
|
// the steer reached the MODEL (it obeys an instruction that only the
|
|
|
|
|
// steering message carries).
|
|
|
|
|
// Web e2e scenarios for both steering entry points: QueueDock strictly
|
|
|
|
|
// transfers one queued occurrence, while the complementary composer gestures
|
|
|
|
|
// choose Queue or Steer. The question tool supplies a deterministic pending-
|
|
|
|
|
// steering snapshot before the step can drain.
|
|
|
|
|
import { readFile } from 'node:fs/promises'
|
|
|
|
|
import { fileURLToPath } from 'node:url'
|
|
|
|
|
import { join } from 'node:path'
|
|
|
|
|
@@ -26,16 +18,18 @@ import { connectFreshWorkspace, newEnglishPage, saveFailureShot } from './suppor
|
|
|
|
|
|
|
|
|
|
const SNAPSHOT_DIR = fileURLToPath(new URL('./snapshots/steering', import.meta.url))
|
|
|
|
|
const FIXTURE = join(SNAPSHOT_DIR, 'session.jsonl')
|
|
|
|
|
// Two goldens for the two distinct states this interaction produces: the
|
|
|
|
|
// mid-turn moment (steer ACCEPTED but deliberately invisible — the loop
|
|
|
|
|
// drains steering at the step boundary, so no steering text exists while
|
|
|
|
|
// the question still blocks the step) and the settled transcript (plain
|
|
|
|
|
// bubble in place, final reply obeying it). The pair pins the timing
|
|
|
|
|
// semantics visually: if the client ever starts rendering pending steers
|
|
|
|
|
// eagerly, the mid-steer golden flips first.
|
|
|
|
|
// Two goldens pin the transient Host projection and its durable handoff: the
|
|
|
|
|
// mid-turn state renders accepted steering from session/queue while the
|
|
|
|
|
// question blocks admission, then the settled state renders the same message
|
|
|
|
|
// from steering/message beside the reply that obeys it.
|
|
|
|
|
const MID_EXPECTED = join(SNAPSHOT_DIR, 'mid-steer.expected.md')
|
|
|
|
|
const SETTLED_EXPECTED = join(SNAPSHOT_DIR, 'settled.expected.md')
|
|
|
|
|
const MODE = webSnapshotMode()
|
|
|
|
|
// The question composer replaces the textarea, so fill → Queue row → Steer
|
|
|
|
|
// must finish inside the first replay chunk window. At 15 ms that window is
|
|
|
|
|
// shorter than Playwright's round trips; 100 ms supplies test-only headroom,
|
|
|
|
|
// while larger values lengthen all three replay scenarios linearly.
|
|
|
|
|
const REPLAY_PACE_MS = 100
|
|
|
|
|
|
|
|
|
|
const PROMPT = 'Use the ask_user_question tool to ask me exactly one question with id "checkpoint", question "Ready to continue?", header "Checkpoint", and options labeled "Yes" and "No". After I answer, reply with one short sentence acknowledging my answer and stop.'
|
|
|
|
|
const STEER = 'Interjection: include the word BANANA in your final reply.'
|
|
|
|
|
@@ -56,15 +50,13 @@ describe('web e2e: mid-turn steering lands durably and visibly', () => {
|
|
|
|
|
let browser: Browser
|
|
|
|
|
let page: Page
|
|
|
|
|
let tripwire: ReturnType<typeof watchConsole>
|
|
|
|
|
let liveSessionId: string | undefined
|
|
|
|
|
const sessionEvents: SessionEvent[] = []
|
|
|
|
|
|
|
|
|
|
beforeAll(async () => {
|
|
|
|
|
scaffold = await launchWebScaffold(MODE === 'record' ? {} : { replayFixture: FIXTURE, paceMs: 15 })
|
|
|
|
|
scaffold.ctx.on('session/event', (session, event) => {
|
|
|
|
|
liveSessionId ??= session.id
|
|
|
|
|
sessionEvents.push(event)
|
|
|
|
|
})
|
|
|
|
|
scaffold = await launchWebScaffold(MODE === 'record'
|
|
|
|
|
? {}
|
|
|
|
|
: { replayFixture: FIXTURE, paceMs: REPLAY_PACE_MS })
|
|
|
|
|
scaffold.ctx.on('session/event', (_session, event) => { sessionEvents.push(event) })
|
|
|
|
|
browser = await chromium.launch()
|
|
|
|
|
page = await newEnglishPage(browser)
|
|
|
|
|
tripwire = watchConsole(page)
|
|
|
|
|
@@ -79,7 +71,7 @@ describe('web e2e: mid-turn steering lands durably and visibly', () => {
|
|
|
|
|
await scaffold?.close()
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
it('steers during the blocked step; the message is logged, rendered, and obeyed', async () => {
|
|
|
|
|
it('strictly steers one queued row; the interjection is logged, rendered, and obeyed', async () => {
|
|
|
|
|
onTestFailed(() => saveFailureShot(page, 'web-e2e-steering'))
|
|
|
|
|
if (MODE !== 'record') {
|
|
|
|
|
// The steer must NOT be a user/message — it lands as steering/message.
|
|
|
|
|
@@ -91,36 +83,29 @@ describe('web e2e: mid-turn steering lands durably and visibly', () => {
|
|
|
|
|
await input.fill(PROMPT)
|
|
|
|
|
await input.press('Enter')
|
|
|
|
|
|
|
|
|
|
// The blocked composer is the mid-turn barrier: its presence proves the
|
|
|
|
|
// ask_user_question step is executing, i.e. the turn is running NOW.
|
|
|
|
|
// Enter remains the Queue gesture. The row action then atomically moves
|
|
|
|
|
// this exact occurrence into the current turn's steering outbox.
|
|
|
|
|
await input.fill(STEER)
|
|
|
|
|
await input.press('Enter')
|
|
|
|
|
const queued = page.getByText(STEER, { exact: true })
|
|
|
|
|
await queued.waitFor({ timeout: 10_000 })
|
|
|
|
|
const queuedRow = page.getByRole('listitem').filter({ hasText: STEER })
|
|
|
|
|
const steerButton = queuedRow.getByRole('button', { name: 'Steer queued message' })
|
|
|
|
|
await expect.poll(() => steerButton.isEnabled(), { timeout: 10_000 }).toBe(true)
|
|
|
|
|
await steerButton.click({ timeout: 10_000 })
|
|
|
|
|
const pendingSteering = page.locator('[data-pending-steering]').filter({ hasText: STEER })
|
|
|
|
|
// A timeout while the Queue row remains means strict steer lost to a
|
|
|
|
|
// closing window (`steer-unavailable`); inspect replay pacing first.
|
|
|
|
|
await pendingSteering.waitFor({ timeout: 10_000 })
|
|
|
|
|
|
|
|
|
|
// The blocked composer keeps steering pending long enough to observe the
|
|
|
|
|
// Host-authoritative mirror before the loop admits it durably.
|
|
|
|
|
const composer = page.locator('[data-question-key]')
|
|
|
|
|
await composer.waitFor({ timeout: MODE === 'record' ? 120_000 : 30_000 })
|
|
|
|
|
|
|
|
|
|
// Steer through the real wire from the page (same envelope + endpoint the
|
|
|
|
|
// web client's session.prompt uses). accepted:true is the transport proof.
|
|
|
|
|
expect(liveSessionId).toBeDefined()
|
|
|
|
|
const reply = await page.evaluate(async ({ sessionId, text }) => {
|
|
|
|
|
const response = await fetch('/api/session.prompt', {
|
|
|
|
|
method: 'POST',
|
|
|
|
|
headers: { 'content-type': 'application/json' },
|
|
|
|
|
body: JSON.stringify({
|
|
|
|
|
type: 'client-request',
|
|
|
|
|
rpcId: crypto.randomUUID(),
|
|
|
|
|
method: 'session.prompt',
|
|
|
|
|
payload: { sessionId, mode: 'steer', content: [{ type: 'text', text }] },
|
|
|
|
|
}),
|
|
|
|
|
})
|
|
|
|
|
return await response.json() as { result?: { ok?: boolean } }
|
|
|
|
|
}, { sessionId: liveSessionId!, text: STEER })
|
|
|
|
|
expect(reply.result?.ok).toBe(true)
|
|
|
|
|
|
|
|
|
|
if (MODE !== 'record') {
|
|
|
|
|
// Mid-turn golden: the ACCEPTED steer is durable in the inbox but the
|
|
|
|
|
// loop drains steering only at the step boundary, so no steering/message
|
|
|
|
|
// exists yet and no steer text renders — the composer still blocks,
|
|
|
|
|
// alone. The DOM is stable here (no further SSE frames can arrive until
|
|
|
|
|
// the question is answered), making this state capturable.
|
|
|
|
|
expect(await page.getByText(STEER, { exact: true }).count()).toBe(0)
|
|
|
|
|
expect(await page.getByText(STEER, { exact: true }).count()).toBe(1)
|
|
|
|
|
expect(await pendingSteering.count()).toBe(1)
|
|
|
|
|
expect(await page.getByRole('button', { name: 'Edit queued message' }).count()).toBe(0)
|
|
|
|
|
const snapshot = await captureStableAria(page, '[class*="centerCol"]', scaffold.workspaceCwd)
|
|
|
|
|
await compareOrRefreshGolden(MID_EXPECTED, snapshot, MODE)
|
|
|
|
|
@@ -156,6 +141,7 @@ describe('web e2e: mid-turn steering lands durably and visibly', () => {
|
|
|
|
|
// Visible: the plain steering bubble plus the reply that obeys it
|
|
|
|
|
// (steer text + final reply each contain the marker word).
|
|
|
|
|
await expect.poll(() => page.getByText(STEER, { exact: true }).count(), { timeout: 15_000 }).toBe(1)
|
|
|
|
|
expect(await pendingSteering.count()).toBe(0)
|
|
|
|
|
await expect.poll(() => page.getByText('BANANA', { exact: false }).count(), { timeout: 10_000 }).toBeGreaterThanOrEqual(2)
|
|
|
|
|
expect(await page.locator('[data-question-key]').count()).toBe(0)
|
|
|
|
|
// Settled golden: steer text between the question round trip and the
|
|
|
|
|
@@ -170,3 +156,120 @@ describe('web e2e: mid-turn steering lands durably and visibly', () => {
|
|
|
|
|
await assertFixtureInventory(SNAPSHOT_DIR, ['session.jsonl', 'mid-steer.expected.md', 'settled.expected.md'])
|
|
|
|
|
})
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
describe('web e2e: composer shortcut steers directly', () => {
|
|
|
|
|
let scaffold: WebScaffold
|
|
|
|
|
let browser: Browser
|
|
|
|
|
let page: Page
|
|
|
|
|
let tripwire: ReturnType<typeof watchConsole>
|
|
|
|
|
const sessionEvents: SessionEvent[] = []
|
|
|
|
|
|
|
|
|
|
beforeAll(async () => {
|
|
|
|
|
scaffold = await launchWebScaffold({ replayFixture: FIXTURE, paceMs: REPLAY_PACE_MS })
|
|
|
|
|
scaffold.ctx.on('session/event', (_session, event) => { sessionEvents.push(event) })
|
|
|
|
|
browser = await chromium.launch()
|
|
|
|
|
page = await newEnglishPage(browser)
|
|
|
|
|
tripwire = watchConsole(page)
|
|
|
|
|
await page.goto(scaffold.baseUrl, { waitUntil: 'load' })
|
|
|
|
|
await page.waitForSelector('[class*="frame"]', { timeout: 30_000 })
|
|
|
|
|
await connectFreshWorkspace(page, scaffold.workspaceCwd)
|
|
|
|
|
}, 120_000)
|
|
|
|
|
|
|
|
|
|
afterAll(async () => {
|
|
|
|
|
await browser?.close()
|
|
|
|
|
await scaffold?.close()
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
it.skipIf(MODE === 'record')('uses Cmd+Enter without creating a Queue row', async () => {
|
|
|
|
|
onTestFailed(() => saveFailureShot(page, 'web-e2e-composer-steering'))
|
|
|
|
|
expect(fixtureUserPrompts(await readFile(FIXTURE, 'utf8'))).toEqual([PROMPT])
|
|
|
|
|
const input = page.locator('textarea').first()
|
|
|
|
|
await input.waitFor({ timeout: 10_000 })
|
|
|
|
|
const settled = scaffold.whenTurnSettled(30_000)
|
|
|
|
|
await input.fill(PROMPT)
|
|
|
|
|
await input.press('Enter')
|
|
|
|
|
await page.getByRole('button', { name: 'Stop generating' }).waitFor({ timeout: 10_000 })
|
|
|
|
|
|
|
|
|
|
await input.fill(STEER)
|
|
|
|
|
await input.press('Meta+Enter')
|
|
|
|
|
await expect.poll(() => input.inputValue(), { timeout: 5_000 }).toBe('')
|
|
|
|
|
expect(await page.locator('[data-queue-dock]').count()).toBe(0)
|
|
|
|
|
|
|
|
|
|
const composer = page.locator('[data-question-key]')
|
|
|
|
|
await composer.waitFor({ timeout: 30_000 })
|
|
|
|
|
const pendingSteering = page.locator('[data-pending-steering]').filter({ hasText: STEER })
|
|
|
|
|
await pendingSteering.waitFor({ timeout: 10_000 })
|
|
|
|
|
await composer.getByRole('radio', { name: 'Yes' }).click()
|
|
|
|
|
await composer.getByRole('radio', { name: 'Yes' }).press('Enter')
|
|
|
|
|
await settled
|
|
|
|
|
|
|
|
|
|
const steerEvents = sessionEvents.filter(event => event.type === 'steering/message')
|
|
|
|
|
expect(steerEvents).toHaveLength(1)
|
|
|
|
|
expect((steerEvents[0] as SessionEvent & { data: { turn: number } }).data.turn).toBe(1)
|
|
|
|
|
await expect.poll(() => page.getByText(STEER, { exact: true }).count(), { timeout: 15_000 }).toBe(1)
|
|
|
|
|
expect(await pendingSteering.count()).toBe(0)
|
|
|
|
|
await expect.poll(() => page.getByText('BANANA', { exact: false }).count(), { timeout: 10_000 })
|
|
|
|
|
.toBeGreaterThanOrEqual(2)
|
|
|
|
|
expect(tripwire.pageErrors).toEqual([])
|
|
|
|
|
expect(tripwire.warnings).toEqual([])
|
|
|
|
|
}, 90_000)
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
describe('web e2e: composer shortcut follows the swapped busy behavior', () => {
|
|
|
|
|
let scaffold: WebScaffold
|
|
|
|
|
let browser: Browser
|
|
|
|
|
let page: Page
|
|
|
|
|
let tripwire: ReturnType<typeof watchConsole>
|
|
|
|
|
const sessionEvents: SessionEvent[] = []
|
|
|
|
|
|
|
|
|
|
beforeAll(async () => {
|
|
|
|
|
scaffold = await launchWebScaffold({ replayFixture: FIXTURE, paceMs: REPLAY_PACE_MS })
|
|
|
|
|
scaffold.ctx.on('session/event', (_session, event) => { sessionEvents.push(event) })
|
|
|
|
|
browser = await chromium.launch()
|
|
|
|
|
page = await newEnglishPage(browser)
|
|
|
|
|
tripwire = watchConsole(page)
|
|
|
|
|
await page.goto(scaffold.baseUrl, { waitUntil: 'load' })
|
|
|
|
|
await page.waitForSelector('[class*="frame"]', { timeout: 30_000 })
|
|
|
|
|
await connectFreshWorkspace(page, scaffold.workspaceCwd)
|
|
|
|
|
}, 120_000)
|
|
|
|
|
|
|
|
|
|
afterAll(async () => {
|
|
|
|
|
await browser?.close()
|
|
|
|
|
await scaffold?.close()
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
it.skipIf(MODE === 'record')('queues Cmd+Enter when plain Enter is configured to Steer', async () => {
|
|
|
|
|
onTestFailed(() => saveFailureShot(page, 'web-e2e-composer-swapped-shortcut'))
|
|
|
|
|
await page.getByRole('button', { name: 'Settings', exact: true }).click()
|
|
|
|
|
const dialog = page.getByRole('dialog', { name: 'Settings' })
|
|
|
|
|
await dialog.getByRole('button', { name: 'Queue' }).click()
|
|
|
|
|
await page.getByRole('menuitem', { name: 'Steer' }).click()
|
|
|
|
|
await dialog.getByRole('button', { name: 'Steer' }).waitFor({ timeout: 10_000 })
|
|
|
|
|
await page.keyboard.press('Escape')
|
|
|
|
|
|
|
|
|
|
const input = page.locator('textarea').first()
|
|
|
|
|
const settled = scaffold.whenTurnSettled(30_000)
|
|
|
|
|
await input.fill(PROMPT)
|
|
|
|
|
await input.press('Enter')
|
|
|
|
|
await page.getByRole('button', { name: 'Stop generating' }).waitFor({ timeout: 10_000 })
|
|
|
|
|
|
|
|
|
|
const queuedText = 'Queued by the complementary Cmd+Enter shortcut.'
|
|
|
|
|
await input.fill(queuedText)
|
|
|
|
|
await input.press('Meta+Enter')
|
|
|
|
|
const queuedRow = page.locator('[data-queue-dock]').getByRole('listitem').filter({ hasText: queuedText })
|
|
|
|
|
await queuedRow.getByText(queuedText, { exact: true }).waitFor({ timeout: 10_000 })
|
|
|
|
|
expect(await page.locator('[data-pending-steering]').filter({ hasText: queuedText }).count()).toBe(0)
|
|
|
|
|
expect(sessionEvents.filter(event => event.type === 'steering/message')).toHaveLength(0)
|
|
|
|
|
|
|
|
|
|
// Remove the asserted Queue row, then finish the recorded question turn
|
|
|
|
|
// so replay teardown still proves that every fixture call was consumed.
|
|
|
|
|
await queuedRow.getByRole('button', { name: 'Remove queued message' }).click()
|
|
|
|
|
const composer = page.locator('[data-question-key]')
|
|
|
|
|
await composer.waitFor({ timeout: 30_000 })
|
|
|
|
|
await composer.getByRole('radio', { name: 'Yes' }).click()
|
|
|
|
|
await composer.getByRole('radio', { name: 'Yes' }).press('Enter')
|
|
|
|
|
await settled
|
|
|
|
|
expect(tripwire.pageErrors).toEqual([])
|
|
|
|
|
expect(tripwire.warnings).toEqual([])
|
|
|
|
|
}, 90_000)
|
|
|
|
|
})
|
|
|
|
|
|
Huanqi Cao