diff --git a/examples/desktop/docs/qa-evals-merge/00-sidebar-full.png b/examples/desktop/docs/qa-evals-merge/00-sidebar-full.png new file mode 100644 index 0000000000..303cf33868 Binary files /dev/null and b/examples/desktop/docs/qa-evals-merge/00-sidebar-full.png differ diff --git a/examples/desktop/docs/qa-evals-merge/01-evals-rubrics.png b/examples/desktop/docs/qa-evals-merge/01-evals-rubrics.png new file mode 100644 index 0000000000..fe0c900ccb Binary files /dev/null and b/examples/desktop/docs/qa-evals-merge/01-evals-rubrics.png differ diff --git a/examples/desktop/docs/qa-evals-merge/02-evals-growth.png b/examples/desktop/docs/qa-evals-merge/02-evals-growth.png new file mode 100644 index 0000000000..19d8039b4d Binary files /dev/null and b/examples/desktop/docs/qa-evals-merge/02-evals-growth.png differ diff --git a/examples/desktop/docs/qa-evals-merge/03-evals-runtime.png b/examples/desktop/docs/qa-evals-merge/03-evals-runtime.png new file mode 100644 index 0000000000..96b976ef4b Binary files /dev/null and b/examples/desktop/docs/qa-evals-merge/03-evals-runtime.png differ diff --git a/examples/desktop/docs/qa-evals-merge/04-evals-selector-runtime.png b/examples/desktop/docs/qa-evals-merge/04-evals-selector-runtime.png new file mode 100644 index 0000000000..b9011202dd Binary files /dev/null and b/examples/desktop/docs/qa-evals-merge/04-evals-selector-runtime.png differ diff --git a/examples/desktop/scripts/qa-cdp-shoot-evals-merge.mjs b/examples/desktop/scripts/qa-cdp-shoot-evals-merge.mjs new file mode 100644 index 0000000000..26d328a348 --- /dev/null +++ b/examples/desktop/scripts/qa-cdp-shoot-evals-merge.mjs @@ -0,0 +1,274 @@ +// Verification script for lane-evals-merge (2026-07-19): +// launches an isolated Electron on CDP :9455 with its own --user-data-dir +// and DSH_DESKTOP_HOME, opens the shell, clicks the new Evals nav item, +// cycles the three inner tabs [ Rubrics | Growth | Runtime ], flips the +// shared rubric selector, and shoots five screenshots into +// docs/qa-evals-merge/: +// 00-sidebar-full.png — left nav shows Evals (not three rows) +// 01-evals-rubrics.png — Rubrics tab active +// 02-evals-growth.png — Growth tab active +// 03-evals-runtime.png — Runtime tab active (all rubrics) +// 04-evals-selector-runtime.png — shared selector picks one rubric, +// Runtime grid filters to that card +// +// Two-root isolation (per 2026-07-18 postmortem in +// scripts/qa-cdp-shoot-affordance.mjs header): +// 1. --user-data-dir= isolates Chromium userdata +// 2. DSH_DESKTOP_HOME= isolates our shell's config root +// Neither the user's running Electron on CDP 9333 nor their ~/.dsh-desktop +// is touched. + +import { spawn } from 'node:child_process' +import { existsSync, mkdirSync, writeFileSync, rmSync } from 'node:fs' +import { resolve, join } from 'node:path' +import { setTimeout as sleep } from 'node:timers/promises' +import { tmpdir } from 'node:os' + +const WORKTREE = resolve(process.env.DSH_WORKTREE || process.cwd()) +const PARENT = resolve(process.env.DSH_REPO || '/Users/ziya/harness/dsh-desktop-demo') +const ELECTRON = join(PARENT, 'node_modules/.bin/electron') +const CDP_PORT = Number(process.env.DSH_EVALS_MERGE_PORT || 9455) +const USER_DATA = join(tmpdir(), 'dsh-evals-merge-userdata') +const DSH_HOME = join(tmpdir(), 'dsh-evals-merge-home') +const OUTDIR = join(WORKTREE, 'docs/qa-evals-merge') + +if (!existsSync(ELECTRON)) { + console.error(`electron binary not found at ${ELECTRON}`) + process.exit(2) +} +mkdirSync(OUTDIR, { recursive: true }) +for (const dir of [USER_DATA, DSH_HOME]) { + try { rmSync(dir, { recursive: true, force: true }) } catch {} + mkdirSync(dir, { recursive: true }) +} +// Minimal onboarding seed so we don't hit the first-run modal. Same shape +// as scripts/qa-cdp-shoot-affordance.mjs — never touches the real ~/.dsh-desktop. +const seedOverlay = [ + '# QA evals-merge seed overlay (tmp, per-run).', + 'plugins:', + ' - "@cordisjs/plugin-include":', + ` path: ${join(WORKTREE, 'config/daemon-echo.yml')}`, + '', +].join('\n') +writeFileSync(join(DSH_HOME, 'user-overlay.cordis.yml'), seedOverlay) +writeFileSync(join(DSH_HOME, 'config.json'), JSON.stringify({ + role: 'coding', approvalMode: 'never', +})) +writeFileSync(join(DSH_HOME, '.onboarded'), new Date().toISOString()) + +async function bootElectron() { + const child = spawn(ELECTRON, [ + `--remote-debugging-port=${CDP_PORT}`, + `--user-data-dir=${USER_DATA}`, + '--disable-gpu', + '--no-sandbox', + '.', + ], { + cwd: WORKTREE, + env: { + ...process.env, + DSH_DESKTOP_HOME: DSH_HOME, + DSH_MAXIMIZE: '1', + // DSH_QA is intentionally NOT set — it injects the `#qa` URL hash + // that turns on qa-harness.js's click-sweep, which fires switchTo + // in the middle of our own driver's shot sequence. We only need + // isolation + a stable renderer, not the auto-walker. + }, + stdio: ['ignore', 'pipe', 'pipe'], + }) + const logs = [] + child.stdout.on('data', d => logs.push(String(d))) + child.stderr.on('data', d => logs.push(String(d))) + for (let i = 0; i < 40; i++) { + await sleep(500) + try { + const r = await fetch(`http://localhost:${CDP_PORT}/json/list`) + if (r.ok) return { child, logs } + } catch {} + } + child.kill('SIGKILL') + console.error('electron CDP did not come up in 20s. logs:\n' + logs.join('')) + process.exit(3) +} + +async function newCdp() { + const targets = await (await fetch(`http://localhost:${CDP_PORT}/json/list`)).json() + const target = targets.find(t => t.type === 'page') + if (!target) throw new Error('no page target on port ' + CDP_PORT) + const ws = new WebSocket(target.webSocketDebuggerUrl) + await new Promise((ok, err) => { ws.onopen = ok; ws.onerror = e => err(e) }) + let id = 1 + const pending = new Map() + ws.onmessage = ev => { + const msg = JSON.parse(typeof ev.data === 'string' ? ev.data : String(ev.data)) + if (msg.id != null && pending.has(msg.id)) { + const [ok, err] = pending.get(msg.id); pending.delete(msg.id) + if (msg.error) err(new Error(msg.error.message)); else ok(msg.result) + } + } + const call = (m, p = {}, ms = 15000) => new Promise((ok, err) => { + const _id = id++ + const t = setTimeout(() => { pending.delete(_id); err(new Error('cdp timeout: ' + m)) }, ms) + pending.set(_id, [v => { clearTimeout(t); ok(v) }, e => { clearTimeout(t); err(e) }]) + ws.send(JSON.stringify({ id: _id, method: m, params: p })) + }) + const evj = async expr => { + const r = await call('Runtime.evaluate', { + expression: `(async()=>{try{return (${expr})}catch(e){return {__err:String(e)}}})()`, + returnByValue: true, awaitPromise: true, + }) + if (r.exceptionDetails) throw new Error(r.exceptionDetails.exception?.description || r.exceptionDetails.text) + return r.result?.value + } + return { ws, call, evj } +} + +async function main() { + const { child, logs } = await bootElectron() + const kill = () => { try { child.kill('SIGKILL') } catch {} } + try { + const cdp = await newCdp() + await cdp.call('Page.enable') + await cdp.call('Runtime.enable') + // Wait until the sidebar Evals button exists — proves index.html + // wired the new nav and the renderer has painted. + let ready = false + for (let i = 0; i < 60; i++) { + await sleep(200) + ready = await cdp.evj(`!!document.querySelector('.tab-btn[data-tab="evals"]')`) + if (ready) break + } + if (!ready) throw new Error('Evals nav button never mounted — check index.html + renderer.js') + + const shoot = async (name) => { + // Retry once — the Runtime tab paint kicks off async listProfiles/ + // runtimeStatus calls; on slow machines the first capture attempt + // occasionally times out while the compositor is busy. A single + // retry with a longer settle is enough in practice. + await sleep(500) + let attempts = 0 + let lastErr + while (attempts < 2) { + attempts++ + try { + const r = await cdp.call('Page.captureScreenshot', { format: 'png' }, 90000) + writeFileSync(join(OUTDIR, name + '.png'), Buffer.from(r.data, 'base64')) + console.error('wrote', name + '.png') + return + } catch (e) { + lastErr = e + console.error(`shoot ${name} attempt ${attempts} failed: ${e.message}; retrying after 800ms`) + await sleep(800) + } + } + throw lastErr + } + + // 00 — sidebar full, before entering Evals. Prove: exactly ONE + // Evals button; the three legacy ids (rubrics/growth/runtimes) + // are gone from the nav. + const sidebar = await cdp.evj(`(() => { + const items = [...document.querySelectorAll('.sidebar-nav .tab-btn')] + return items.map(b => ({ tab: b.dataset.tab, label: (b.textContent || '').trim() })) + })()`) + console.error('sidebar snapshot:', JSON.stringify(sidebar)) + const tabs = sidebar.map(s => s.tab) + if (tabs.filter(t => t === 'evals').length !== 1) throw new Error('expected exactly one evals button, got ' + tabs.filter(t => t === 'evals').length) + for (const legacy of ['rubrics', 'growth', 'runtimes']) { + if (tabs.includes(legacy)) throw new Error(`legacy nav id still present: ${legacy}`) + } + await shoot('00-sidebar-full') + + // 01 — click Evals, expect Rubrics tab active + rubrics-catalog painted. + await cdp.evj(`document.querySelector('.tab-btn[data-tab="evals"]').click()`) + await sleep(500) + const rubricsActive = await cdp.evj(`(() => { + const pane = document.querySelector('.pane[data-pane="evals"]') + if (!pane) return { ok: false, reason: 'no evals pane' } + if (pane.hidden) return { ok: false, reason: 'pane hidden after switchTo' } + const active = pane.dataset.evalsActive + const rubricsPane = pane.querySelector('.evals-tab-pane[data-evals-tab-pane="rubrics"]') + const growthPane = pane.querySelector('.evals-tab-pane[data-evals-tab-pane="growth"]') + const runtimePane = pane.querySelector('.evals-tab-pane[data-evals-tab-pane="runtime"]') + return { + ok: active === 'rubrics' && !rubricsPane.hidden && growthPane.hidden && runtimePane.hidden, + active, + rubricsHidden: rubricsPane.hidden, + growthHidden: growthPane.hidden, + runtimeHidden: runtimePane.hidden, + catalogChildren: pane.querySelectorAll('#rubrics-catalog *').length, + } + })()`) + console.error('rubrics-active state:', JSON.stringify(rubricsActive)) + if (!rubricsActive.ok) throw new Error('rubrics tab did not activate as expected: ' + JSON.stringify(rubricsActive)) + await shoot('01-evals-rubrics') + + // 02 — click Growth tab, expect only Growth pane visible. + await cdp.evj(`document.querySelector('.evals-tab[data-evals-tab="growth"]').click()`) + await sleep(500) + const growthActive = await cdp.evj(`(() => { + const pane = document.querySelector('.pane[data-pane="evals"]') + const active = pane.dataset.evalsActive + const growthPane = pane.querySelector('.evals-tab-pane[data-evals-tab-pane="growth"]') + const rubricsPane = pane.querySelector('.evals-tab-pane[data-evals-tab-pane="rubrics"]') + const runtimePane = pane.querySelector('.evals-tab-pane[data-evals-tab-pane="runtime"]') + return { ok: active === 'growth' && !growthPane.hidden && rubricsPane.hidden && runtimePane.hidden, active } + })()`) + console.error('growth-active state:', JSON.stringify(growthActive)) + if (!growthActive.ok) throw new Error('growth tab did not activate as expected: ' + JSON.stringify(growthActive)) + await shoot('02-evals-growth') + + // 03 — click Runtime tab. Confirm the rollout-grid card container + // for the fusion-seeded rubrics is populated. + await cdp.evj(`document.querySelector('.evals-tab[data-evals-tab="runtime"]').click()`) + await sleep(700) + const runtimeState = await cdp.evj(`(() => { + const evals = document.querySelector('.pane[data-pane="evals"]') + const active = evals ? evals.dataset.evalsActive : null + const runtimePane = evals ? evals.querySelector('.evals-tab-pane[data-evals-tab-pane="runtime"]') : null + const gridCards = evals ? evals.querySelectorAll('[data-testid^="rubric-grid-card-"]').length : 0 + return { + ok: !!evals && !evals.hidden && active === 'runtime' && runtimePane && !runtimePane.hidden, + active, gridCards, + } + })()`) + console.error('runtime-active state:', JSON.stringify(runtimeState)) + if (!runtimeState.ok) throw new Error('runtime tab did not activate as expected: ' + JSON.stringify(runtimeState)) + await shoot('03-evals-runtime') + + // 04 — pick a rubric in the shared selector and confirm the Runtime + // grid filters down. The fusion seed loads a handful of rubrics; we + // take the first available id. + const filterProof = await cdp.evj(`(async () => { + const sel = document.getElementById('evals-shared-rubric') + if (!sel) return { ok: false, reason: 'no shared selector' } + // Skip the "All rubrics" sentinel option; pick the first real one. + const opts = [...sel.querySelectorAll('option')].filter(o => o.value) + if (!opts.length) return { ok: false, reason: 'no rubric options' } + const pick = opts[0].value + sel.value = pick + sel.dispatchEvent(new Event('change', { bubbles: true })) + // Give the change listener + Runtime repaint a moment. + await new Promise(r => setTimeout(r, 400)) + const gridCards = [...document.querySelectorAll('[data-testid^="rubric-grid-card-"]')] + return { + ok: gridCards.length === 1, + picked: pick, + gridCardIds: gridCards.map(el => el.getAttribute('data-testid')), + } + })()`) + console.error('shared-selector filter proof:', JSON.stringify(filterProof)) + if (!filterProof.ok) throw new Error('shared-selector filter did not narrow the Runtime grid: ' + JSON.stringify(filterProof)) + await shoot('04-evals-selector-runtime') + + console.error('all evals-merge checkpoints passed') + kill() + } catch (e) { + console.error('shoot failed:', e.message) + console.error('captured logs:\n' + logs.slice(-30).join('')) + kill() + process.exit(1) + } +} + +main().catch(e => { console.error(e); process.exit(1) }) diff --git a/examples/desktop/src/renderer/index.html b/examples/desktop/src/renderer/index.html index 2d65cbcc99..993ca81f5a 100644 --- a/examples/desktop/src/renderer/index.html +++ b/examples/desktop/src/renderer/index.html @@ -104,35 +104,31 @@ -
+
runtime
- - - -
- - - - +
@@ -937,13 +923,67 @@ - +