diff --git a/examples/desktop/docs/rubric-fusion-fixture.json b/examples/desktop/docs/rubric-fusion-fixture.json new file mode 100644 index 0000000000..77cafcc07c --- /dev/null +++ b/examples/desktop/docs/rubric-fusion-fixture.json @@ -0,0 +1,94 @@ +{ + "_note": "Rubric fusion fixture — feeds Rubrics/Growth/Runtime demo. All demo-tier data.", + "rubrics": [ + { + "id": "svg-gen", + "name": "SVG generation", + "group": "interaction-reasoning", + "description": "Iterative SVG editing over 3-5 turns. Judged for shape correctness, style consistency, and no-regression.", + "dims": [ + { "id": "shape-correct", "label": "Shape correctness", "type": "continuous", "min": 0, "max": 1 }, + { "id": "style-consistent", "label": "Style consistency", "type": "continuous", "min": 0, "max": 1 }, + { "id": "no-regress", "label": "No regression", "type": "boolean", "labels": { "true": "pass", "false": "fail" } } + ] + }, + { + "id": "bug-fix", + "name": "Bug fix", + "group": "fix-optimize", + "description": "Reproduce, patch, verify. Judged on minimality and coverage.", + "dims": [ + { "id": "reproduces", "label": "Reproduces failure", "type": "boolean", "labels": { "true": "yes", "false": "no" } }, + { "id": "minimality", "label": "Patch minimality", "type": "categorical", "values": ["heavy", "ok", "clean"] }, + { "id": "no-regress", "label": "No regression", "type": "continuous", "min": 0, "max": 1 } + ] + }, + { + "id": "multi-turn-feedback", + "name": "Multi-turn feedback loop", + "group": "interaction-reasoning", + "description": "Five-dim fixed rubric — the classic RL-plan Stage-2 rubric family.", + "dims": [ + { "id": "feedback-understanding", "label": "Feedback understanding", "type": "continuous", "min": 1, "max": 5 }, + { "id": "fix-effectiveness", "label": "Fix effectiveness", "type": "continuous", "min": 1, "max": 5 }, + { "id": "convergence", "label": "Convergence", "type": "continuous", "min": 1, "max": 5 } + ] + } + ], + "similarClasses": [ + { + "id": "similar-svg-gen", + "signature": "prompt-shape: 'draw/adjust SVG'", + "count": 5, + "sessionIds": ["s-svg-1", "s-svg-2", "s-svg-3", "s-svg-4", "s-svg-5"], + "promptSummary": "5 sessions this week iterated on SVG output (draw/adjust/refine)" + } + ], + "events": [ + { "ts": 1720915200000, "rubricId": "svg-gen", "dimId": "shape-correct", "sessionId": "s-svg-1", "turnId": "t1", "rolloutIdx": 1, "score": 0.42, "harnessVersion": "v0.9", "model": "deepseek-r1", "dataMix": "mix-a" }, + { "ts": 1720915200000, "rubricId": "svg-gen", "dimId": "style-consistent", "sessionId": "s-svg-1", "turnId": "t1", "rolloutIdx": 1, "score": 0.55, "harnessVersion": "v0.9", "model": "deepseek-r1", "dataMix": "mix-a" }, + { "ts": 1720915200000, "rubricId": "svg-gen", "dimId": "no-regress", "sessionId": "s-svg-1", "turnId": "t1", "rolloutIdx": 1, "score": true, "harnessVersion": "v0.9", "model": "deepseek-r1", "dataMix": "mix-a" }, + { "ts": 1721001600000, "rubricId": "svg-gen", "dimId": "shape-correct", "sessionId": "s-svg-2", "turnId": "t2", "rolloutIdx": 2, "score": 0.51, "harnessVersion": "v0.9", "model": "deepseek-r1", "dataMix": "mix-a" }, + { "ts": 1721001600000, "rubricId": "svg-gen", "dimId": "style-consistent", "sessionId": "s-svg-2", "turnId": "t2", "rolloutIdx": 2, "score": 0.60, "harnessVersion": "v0.9", "model": "deepseek-r1", "dataMix": "mix-a" }, + { "ts": 1721001600000, "rubricId": "svg-gen", "dimId": "no-regress", "sessionId": "s-svg-2", "turnId": "t2", "rolloutIdx": 2, "score": false, "harnessVersion": "v0.9", "model": "deepseek-r1", "dataMix": "mix-a" }, + { "ts": 1721088000000, "rubricId": "svg-gen", "dimId": "shape-correct", "sessionId": "s-svg-3", "turnId": "t3", "rolloutIdx": 3, "score": 0.72, "harnessVersion": "v0.10", "model": "deepseek-r1", "dataMix": "mix-b" }, + { "ts": 1721088000000, "rubricId": "svg-gen", "dimId": "style-consistent", "sessionId": "s-svg-3", "turnId": "t3", "rolloutIdx": 3, "score": 0.78, "harnessVersion": "v0.10", "model": "deepseek-r1", "dataMix": "mix-b" }, + { "ts": 1721088000000, "rubricId": "svg-gen", "dimId": "no-regress", "sessionId": "s-svg-3", "turnId": "t3", "rolloutIdx": 3, "score": true, "harnessVersion": "v0.10", "model": "deepseek-r1", "dataMix": "mix-b" }, + { "ts": 1721174400000, "rubricId": "svg-gen", "dimId": "shape-correct", "sessionId": "s-svg-4", "turnId": "t4", "rolloutIdx": 4, "score": 0.85, "harnessVersion": "v0.11", "model": "deepseek-r1", "dataMix": "mix-b" }, + { "ts": 1721174400000, "rubricId": "svg-gen", "dimId": "style-consistent", "sessionId": "s-svg-4", "turnId": "t4", "rolloutIdx": 4, "score": 0.82, "harnessVersion": "v0.11", "model": "deepseek-r1", "dataMix": "mix-b" }, + { "ts": 1721174400000, "rubricId": "svg-gen", "dimId": "no-regress", "sessionId": "s-svg-4", "turnId": "t4", "rolloutIdx": 4, "score": true, "harnessVersion": "v0.11", "model": "deepseek-r1", "dataMix": "mix-b" }, + { "ts": 1721260800000, "rubricId": "svg-gen", "dimId": "shape-correct", "sessionId": "s-svg-5", "turnId": "t5", "rolloutIdx": 5, "score": 0.91, "harnessVersion": "v0.11", "model": "deepseek-r1", "dataMix": "mix-c" }, + { "ts": 1721260800000, "rubricId": "svg-gen", "dimId": "style-consistent", "sessionId": "s-svg-5", "turnId": "t5", "rolloutIdx": 5, "score": 0.88, "harnessVersion": "v0.11", "model": "deepseek-r1", "dataMix": "mix-c" }, + { "ts": 1721260800000, "rubricId": "svg-gen", "dimId": "no-regress", "sessionId": "s-svg-5", "turnId": "t5", "rolloutIdx": 5, "score": true, "harnessVersion": "v0.11", "model": "deepseek-r1", "dataMix": "mix-c" }, + + { "ts": 1721260800000, "rubricId": "svg-gen", "dimId": "shape-correct", "sessionId": "s-svg-live", "turnId": "tL", "rolloutIdx": 6, "score": 0.63, "harnessVersion": "v0.11", "model": "deepseek-r1", "dataMix": "mix-c" }, + { "ts": 1721260800000, "rubricId": "svg-gen", "dimId": "style-consistent", "sessionId": "s-svg-live", "turnId": "tL", "rolloutIdx": 6, "score": 0.71, "harnessVersion": "v0.11", "model": "deepseek-r1", "dataMix": "mix-c" }, + { "ts": 1721260800000, "rubricId": "svg-gen", "dimId": "no-regress", "sessionId": "s-svg-live", "turnId": "tL", "rolloutIdx": 6, "score": false, "harnessVersion": "v0.11", "model": "deepseek-r1", "dataMix": "mix-c" }, + { "ts": 1721260800000, "rubricId": "svg-gen", "dimId": "shape-correct", "sessionId": "s-svg-live", "turnId": "tL", "rolloutIdx": 7, "score": 0.80, "harnessVersion": "v0.11", "model": "deepseek-r1", "dataMix": "mix-c" }, + { "ts": 1721260800000, "rubricId": "svg-gen", "dimId": "style-consistent", "sessionId": "s-svg-live", "turnId": "tL", "rolloutIdx": 7, "score": 0.83, "harnessVersion": "v0.11", "model": "deepseek-r1", "dataMix": "mix-c" }, + { "ts": 1721260800000, "rubricId": "svg-gen", "dimId": "no-regress", "sessionId": "s-svg-live", "turnId": "tL", "rolloutIdx": 7, "score": true, "harnessVersion": "v0.11", "model": "deepseek-r1", "dataMix": "mix-c" }, + { "ts": 1721260800000, "rubricId": "svg-gen", "dimId": "shape-correct", "sessionId": "s-svg-live", "turnId": "tL", "rolloutIdx": 8, "score": 0.88, "harnessVersion": "v0.11", "model": "deepseek-r1", "dataMix": "mix-c" }, + { "ts": 1721260800000, "rubricId": "svg-gen", "dimId": "style-consistent", "sessionId": "s-svg-live", "turnId": "tL", "rolloutIdx": 8, "score": 0.85, "harnessVersion": "v0.11", "model": "deepseek-r1", "dataMix": "mix-c" }, + { "ts": 1721260800000, "rubricId": "svg-gen", "dimId": "no-regress", "sessionId": "s-svg-live", "turnId": "tL", "rolloutIdx": 8, "score": true, "harnessVersion": "v0.11", "model": "deepseek-r1", "dataMix": "mix-c" }, + + { "ts": 1720915200000, "rubricId": "bug-fix", "dimId": "reproduces", "sessionId": "s-bug-1", "turnId": "t1", "rolloutIdx": 1, "score": false, "harnessVersion": "v0.9", "model": "deepseek-r1", "dataMix": "mix-a" }, + { "ts": 1720915200000, "rubricId": "bug-fix", "dimId": "minimality", "sessionId": "s-bug-1", "turnId": "t1", "rolloutIdx": 1, "score": "heavy", "harnessVersion": "v0.9", "model": "deepseek-r1", "dataMix": "mix-a" }, + { "ts": 1720915200000, "rubricId": "bug-fix", "dimId": "no-regress", "sessionId": "s-bug-1", "turnId": "t1", "rolloutIdx": 1, "score": 0.40, "harnessVersion": "v0.9", "model": "deepseek-r1", "dataMix": "mix-a" }, + { "ts": 1721088000000, "rubricId": "bug-fix", "dimId": "reproduces", "sessionId": "s-bug-2", "turnId": "t2", "rolloutIdx": 1, "score": true, "harnessVersion": "v0.10", "model": "deepseek-r1", "dataMix": "mix-b" }, + { "ts": 1721088000000, "rubricId": "bug-fix", "dimId": "minimality", "sessionId": "s-bug-2", "turnId": "t2", "rolloutIdx": 1, "score": "ok", "harnessVersion": "v0.10", "model": "deepseek-r1", "dataMix": "mix-b" }, + { "ts": 1721088000000, "rubricId": "bug-fix", "dimId": "no-regress", "sessionId": "s-bug-2", "turnId": "t2", "rolloutIdx": 1, "score": 0.75, "harnessVersion": "v0.10", "model": "deepseek-r1", "dataMix": "mix-b" }, + { "ts": 1721260800000, "rubricId": "bug-fix", "dimId": "reproduces", "sessionId": "s-bug-3", "turnId": "t3", "rolloutIdx": 1, "score": true, "harnessVersion": "v0.11", "model": "deepseek-r1", "dataMix": "mix-c" }, + { "ts": 1721260800000, "rubricId": "bug-fix", "dimId": "minimality", "sessionId": "s-bug-3", "turnId": "t3", "rolloutIdx": 1, "score": "clean", "harnessVersion": "v0.11", "model": "deepseek-r1", "dataMix": "mix-c" }, + { "ts": 1721260800000, "rubricId": "bug-fix", "dimId": "no-regress", "sessionId": "s-bug-3", "turnId": "t3", "rolloutIdx": 1, "score": 0.92, "harnessVersion": "v0.11", "model": "deepseek-r1", "dataMix": "mix-c" }, + + { "ts": 1721001600000, "rubricId": "multi-turn-feedback", "dimId": "feedback-understanding", "sessionId": "s-mt-1", "turnId": "t1", "rolloutIdx": 1, "score": 3, "harnessVersion": "v0.10", "model": "deepseek-r1", "dataMix": "mix-b" }, + { "ts": 1721001600000, "rubricId": "multi-turn-feedback", "dimId": "fix-effectiveness", "sessionId": "s-mt-1", "turnId": "t1", "rolloutIdx": 1, "score": 3, "harnessVersion": "v0.10", "model": "deepseek-r1", "dataMix": "mix-b" }, + { "ts": 1721001600000, "rubricId": "multi-turn-feedback", "dimId": "convergence", "sessionId": "s-mt-1", "turnId": "t1", "rolloutIdx": 1, "score": 4, "harnessVersion": "v0.10", "model": "deepseek-r1", "dataMix": "mix-b" }, + { "ts": 1721174400000, "rubricId": "multi-turn-feedback", "dimId": "feedback-understanding", "sessionId": "s-mt-2", "turnId": "t2", "rolloutIdx": 1, "score": 4, "harnessVersion": "v0.11", "model": "deepseek-r1", "dataMix": "mix-b" }, + { "ts": 1721174400000, "rubricId": "multi-turn-feedback", "dimId": "fix-effectiveness", "sessionId": "s-mt-2", "turnId": "t2", "rolloutIdx": 1, "score": 4, "harnessVersion": "v0.11", "model": "deepseek-r1", "dataMix": "mix-b" }, + { "ts": 1721174400000, "rubricId": "multi-turn-feedback", "dimId": "convergence", "sessionId": "s-mt-2", "turnId": "t2", "rolloutIdx": 1, "score": 4, "harnessVersion": "v0.11", "model": "deepseek-r1", "dataMix": "mix-b" }, + { "ts": 1721260800000, "rubricId": "multi-turn-feedback", "dimId": "feedback-understanding", "sessionId": "s-mt-3", "turnId": "t3", "rolloutIdx": 1, "score": 5, "harnessVersion": "v0.11", "model": "deepseek-r1", "dataMix": "mix-c" }, + { "ts": 1721260800000, "rubricId": "multi-turn-feedback", "dimId": "fix-effectiveness", "sessionId": "s-mt-3", "turnId": "t3", "rolloutIdx": 1, "score": 5, "harnessVersion": "v0.11", "model": "deepseek-r1", "dataMix": "mix-c" }, + { "ts": 1721260800000, "rubricId": "multi-turn-feedback", "dimId": "convergence", "sessionId": "s-mt-3", "turnId": "t3", "rolloutIdx": 1, "score": 5, "harnessVersion": "v0.11", "model": "deepseek-r1", "dataMix": "mix-c" } + ] +} diff --git a/examples/desktop/docs/rubric-fusion-shots/01-rubrics.png b/examples/desktop/docs/rubric-fusion-shots/01-rubrics.png new file mode 100644 index 0000000000..3f1fcedef7 Binary files /dev/null and b/examples/desktop/docs/rubric-fusion-shots/01-rubrics.png differ diff --git a/examples/desktop/docs/rubric-fusion-shots/02-growth.png b/examples/desktop/docs/rubric-fusion-shots/02-growth.png new file mode 100644 index 0000000000..6918032c85 Binary files /dev/null and b/examples/desktop/docs/rubric-fusion-shots/02-growth.png differ diff --git a/examples/desktop/docs/rubric-fusion-shots/03-runtimes.png b/examples/desktop/docs/rubric-fusion-shots/03-runtimes.png new file mode 100644 index 0000000000..d8cee25f38 Binary files /dev/null and b/examples/desktop/docs/rubric-fusion-shots/03-runtimes.png differ diff --git a/examples/desktop/src/renderer/growth-v2.js b/examples/desktop/src/renderer/growth-v2.js index c54fdc6e29..93c9a563b6 100644 --- a/examples/desktop/src/renderer/growth-v2.js +++ b/examples/desktop/src/renderer/growth-v2.js @@ -1,40 +1,47 @@ -// Growth v2 — the recompacted "harness evolution log" for researchers. +// Growth — rubric-evolution time-series view. // -//. Data source is the growth-v2 IPC → compact-window history + -// per-window rubrics/errors that tell a "prompt was rough → added rubric → -// +42% pass" evolution story. +// This page was the "compact-window evolution log" (task #140). It has been +// re-authored as a rubric-score time-series view backed by the rubric-fusion +// event log. Horizontal axis = time (day) or harness version; each series = +// one rubric-dim; y = mean-01 score (or pass-rate, toggled by the header +// chip). Filter chips: harness version / model / data mix. // -// Modes: -// - novice (default): each compact window is a single row with a summary -// + a compression ratio + the R × N / E × M sticky-note badge. Nothing -// unfolds. Reads "impressive but not overwhelming". -// - researcher: same rows, but each expands into -// · shadowed range + full summary + eval strip (42%→94%) -// · rubric list (fixture + user-written) + `+ rubric` button -// · error list (fixture + user-written) + `› error` button -// · dev-mode metadata row (model, cw id, log path) -// — mirroring the dispatch's "对话逐条 / dev-mode 级 header/请求/raw -// response" call. In the demo we don't have per-request headers wired -// to compact windows yet, so the dev-mode row surfaces what we do have. +// This makes the page answer the question "is the harness getting better or +// worse over time, and on which rubric?" — which is what researchers +// actually need to see when they look at a Growth tab. // -// Persistence: rubric/error forms POST through window.dsh.growth.v2Add*. -// The main process writes ~/.dsh/growth/{rubrics,errors}/.json and -// echoes back the persisted entry. We re-read on success so the DOM -// reflects disk instead of trusting an optimistic in-memory push. +// Data source: window.__dshRubricFusion. Seed loaded once from +// window.__dshRubricFusionSeed if not already primed by another view. 'use strict' ;(function () { const state = { - mode: 'novice', // 'novice' | 'researcher' - payload: null, // { compactWindows, installedAt, logPath, userWrites, seedNote } - expanded: new Set(), // Set — researcher-mode row expansion - form: null, // { kind:'rubric'|'error', cwId } when a form is open - loading: false, + by: 'day', // 'day' | 'version' + metric: 'mean01', // 'mean01' | 'passRate' + groupBy: 'dim', // 'dim' | 'rubric' + filters: { // active filter chip values (null = all) + harnessVersion: null, + model: null, + dataMix: null, + }, + seeded: false, } let els = null +function fusion() { + return typeof window !== 'undefined' ? window.__dshRubricFusion : null +} + +function seedOnce() { + const f = fusion() + if (!f || state.seeded) return + const seed = window.__dshRubricFusionSeed + if (seed) f.loadFixture(seed) + state.seeded = true +} + function $(id) { return document.getElementById(id) } function el(tag, cls, text) { const n = document.createElement(tag) @@ -46,445 +53,251 @@ function el(tag, cls, text) { function mount() { const pane = document.querySelector('.pane[data-pane="growth"]') if (!pane) return - // Replace the pane body wholesale — explicitly says "推倒 - // 现有 Growth 页重来". We keep the outer
so the tab wiring - // (renderer.js:2936) still finds the pane. pane.innerHTML = '' + const header = el('header', 'header') const lead = el('div', 'header-lead') - lead.appendChild(el('div', 'page-title', 'Growth')) - const sub = el('div', 'page-sub muted') - sub.textContent = 'Compact-window history: what the runtime remembered, what it forgot, and what we taught it since.' - lead.appendChild(sub) + const title = el('div', 'page-title', 'Growth') + const chip = el('span', 'demo-tier-chip') + chip.textContent = 'demo · fusion' + chip.title = 'Time-series driven from the rubric-fusion event log (docs/rubric-fusion-fixture.json).' + title.appendChild(chip) + lead.appendChild(title) + lead.appendChild(el('div', 'page-sub muted', 'Rubric-score evolution over time. Each line is one rubric dim; toggle mean vs pass-rate, and bucket by day or by harness version.')) header.appendChild(lead) const acts = el('div', 'header-actions') - const noviceBtn = el('button', 'ghost small growth-v2-mode-chip active', 'Novice') - noviceBtn.type = 'button' - noviceBtn.dataset.mode = 'novice' - noviceBtn.title = 'One-line summary per compact window. No fold-outs.' - const researcherBtn = el('button', 'ghost small growth-v2-mode-chip', 'Researcher') - researcherBtn.type = 'button' - researcherBtn.dataset.mode = 'researcher' - researcherBtn.title = 'Expand every window to see rubrics, errors, evals, and dev-mode metadata.' - acts.append(noviceBtn, researcherBtn) + const byDay = el('button', 'ghost small growth-fusion-by-chip active', 'By day') + byDay.type = 'button'; byDay.dataset.by = 'day' + const byVersion = el('button', 'ghost small growth-fusion-by-chip', 'By version') + byVersion.type = 'button'; byVersion.dataset.by = 'version' + const mMean = el('button', 'ghost small growth-fusion-metric-chip active', 'Mean score') + mMean.type = 'button'; mMean.dataset.metric = 'mean01' + const mPass = el('button', 'ghost small growth-fusion-metric-chip', 'Pass-rate') + mPass.type = 'button'; mPass.dataset.metric = 'passRate' + acts.append(byDay, byVersion, mMean, mPass) header.appendChild(acts) pane.appendChild(header) - const body = el('section', 'growth-v2-body') - const seedNote = el('div', 'growth-v2-seed-note muted small') - seedNote.id = 'growth-v2-seed-note' - body.appendChild(seedNote) + const body = el('section', 'growth-fusion-body') - // Evolution arc — a global "42% -> 94%" summary strip at the top of the - // body. Written into `arcHost` from render() when the payload carries eval - // data; hidden otherwise so a blank page doesn't look broken. - const arcHost = el('div', 'growth-v2-arc') - arcHost.id = 'growth-v2-arc' - arcHost.hidden = true - body.appendChild(arcHost) + // Filter chip row. + const filterRow = el('div', 'growth-fusion-filter-row') + filterRow.id = 'growth-fusion-filter-row' + body.appendChild(filterRow) - const empty = el('div', 'growth-v2-empty') - empty.id = 'growth-v2-empty' + // Empty state. + const empty = el('div', 'growth-fusion-empty muted') + empty.id = 'growth-fusion-empty' + empty.textContent = 'No rubric scores yet. Once a run scores against a rubric, the curve will land here.' empty.hidden = true - empty.textContent = 'No compact windows yet. Once the runtime compacts a session, it will land here.' body.appendChild(empty) - const list = el('ol', 'growth-v2-list') - list.id = 'growth-v2-list' - body.appendChild(list) + // Chart host. + const chart = el('div', 'growth-fusion-chart') + chart.id = 'growth-fusion-chart' + body.appendChild(chart) - const formHost = el('div', 'growth-v2-form-host') - formHost.id = 'growth-v2-form-host' - formHost.hidden = true - body.appendChild(formHost) + // Legend + summary. + const legend = el('div', 'growth-fusion-legend') + legend.id = 'growth-fusion-legend' + body.appendChild(legend) pane.appendChild(body) - els = { pane, list, empty, seedNote, arcHost, formHost, noviceBtn, researcherBtn } + els = { pane, filterRow, chart, legend, empty, byDay, byVersion, mMean, mPass } - noviceBtn.addEventListener('click', () => setMode('novice')) - researcherBtn.addEventListener('click', () => setMode('researcher')) -} - -function setMode(mode) { - if (mode !== 'novice' && mode !== 'researcher') return - state.mode = mode - if (mode === 'novice') state.expanded.clear() - if (els) { - els.noviceBtn.classList.toggle('active', mode === 'novice') - els.researcherBtn.classList.toggle('active', mode === 'researcher') + for (const btn of [byDay, byVersion]) { + btn.addEventListener('click', () => { + state.by = btn.dataset.by + byDay.classList.toggle('active', state.by === 'day') + byVersion.classList.toggle('active', state.by === 'version') + render() + }) + } + for (const btn of [mMean, mPass]) { + btn.addEventListener('click', () => { + state.metric = btn.dataset.metric + mMean.classList.toggle('active', state.metric === 'mean01') + mPass.classList.toggle('active', state.metric === 'passRate') + render() + }) } - render() } async function show() { if (!els) mount() - if (state.loading) return - state.loading = true - try { - if (window.dsh && window.dsh.growth && typeof window.dsh.growth.v2Read === 'function') { - const payload = await window.dsh.growth.v2Read() - state.payload = payload - } else { - state.payload = { compactWindows: [], installedAt: null, logPath: null, userWrites: { rubrics: {}, errors: {} }, seedNote: null } - } - } finally { - state.loading = false + seedOnce() + const f = fusion() + if (f && typeof f.subscribe === 'function' && !state._subscribed) { + f.subscribe(() => { if (els) render() }) + state._subscribed = true } render() } -function projected() { - const M = window.__dshGrowthV2Model - const p = state.payload - if (!M || !p) return { compactWindows: [] } - return M.mergeAll({ compactWindows: p.compactWindows, installedAt: p.installedAt, logPath: p.logPath }, p.userWrites || {}) -} - function render() { if (!els) return - const model = window.__dshGrowthV2Model - const p = projected() - els.list.innerHTML = '' - const cws = p.compactWindows - els.empty.hidden = cws.length > 0 + const f = fusion() + if (!f) { els.empty.hidden = false; return } - if (state.payload && state.payload.seedNote) { - els.seedNote.hidden = false - els.seedNote.textContent = 'Demo data (fixture): ' + state.payload.seedNote - } else { - els.seedNote.hidden = true - els.seedNote.textContent = '' + renderFilterRow(f) + + const opts = { + by: state.by, + groupBy: state.groupBy, + filter: { + harnessVersion: state.filters.harnessVersion || undefined, + model: state.filters.model || undefined, + dataMix: state.filters.dataMix || undefined, + }, } + const data = f.timeSeriesFor(opts) + els.empty.hidden = data.xAxis.length > 0 + els.chart.innerHTML = '' + els.legend.innerHTML = '' + if (!data.xAxis.length) return - renderArc(cws) - - for (const cw of cws) { - const li = el('li', 'growth-v2-row') - li.dataset.cwId = cw.id - if (state.expanded.has(cw.id)) li.classList.add('expanded') - - // — head row: [big compression tile] [meta stack + summary] [pass-rate chip] [badges] [chevron] — - // The compression ratio is the one number that reads at a glance on a - // researcher's diary page ("28.5k tokens collapsed into 26"), so it gets - // a tile with the survival percentage large and centered. - const head = el('div', 'growth-v2-head') - - const ratio = model.compressionRatio(cw) - const tile = el('div', 'growth-v2-comp-tile') - if (ratio) { - const pct = (ratio.ratio * 100) - const big = el('div', 'growth-v2-comp-pct', pct < 0.05 ? '<0.1%' : pct.toFixed(1) + '%') - const label = el('div', 'growth-v2-comp-label muted small', 'survived') - const sub = el('div', 'growth-v2-comp-sub muted small', `${model.shortTokens(cw.shadowedTokenCount)} → ${model.shortTokens(ratio.summaryTokens)} tok`) - tile.append(big, label, sub) - } else { - tile.classList.add('empty') - } - head.appendChild(tile) - - const meta = el('div', 'growth-v2-meta') - const metaTop = el('div', 'growth-v2-meta-top') - metaTop.append( - el('div', 'growth-v2-time', model.fmtTime(cw.time)), - el('div', 'growth-v2-trigger pill', model.triggerLabel(cw.trigger || 'auto')), - el('div', 'growth-v2-range muted small', model.formatShadowedRange(cw)), - ) - meta.appendChild(metaTop) - head.appendChild(meta) - - // Pass-rate chip on the row itself (novice-visible) — echoes the arc so - // each row has its own memorable data point. Only when eval data exists. - const evalStrip = model.evalStrip(cw) - if (evalStrip) { - const chip = el('div', 'growth-v2-passrate-chip') - chip.append( - el('span', 'growth-v2-passrate-num', evalStrip.improvedTo), - el('span', 'growth-v2-passrate-label muted small', 'pass'), - ) - chip.title = `${evalStrip.name}: ${evalStrip.pass} / ${evalStrip.total} passed` - head.appendChild(chip) - } else { - head.appendChild(el('div', 'growth-v2-passrate-chip empty')) - } - - const counts = model.badgeCounts(cw) - const badge = el('div', 'growth-v2-badge pill', `R×${counts.rubrics} · E×${counts.errors}`) - badge.title = `${counts.rubrics} rubric(s), ${counts.errors} error(s) captured for this window` - if (counts.rubrics === 0 && counts.errors === 0) badge.classList.add('muted') - head.appendChild(badge) - - // Novice mode gets a click on the row itself — a subtle expand chevron - // stays visible so it doesn't look inert. - const chevron = el('button', 'ghost icon-btn growth-v2-chevron', state.expanded.has(cw.id) ? '−' : '+') - chevron.type = 'button' - chevron.title = state.expanded.has(cw.id) ? 'Collapse' : 'Expand' - chevron.addEventListener('click', (e) => { - e.stopPropagation() - if (state.expanded.has(cw.id)) state.expanded.delete(cw.id) - else state.expanded.add(cw.id) - render() - }) - head.appendChild(chevron) - li.appendChild(head) - - // — one-line summary (both modes) — - const summary = el('div', 'growth-v2-summary', cw.summary || '') - li.appendChild(summary) - - // — expanded body: eval strip + rubric list + error list + dev-mode — - if (state.expanded.has(cw.id)) { - const body = el('div', 'growth-v2-expanded') - - const evalStrip = model.evalStrip(cw) - if (evalStrip) { - const es = el('div', 'growth-v2-eval') - es.append( - el('div', 'growth-v2-eval-name', evalStrip.name), - el('div', 'growth-v2-eval-arc', `${evalStrip.improvedFrom || '—'} → ${evalStrip.improvedTo}`), - el('div', 'growth-v2-eval-count muted small', `${evalStrip.pass} / ${evalStrip.total} passed`), - ) - body.appendChild(es) - } - - body.appendChild(renderCollection('Rubrics', 'rubric', cw)) - body.appendChild(renderCollection('Errors', 'error', cw)) - - const devRow = el('div', 'growth-v2-devrow muted small') - devRow.append( - span(`cw id: ${cw.id}`), - span(`model: ${cw.model || '?'}`), - span(`store: ${state.payload && state.payload.logPath || '~/.dsh/growth/'}`), - ) - body.appendChild(devRow) - - li.appendChild(body) - } - - els.list.appendChild(li) - } + renderChart(data) + renderLegend(data) } -function span(txt) { - const s = document.createElement('span') - s.className = 'growth-v2-dev-span' - s.textContent = txt - return s +function renderFilterRow(f) { + els.filterRow.innerHTML = '' + const events = f.listEvents({}) + const versions = new Set(), models = new Set(), mixes = new Set() + for (const e of events) { + if (e.harnessVersion) versions.add(e.harnessVersion) + if (e.model) models.add(e.model) + if (e.dataMix) mixes.add(e.dataMix) + } + els.filterRow.appendChild(makeFilterGroup('Harness', 'harnessVersion', Array.from(versions).sort())) + els.filterRow.appendChild(makeFilterGroup('Model', 'model', Array.from(models).sort())) + els.filterRow.appendChild(makeFilterGroup('Data mix', 'dataMix', Array.from(mixes).sort())) } -// Evolution arc — one SVG polyline across the current window list, plotting -// each window's pass rate on a normalized 0..100 y-axis. Windows without eval -// data still contribute an x-slot but render as a hollow marker; that way the -// arc reads as "we had three compacts, only the last two have an eval, and -// look how it climbed". Skipped entirely when zero windows carry eval data. -function renderArc(cws) { - if (!els || !els.arcHost) return - const model = window.__dshGrowthV2Model - const points = cws.map((cw) => { - const ev = model.evalStrip(cw) - return { - cwId: cw.id, - time: cw.time, - rate: ev ? ev.rate : null, - prevRate: ev ? ev.prevRate : null, - label: ev ? ev.improvedTo : null, - } - }) - const hasEval = points.some((p) => p.rate != null || p.prevRate != null) - if (!hasEval || points.length === 0) { - els.arcHost.hidden = true - els.arcHost.innerHTML = '' - return +function makeFilterGroup(label, key, values) { + const group = el('div', 'growth-fusion-filter-group') + group.appendChild(el('span', 'growth-fusion-filter-label muted small', label)) + const allBtn = el('button', 'ghost small growth-fusion-filter-chip' + (state.filters[key] == null ? ' active' : ''), 'all') + allBtn.type = 'button' + allBtn.addEventListener('click', () => { state.filters[key] = null; render() }) + group.appendChild(allBtn) + for (const v of values) { + const b = el('button', 'ghost small growth-fusion-filter-chip' + (state.filters[key] === v ? ' active' : ''), v) + b.type = 'button' + b.addEventListener('click', () => { state.filters[key] = v; render() }) + group.appendChild(b) } + return group +} - // Seed the plot with the earliest prevRate so a single-eval fixture still - // draws the "from -> to" jump instead of a flat dot. - const first = points.find((p) => p.rate != null) - const seedRate = first && first.prevRate != null ? first.prevRate : (first ? first.rate : 0) - const plot = [] - if (first && first.prevRate != null) plot.push({ x: 0, rate: seedRate, label: first.prevRate != null ? Math.round(first.prevRate * 100) + '%' : null, seed: true }) - for (const p of points) { - plot.push({ x: plot.length, rate: p.rate != null ? p.rate : null, label: p.label, cwId: p.cwId }) - } - - const W = 640 - const H = 72 - const padX = 24 - const padY = 12 - const stepX = plot.length > 1 ? (W - padX * 2) / (plot.length - 1) : 0 - const yFor = (r) => H - padY - (r != null ? r : 0) * (H - padY * 2) - +// Line-chart SVG. X is index into xAxis (equal spacing); Y is metric on 0-1 +// scale (pass-rate) or normalized mean-01 (same 0-1 range so a single Y +// axis reads both metrics). +function renderChart(data) { + const W = 720 + const H = 240 + const padL = 40, padR = 24, padT = 16, padB = 40 const svgNS = 'http://www.w3.org/2000/svg' const svg = document.createElementNS(svgNS, 'svg') svg.setAttribute('viewBox', `0 0 ${W} ${H}`) - svg.setAttribute('class', 'growth-v2-arc-svg') + svg.setAttribute('class', 'growth-fusion-chart-svg') svg.setAttribute('role', 'img') - svg.setAttribute('aria-label', `Pass-rate evolution across ${cws.length} compact windows`) + svg.setAttribute('aria-label', `Rubric score evolution over ${data.xAxis.length} ${state.by === 'day' ? 'days' : 'versions'}`) - // Baseline (0%) + top-of-scale (100%) reference lines. - for (const [y, cls] of [[yFor(0), 'baseline'], [yFor(1), 'ceiling']]) { + const xCount = data.xAxis.length + const xStep = xCount > 1 ? (W - padL - padR) / (xCount - 1) : 0 + const xFor = (i) => padL + i * xStep + const yFor = (v) => padT + (1 - v) * (H - padT - padB) + + // gridlines at 0/0.5/1 + for (const [v, label] of [[0, '0'], [0.5, '0.5'], [1, '1']]) { + const y = yFor(v) const ln = document.createElementNS(svgNS, 'line') - ln.setAttribute('x1', padX); ln.setAttribute('x2', W - padX) + ln.setAttribute('x1', padL); ln.setAttribute('x2', W - padR) ln.setAttribute('y1', y); ln.setAttribute('y2', y) - ln.setAttribute('class', 'growth-v2-arc-ref-' + cls) + ln.setAttribute('class', 'growth-fusion-grid') svg.appendChild(ln) + const t = document.createElementNS(svgNS, 'text') + t.setAttribute('x', padL - 6); t.setAttribute('y', y + 4) + t.setAttribute('text-anchor', 'end') + t.setAttribute('class', 'growth-fusion-axis-tick') + t.textContent = label + svg.appendChild(t) } - const known = plot.map((p, i) => ({ ...p, i })).filter((p) => p.rate != null) - if (known.length >= 2) { - const poly = document.createElementNS(svgNS, 'polyline') - poly.setAttribute('points', known.map((p) => `${padX + p.i * stepX},${yFor(p.rate)}`).join(' ')) - poly.setAttribute('class', 'growth-v2-arc-line') - svg.appendChild(poly) + // x-axis labels (subsample if > 8) + const maxLabels = 8 + const stride = Math.max(1, Math.ceil(xCount / maxLabels)) + for (let i = 0; i < xCount; i++) { + if (i % stride !== 0 && i !== xCount - 1) continue + const t = document.createElementNS(svgNS, 'text') + t.setAttribute('x', xFor(i)); t.setAttribute('y', H - padB + 16) + t.setAttribute('text-anchor', 'middle') + t.setAttribute('class', 'growth-fusion-axis-tick') + t.textContent = data.xAxis[i] + svg.appendChild(t) } - for (const p of plot) { - if (p.rate == null) continue - const cx = padX + p.x * stepX - const cy = yFor(p.rate) - const dot = document.createElementNS(svgNS, 'circle') - dot.setAttribute('cx', cx); dot.setAttribute('cy', cy); dot.setAttribute('r', p.seed ? 3.5 : 5) - dot.setAttribute('class', p.seed ? 'growth-v2-arc-dot seed' : 'growth-v2-arc-dot') - svg.appendChild(dot) - if (p.label) { - const tx = document.createElementNS(svgNS, 'text') - tx.setAttribute('x', cx) - tx.setAttribute('y', cy - 8) - tx.setAttribute('text-anchor', 'middle') - tx.setAttribute('class', 'growth-v2-arc-label' + (p.seed ? ' seed' : '')) - tx.textContent = p.label - svg.appendChild(tx) + // series + const palette = [ + 'var(--dsh-viz-1, #7a5af8)', + 'var(--dsh-viz-2, #ffb347)', + 'var(--dsh-viz-3, #79d17b)', + 'var(--dsh-viz-4, #f96e6e)', + 'var(--dsh-viz-5, #71c9ce)', + 'var(--dsh-viz-6, #e0aaff)', + 'var(--dsh-viz-7, #c0c0c0)', + 'var(--dsh-viz-8, #ffd166)', + ] + data._colors = {} + data.series.forEach((series, si) => { + const color = palette[si % palette.length] + data._colors[series.key] = color + // Map series points (sparse — some buckets missing) to indices on xAxis + const pointIdx = series.points.map(p => ({ x: data.xAxis.indexOf(p.x), v: state.metric === 'passRate' ? p.passRate : p.mean01 })) + .filter(p => p.x >= 0 && p.v != null) + if (pointIdx.length >= 2) { + const poly = document.createElementNS(svgNS, 'polyline') + poly.setAttribute('points', pointIdx.map(p => `${xFor(p.x)},${yFor(p.v)}`).join(' ')) + poly.setAttribute('fill', 'none') + poly.setAttribute('stroke', color) + poly.setAttribute('stroke-width', '2') + poly.setAttribute('class', 'growth-fusion-line') + svg.appendChild(poly) } - } - - const caption = el('div', 'growth-v2-arc-caption muted small', `Pass-rate evolution · ${cws.length} compact window${cws.length === 1 ? '' : 's'}`) - - els.arcHost.hidden = false - els.arcHost.innerHTML = '' - els.arcHost.appendChild(svg) - els.arcHost.appendChild(caption) -} - -function renderCollection(label, kind, cw) { - const box = el('div', `growth-v2-collection growth-v2-collection-${kind}`) - const head = el('div', 'growth-v2-collection-head') - head.appendChild(el('h4', 'growth-v2-collection-title', label)) - const add = el('button', 'ghost small growth-v2-add', kind === 'rubric' ? '+ rubric' : '› error') - add.type = 'button' - add.title = kind === 'rubric' - ? 'Capture a rubric assertion for this compact window (writes ~/.dsh/growth/rubrics/…)' - : 'Flag an error case for this compact window (writes ~/.dsh/growth/errors/…)' - add.addEventListener('click', (e) => { e.stopPropagation(); openForm(kind, cw.id) }) - head.appendChild(add) - box.appendChild(head) - - const items = kind === 'rubric' ? cw.rubrics : cw.errors - if (!items || !items.length) { - box.appendChild(el('div', 'growth-v2-collection-empty muted small', kind === 'rubric' ? 'No rubrics yet.' : 'No errors flagged yet.')) - } else { - const ul = el('ul', 'growth-v2-collection-list') - for (const it of items) { - const li = el('li', 'growth-v2-collection-item') - if (kind === 'rubric') { - li.appendChild(el('div', 'growth-v2-item-primary', it.assertion || '')) - if (it.expected) li.appendChild(el('div', 'growth-v2-item-secondary muted small', `expected: ${it.expected}`)) - if (it.tag) li.appendChild(el('div', 'growth-v2-item-tag pill muted small', it.tag)) - } else { - li.appendChild(el('div', 'growth-v2-item-primary', it.text || '')) - if (it.cause) li.appendChild(el('div', 'growth-v2-item-secondary muted small', `cause: ${it.cause}`)) - if (it.todo) li.appendChild(el('div', 'growth-v2-item-tag pill muted small', `todo: ${it.todo}`)) - } - ul.appendChild(li) + for (const p of pointIdx) { + const c = document.createElementNS(svgNS, 'circle') + c.setAttribute('cx', xFor(p.x)) + c.setAttribute('cy', yFor(p.v)) + c.setAttribute('r', '3.5') + c.setAttribute('fill', color) + c.setAttribute('class', 'growth-fusion-dot') + svg.appendChild(c) } - box.appendChild(ul) - } - return box -} - -function openForm(kind, cwId) { - state.form = { kind, cwId } - const host = els.formHost - host.hidden = false - host.innerHTML = '' - const wrap = el('form', `growth-v2-form growth-v2-form-${kind}`) - wrap.appendChild(el('h4', 'growth-v2-form-title', kind === 'rubric' ? `+ rubric for ${cwId}` : `› error for ${cwId}`)) - - const fields = kind === 'rubric' - ? [ - { name: 'assertion', label: 'Assertion (natural language, required)', type: 'textarea' }, - { name: 'expected', label: 'Expected value / behavior', type: 'text' }, - { name: 'tag', label: 'Tag (optional)', type: 'text' }, - ] - : [ - { name: 'text', label: 'Error description (required)', type: 'textarea' }, - { name: 'cause', label: 'Suspected root cause', type: 'text' }, - { name: 'todo', label: 'Todo (optional)', type: 'text' }, - ] - - for (const f of fields) { - const row = el('label', 'growth-v2-form-row') - row.appendChild(el('span', 'growth-v2-form-label small', f.label)) - const input = document.createElement(f.type === 'textarea' ? 'textarea' : 'input') - input.name = f.name - if (f.type === 'text') input.type = 'text' - row.appendChild(input) - wrap.appendChild(row) - } - - const actions = el('div', 'growth-v2-form-actions') - const submit = el('button', 'primary small', 'Save') - submit.type = 'submit' - const cancel = el('button', 'ghost small', 'Cancel') - cancel.type = 'button' - cancel.addEventListener('click', closeForm) - actions.append(submit, cancel) - wrap.appendChild(actions) - - const errRow = el('div', 'growth-v2-form-error muted small') - errRow.hidden = true - wrap.appendChild(errRow) - - wrap.addEventListener('submit', async (ev) => { - ev.preventDefault() - errRow.hidden = true - const form = {} - for (const f of fields) { - const input = wrap.querySelector(`[name="${f.name}"]`) - form[f.name] = input ? String(input.value || '').trim() : '' - } - const method = kind === 'rubric' ? 'v2AddRubric' : 'v2AddError' - if (!(window.dsh && window.dsh.growth && typeof window.dsh.growth[method] === 'function')) { - errRow.hidden = false - errRow.textContent = 'IPC unavailable. Growth v2 needs the preload wire.' - return - } - const res = await window.dsh.growth[method](cwId, form) - if (!res || !res.ok) { - errRow.hidden = false - errRow.textContent = (res && res.reason === 'assertion-required') ? 'Assertion cannot be blank.' - : (res && res.reason === 'text-required') ? 'Error description cannot be blank.' - : 'Save failed.' - return - } - closeForm() - await show() }) - host.appendChild(wrap) - const first = wrap.querySelector('textarea, input[type="text"]') - if (first) first.focus() + els.chart.appendChild(svg) } -function closeForm() { - state.form = null - if (els && els.formHost) { - els.formHost.hidden = true - els.formHost.innerHTML = '' +function renderLegend(data) { + const wrap = el('div', 'growth-fusion-legend-wrap') + for (const series of data.series) { + const color = (data._colors || {})[series.key] || '#7a5af8' + const item = el('div', 'growth-fusion-legend-item') + const swatch = el('span', 'growth-fusion-legend-swatch') + swatch.style.background = color + item.appendChild(swatch) + item.appendChild(el('span', 'growth-fusion-legend-label small', series.label)) + const nPoints = series.points.length + item.appendChild(el('span', 'muted tiny', ` · ${nPoints} pt${nPoints === 1 ? '' : 's'}`)) + wrap.appendChild(item) } + els.legend.appendChild(wrap) } -const api = { mount, show, render, setMode, _state: state } -window.__dshGrowthV2 = api +const api = { mount, show, render, _state: state } +if (typeof window !== 'undefined') window.__dshGrowthV2 = api })() diff --git a/examples/desktop/src/renderer/index.html b/examples/desktop/src/renderer/index.html index 8026af5523..efbff9e0f4 100644 --- a/examples/desktop/src/renderer/index.html +++ b/examples/desktop/src/renderer/index.html @@ -1402,6 +1402,13 @@ (inlined fixture blobs), then the page renderers. --> + + + diff --git a/examples/desktop/src/renderer/rubric-fusion-model.js b/examples/desktop/src/renderer/rubric-fusion-model.js new file mode 100644 index 0000000000..fedcd7f5cd --- /dev/null +++ b/examples/desktop/src/renderer/rubric-fusion-model.js @@ -0,0 +1,419 @@ +// Rubric fusion model — a single event-log store that Rubrics, Growth, and +// Runtime pages all subscribe to. Sits above rubrics-model.js (rubric +// definitions / dim specs) and annotation-model.js (per-session records) +// and exposes derived views so each page reads what it needs without +// re-deriving from raw sessions. +// +// Shape lock: +// +// rubricDef = { +// id, name, group?, description?, +// dims: [ normalizedDimSpec ] // continuous | categorical | boolean +// } +// +// scoreEvent = { +// ts, // ms epoch +// rubricId, // key into the def registry +// dimId, // one of def.dims[*].id +// sessionId, // the session that produced this score +// turnId, // opaque turn identifier — grounds +// // back to a trace step +// rolloutIdx?, // 1..N when multiple rollouts of the +// // same session/turn are scored +// score, // dim-typed value (number for continuous, +// // enum string for categorical, bool) +// passed, // derived pass/fail (green/red) +// harnessVersion?, model?, dataMix? // filter dimensions for Growth +// } +// +// similarSessionsClass = { +// id, // 'similar-svg-gen', etc +// signature, // heuristic tag: prompt-shape / tool-usage +// count, +// sessionIds: [ ... ], +// promptSummary // one-line hint used to prefill the form +// } +// +// The store is a plain in-memory object; a subscribe() callback is fired on +// every mutation so views can re-render. Fixture data is loaded via +// loadFixture(json) — no disk I/O in the model layer. +// +// All derived views (recentScoresFor / timeSeriesFor / rolloutGridFor / +// detectSimilarSessions) are pure over the current state, so tests can +// snapshot them without a DOM. + +'use strict' +;(function () { + +const R = (typeof window !== 'undefined' && window.__dshRubricsModel) + ? window.__dshRubricsModel + : (typeof require === 'function' ? require('./rubrics-model.js') : null) + +// Default pass threshold when the caller doesn't specify one — anything +// >= 0.5 on the [0,1] normalization is green. Booleans use their literal +// value; categoricals use "not the lowest value". +const DEFAULT_PASS_THRESHOLD = 0.5 + +function createStore() { + const state = { + rubrics: new Map(), // id -> rubricDef + events: [], // scoreEvent[] + similarClasses: [], // similarSessionsClass[] + subscribers: new Set(), + } + + function notify() { + for (const cb of state.subscribers) { + try { cb() } catch (_) { /* subscriber errors don't cascade */ } + } + } + + function subscribe(cb) { + if (typeof cb !== 'function') return () => {} + state.subscribers.add(cb) + return () => state.subscribers.delete(cb) + } + + // Register (or replace) a rubric definition. Missing/invalid dims drop. + function registerRubric(def) { + if (!def || typeof def !== 'object' || !def.id) return null + const normalized = { + id: String(def.id), + name: String(def.name || def.id), + group: def.group || null, + description: def.description || '', + dims: [], + } + const rawDims = Array.isArray(def.dims) ? def.dims : [] + for (const d of rawDims) { + const spec = R && R.normalizeDimSpec ? R.normalizeDimSpec(d) : d + if (spec) normalized.dims.push(spec) + } + state.rubrics.set(normalized.id, normalized) + notify() + return normalized + } + + function listRubrics() { + return Array.from(state.rubrics.values()) + } + + function getRubric(id) { + return state.rubrics.get(String(id || '')) || null + } + + // Derive pass/fail for a score, given a dim spec. Contract: + // continuous: normalize(score) >= threshold + // categorical: score !== spec.values[0] (lowest ordinal) + // boolean: score === true (or maps to the "positive" label) + function computePassed(spec, score, threshold) { + if (!spec || score == null) return false + const t = Number.isFinite(threshold) ? threshold : DEFAULT_PASS_THRESHOLD + if (spec.type === 'boolean') return score === true || score === 'true' + if (spec.type === 'categorical') { + if (!Array.isArray(spec.values) || !spec.values.length) return false + // Lowest-value = fail; anything else = pass. + return String(score) !== String(spec.values[0]) + } + // continuous + if (R && R.normalizeReward) { + const n = R.normalizeReward(spec, score) + return n != null && n >= t + } + const num = Number(score) + const span = spec.max - spec.min + if (!Number.isFinite(num) || span <= 0) return false + return ((num - spec.min) / span) >= t + } + + // Add a raw score event. `passed` is auto-derived if omitted. + function addEvent(raw) { + if (!raw || typeof raw !== 'object') return null + const rubric = state.rubrics.get(String(raw.rubricId || '')) + if (!rubric) return null + const spec = rubric.dims.find(d => d.id === raw.dimId) + if (!spec) return null + const ts = Number(raw.ts) || Date.now() + const evt = { + ts, + rubricId: rubric.id, + dimId: spec.id, + sessionId: String(raw.sessionId || ''), + turnId: String(raw.turnId || ''), + rolloutIdx: Number.isFinite(Number(raw.rolloutIdx)) ? Number(raw.rolloutIdx) : null, + score: raw.score, + passed: raw.passed != null ? !!raw.passed : computePassed(spec, raw.score, raw.threshold), + harnessVersion: raw.harnessVersion || null, + model: raw.model || null, + dataMix: raw.dataMix || null, + } + state.events.push(evt) + notify() + return evt + } + + // Bulk load from a fixture JSON blob. Shape: + // { rubrics: [rubricDef], events: [scoreEvent], similarClasses: [class] } + function loadFixture(json) { + if (!json || typeof json !== 'object') return { rubrics: 0, events: 0 } + let rn = 0, en = 0 + for (const r of json.rubrics || []) { if (registerRubric(r)) rn++ } + for (const e of json.events || []) { if (addEvent(e)) en++ } + state.similarClasses = Array.isArray(json.similarClasses) ? json.similarClasses.slice() : [] + notify() + return { rubrics: rn, events: en } + } + + function clearAll() { + state.rubrics.clear() + state.events.length = 0 + state.similarClasses.length = 0 + notify() + } + + // ---------- filters ---------- + + function eventsMatching(filter) { + if (!filter || typeof filter !== 'object') return state.events.slice() + return state.events.filter(e => { + if (filter.rubricId && e.rubricId !== filter.rubricId) return false + if (filter.dimId && e.dimId !== filter.dimId) return false + if (filter.sessionId && e.sessionId !== filter.sessionId) return false + if (filter.since != null && e.ts < filter.since) return false + if (filter.until != null && e.ts > filter.until) return false + if (filter.harnessVersion && e.harnessVersion !== filter.harnessVersion) return false + if (filter.model && e.model !== filter.model) return false + if (filter.dataMix && e.dataMix !== filter.dataMix) return false + return true + }) + } + + function listEvents(filter) { + return eventsMatching(filter) + } + + // ---------- Rubrics-view derivation: recent scores stats ---------- + + // { total, passRate, meanScore01, byDim: { dimId -> {n, passRate, mean01} }, + // latest: [event...] (N most recent) }. + function recentScoresFor(rubricId, N = 20) { + const rubric = state.rubrics.get(String(rubricId || '')) + if (!rubric) return { total: 0, passRate: 0, meanScore01: 0, byDim: {}, latest: [] } + const all = eventsMatching({ rubricId: rubric.id }) + const total = all.length + let passed = 0 + let mean01Sum = 0 + let mean01Count = 0 + const byDim = {} + for (const d of rubric.dims) byDim[d.id] = { n: 0, passed: 0, sum01: 0, count01: 0 } + for (const e of all) { + if (e.passed) passed++ + const spec = rubric.dims.find(d => d.id === e.dimId) + const norm = spec && R && R.normalizeReward ? R.normalizeReward(spec, e.score) : null + if (norm != null) { mean01Sum += norm; mean01Count++ } + const bucket = byDim[e.dimId] + if (bucket) { + bucket.n++ + if (e.passed) bucket.passed++ + if (norm != null) { bucket.sum01 += norm; bucket.count01++ } + } + } + // Reshape byDim to final form. + const byDimOut = {} + for (const [id, b] of Object.entries(byDim)) { + byDimOut[id] = { + n: b.n, + passRate: b.n ? Math.round((b.passed / b.n) * 1000) / 1000 : 0, + mean01: b.count01 ? Math.round((b.sum01 / b.count01) * 1000) / 1000 : 0, + } + } + const latest = all.slice().sort((a, b) => b.ts - a.ts).slice(0, N) + return { + total, + passRate: total ? Math.round((passed / total) * 1000) / 1000 : 0, + meanScore01: mean01Count ? Math.round((mean01Sum / mean01Count) * 1000) / 1000 : 0, + byDim: byDimOut, + latest, + } + } + + // ---------- Growth-view derivation: time-series curves ---------- + + // Group events by { bucketKey, seriesKey } → { pass, total, sum01, cnt01 }. + // bucketKey is derived from `by`: 'day' (YYYY-MM-DD in UTC), 'version' + // (harnessVersion), or 'model' (model id). seriesKey is per-dim by default; + // callers can override with `groupBy: 'rubric' | 'dim' | 'model' | 'version'`. + // + // Returns { xAxis: [bucketKey], series: [{ key, label, points: [{x, passRate, mean01, n}] }] } + function timeSeriesFor(opts) { + opts = opts || {} + const by = opts.by || 'day' + const groupBy = opts.groupBy || 'dim' + const filter = opts.filter || {} + const filtered = eventsMatching(filter) + if (!filtered.length) return { xAxis: [], series: [] } + const bucketFn = pickBucketFn(by) + const seriesFn = pickSeriesFn(groupBy) + const rubricsById = state.rubrics + + // buckets: Map> + const buckets = new Map() + const bucketOrder = [] + for (const e of filtered) { + const bk = bucketFn(e) + if (!bk) continue + let byBucket = buckets.get(bk) + if (!byBucket) { byBucket = new Map(); buckets.set(bk, byBucket); bucketOrder.push(bk) } + const sk = seriesFn(e, rubricsById) + if (!sk) continue + let acc = byBucket.get(sk) + if (!acc) { acc = { pass: 0, total: 0, sum01: 0, cnt01: 0 }; byBucket.set(sk, acc) } + acc.total++ + if (e.passed) acc.pass++ + const rubric = rubricsById.get(e.rubricId) + const spec = rubric && rubric.dims.find(d => d.id === e.dimId) + const norm = spec && R && R.normalizeReward ? R.normalizeReward(spec, e.score) : null + if (norm != null) { acc.sum01 += norm; acc.cnt01++ } + } + bucketOrder.sort() + + // Flatten to series-major shape. + const seriesKeys = new Set() + for (const byBucket of buckets.values()) for (const k of byBucket.keys()) seriesKeys.add(k) + const series = [] + for (const key of Array.from(seriesKeys).sort()) { + const points = [] + for (const bk of bucketOrder) { + const acc = buckets.get(bk).get(key) + if (!acc) continue + points.push({ + x: bk, + passRate: acc.total ? Math.round((acc.pass / acc.total) * 1000) / 1000 : 0, + mean01: acc.cnt01 ? Math.round((acc.sum01 / acc.cnt01) * 1000) / 1000 : 0, + n: acc.total, + }) + } + series.push({ key, label: seriesLabel(key, groupBy, rubricsById), points }) + } + return { xAxis: bucketOrder, series } + } + + function pickBucketFn(by) { + if (by === 'version') return e => e.harnessVersion || 'unknown' + if (by === 'model') return e => e.model || 'unknown' + // default: day + return e => { + const d = new Date(e.ts || 0) + if (!Number.isFinite(d.getTime())) return null + const y = d.getUTCFullYear() + const m = String(d.getUTCMonth() + 1).padStart(2, '0') + const day = String(d.getUTCDate()).padStart(2, '0') + return `${y}-${m}-${day}` + } + } + + function pickSeriesFn(groupBy) { + if (groupBy === 'rubric') return e => e.rubricId + if (groupBy === 'model') return e => e.model || 'unknown' + if (groupBy === 'version') return e => e.harnessVersion || 'unknown' + // dim (default) — composite so different rubrics with same dim id stay separate + return e => `${e.rubricId}::${e.dimId}` + } + + function seriesLabel(key, groupBy, rubricsById) { + if (groupBy === 'dim') { + const [rid, did] = String(key).split('::') + const rubric = rubricsById.get(rid) + if (!rubric) return key + const spec = rubric.dims.find(d => d.id === did) + return spec ? `${rubric.name} · ${spec.label}` : `${rubric.name} · ${did}` + } + if (groupBy === 'rubric') { + const rubric = rubricsById.get(String(key)) + return rubric ? rubric.name : String(key) + } + return String(key) + } + + // ---------- Runtime-view derivation: rollout grid ---------- + + // Build a rubric x rollout matrix for a session. If sessionId is null, use + // the whole event corpus (Runtime "live" view — every session's most + // recent rollout). + // + // Returns { rubric, dims: [{id, label}], rollouts: [1..N], + // cells: [{dimId, rolloutIdx, sessionId, turnId, passed, score, ts}] } + function rolloutGridFor(rubricId, sessionId) { + const rubric = state.rubrics.get(String(rubricId || '')) + if (!rubric) return { rubric: null, dims: [], rollouts: [], cells: [] } + const filter = { rubricId: rubric.id } + if (sessionId) filter.sessionId = String(sessionId) + const all = eventsMatching(filter) + const rolloutSet = new Set() + for (const e of all) if (e.rolloutIdx != null) rolloutSet.add(e.rolloutIdx) + const rollouts = Array.from(rolloutSet).sort((a, b) => a - b) + const dims = rubric.dims.map(d => ({ id: d.id, label: d.label, type: d.type })) + const cells = [] + for (const dim of dims) { + for (const r of rollouts) { + // If multiple events for (dim, rollout), take the most recent. + let latest = null + for (const e of all) { + if (e.dimId !== dim.id) continue + if (e.rolloutIdx !== r) continue + if (!latest || e.ts > latest.ts) latest = e + } + cells.push({ + dimId: dim.id, + rolloutIdx: r, + sessionId: latest ? latest.sessionId : null, + turnId: latest ? latest.turnId : null, + passed: latest ? latest.passed : null, + score: latest ? latest.score : null, + ts: latest ? latest.ts : null, + }) + } + } + return { rubric, dims, rollouts, cells } + } + + // ---------- Hint card: similar-session detection ---------- + + // Returns the fixture-provided similarClasses for now (TODO: real + // signature heuristic once the shell has session-index access). Filters to + // classes with count >= minCount so the hint doesn't fire on singletons. + function detectSimilarSessions(opts) { + opts = opts || {} + const minCount = Number.isFinite(opts.minCount) ? opts.minCount : 3 + return state.similarClasses.filter(c => c && Number(c.count) >= minCount).slice() + } + + return { + subscribe, + registerRubric, + getRubric, + listRubrics, + addEvent, + loadFixture, + listEvents, + clearAll, + recentScoresFor, + timeSeriesFor, + rolloutGridFor, + detectSimilarSessions, + computePassed, + _state: state, // test hook + } +} + +const singleton = createStore() + +const api = { + create: createStore, + ...singleton, +} + +if (typeof module !== 'undefined' && module.exports) module.exports = api +if (typeof window !== 'undefined') window.__dshRubricFusion = api + +})() diff --git a/examples/desktop/src/renderer/rubric-fusion-seed.js b/examples/desktop/src/renderer/rubric-fusion-seed.js new file mode 100644 index 0000000000..66747304b2 --- /dev/null +++ b/examples/desktop/src/renderer/rubric-fusion-seed.js @@ -0,0 +1,624 @@ +'use strict' +;(function () { + if (typeof window === 'undefined') return + window.__dshRubricFusionSeed = { + "_note": "Rubric fusion fixture — feeds Rubrics/Growth/Runtime demo. All demo-tier data.", + "rubrics": [ + { + "id": "svg-gen", + "name": "SVG generation", + "group": "interaction-reasoning", + "description": "Iterative SVG editing over 3-5 turns. Judged for shape correctness, style consistency, and no-regression.", + "dims": [ + { + "id": "shape-correct", + "label": "Shape correctness", + "type": "continuous", + "min": 0, + "max": 1 + }, + { + "id": "style-consistent", + "label": "Style consistency", + "type": "continuous", + "min": 0, + "max": 1 + }, + { + "id": "no-regress", + "label": "No regression", + "type": "boolean", + "labels": { + "true": "pass", + "false": "fail" + } + } + ] + }, + { + "id": "bug-fix", + "name": "Bug fix", + "group": "fix-optimize", + "description": "Reproduce, patch, verify. Judged on minimality and coverage.", + "dims": [ + { + "id": "reproduces", + "label": "Reproduces failure", + "type": "boolean", + "labels": { + "true": "yes", + "false": "no" + } + }, + { + "id": "minimality", + "label": "Patch minimality", + "type": "categorical", + "values": [ + "heavy", + "ok", + "clean" + ] + }, + { + "id": "no-regress", + "label": "No regression", + "type": "continuous", + "min": 0, + "max": 1 + } + ] + }, + { + "id": "multi-turn-feedback", + "name": "Multi-turn feedback loop", + "group": "interaction-reasoning", + "description": "Five-dim fixed rubric — the classic RL-plan Stage-2 rubric family.", + "dims": [ + { + "id": "feedback-understanding", + "label": "Feedback understanding", + "type": "continuous", + "min": 1, + "max": 5 + }, + { + "id": "fix-effectiveness", + "label": "Fix effectiveness", + "type": "continuous", + "min": 1, + "max": 5 + }, + { + "id": "convergence", + "label": "Convergence", + "type": "continuous", + "min": 1, + "max": 5 + } + ] + } + ], + "similarClasses": [ + { + "id": "similar-svg-gen", + "signature": "prompt-shape: 'draw/adjust SVG'", + "count": 5, + "sessionIds": [ + "s-svg-1", + "s-svg-2", + "s-svg-3", + "s-svg-4", + "s-svg-5" + ], + "promptSummary": "5 sessions this week iterated on SVG output (draw/adjust/refine)" + } + ], + "events": [ + { + "ts": 1720915200000, + "rubricId": "svg-gen", + "dimId": "shape-correct", + "sessionId": "s-svg-1", + "turnId": "t1", + "rolloutIdx": 1, + "score": 0.42, + "harnessVersion": "v0.9", + "model": "deepseek-r1", + "dataMix": "mix-a" + }, + { + "ts": 1720915200000, + "rubricId": "svg-gen", + "dimId": "style-consistent", + "sessionId": "s-svg-1", + "turnId": "t1", + "rolloutIdx": 1, + "score": 0.55, + "harnessVersion": "v0.9", + "model": "deepseek-r1", + "dataMix": "mix-a" + }, + { + "ts": 1720915200000, + "rubricId": "svg-gen", + "dimId": "no-regress", + "sessionId": "s-svg-1", + "turnId": "t1", + "rolloutIdx": 1, + "score": true, + "harnessVersion": "v0.9", + "model": "deepseek-r1", + "dataMix": "mix-a" + }, + { + "ts": 1721001600000, + "rubricId": "svg-gen", + "dimId": "shape-correct", + "sessionId": "s-svg-2", + "turnId": "t2", + "rolloutIdx": 2, + "score": 0.51, + "harnessVersion": "v0.9", + "model": "deepseek-r1", + "dataMix": "mix-a" + }, + { + "ts": 1721001600000, + "rubricId": "svg-gen", + "dimId": "style-consistent", + "sessionId": "s-svg-2", + "turnId": "t2", + "rolloutIdx": 2, + "score": 0.6, + "harnessVersion": "v0.9", + "model": "deepseek-r1", + "dataMix": "mix-a" + }, + { + "ts": 1721001600000, + "rubricId": "svg-gen", + "dimId": "no-regress", + "sessionId": "s-svg-2", + "turnId": "t2", + "rolloutIdx": 2, + "score": false, + "harnessVersion": "v0.9", + "model": "deepseek-r1", + "dataMix": "mix-a" + }, + { + "ts": 1721088000000, + "rubricId": "svg-gen", + "dimId": "shape-correct", + "sessionId": "s-svg-3", + "turnId": "t3", + "rolloutIdx": 3, + "score": 0.72, + "harnessVersion": "v0.10", + "model": "deepseek-r1", + "dataMix": "mix-b" + }, + { + "ts": 1721088000000, + "rubricId": "svg-gen", + "dimId": "style-consistent", + "sessionId": "s-svg-3", + "turnId": "t3", + "rolloutIdx": 3, + "score": 0.78, + "harnessVersion": "v0.10", + "model": "deepseek-r1", + "dataMix": "mix-b" + }, + { + "ts": 1721088000000, + "rubricId": "svg-gen", + "dimId": "no-regress", + "sessionId": "s-svg-3", + "turnId": "t3", + "rolloutIdx": 3, + "score": true, + "harnessVersion": "v0.10", + "model": "deepseek-r1", + "dataMix": "mix-b" + }, + { + "ts": 1721174400000, + "rubricId": "svg-gen", + "dimId": "shape-correct", + "sessionId": "s-svg-4", + "turnId": "t4", + "rolloutIdx": 4, + "score": 0.85, + "harnessVersion": "v0.11", + "model": "deepseek-r1", + "dataMix": "mix-b" + }, + { + "ts": 1721174400000, + "rubricId": "svg-gen", + "dimId": "style-consistent", + "sessionId": "s-svg-4", + "turnId": "t4", + "rolloutIdx": 4, + "score": 0.82, + "harnessVersion": "v0.11", + "model": "deepseek-r1", + "dataMix": "mix-b" + }, + { + "ts": 1721174400000, + "rubricId": "svg-gen", + "dimId": "no-regress", + "sessionId": "s-svg-4", + "turnId": "t4", + "rolloutIdx": 4, + "score": true, + "harnessVersion": "v0.11", + "model": "deepseek-r1", + "dataMix": "mix-b" + }, + { + "ts": 1721260800000, + "rubricId": "svg-gen", + "dimId": "shape-correct", + "sessionId": "s-svg-5", + "turnId": "t5", + "rolloutIdx": 5, + "score": 0.91, + "harnessVersion": "v0.11", + "model": "deepseek-r1", + "dataMix": "mix-c" + }, + { + "ts": 1721260800000, + "rubricId": "svg-gen", + "dimId": "style-consistent", + "sessionId": "s-svg-5", + "turnId": "t5", + "rolloutIdx": 5, + "score": 0.88, + "harnessVersion": "v0.11", + "model": "deepseek-r1", + "dataMix": "mix-c" + }, + { + "ts": 1721260800000, + "rubricId": "svg-gen", + "dimId": "no-regress", + "sessionId": "s-svg-5", + "turnId": "t5", + "rolloutIdx": 5, + "score": true, + "harnessVersion": "v0.11", + "model": "deepseek-r1", + "dataMix": "mix-c" + }, + { + "ts": 1721260800000, + "rubricId": "svg-gen", + "dimId": "shape-correct", + "sessionId": "s-svg-live", + "turnId": "tL", + "rolloutIdx": 6, + "score": 0.63, + "harnessVersion": "v0.11", + "model": "deepseek-r1", + "dataMix": "mix-c" + }, + { + "ts": 1721260800000, + "rubricId": "svg-gen", + "dimId": "style-consistent", + "sessionId": "s-svg-live", + "turnId": "tL", + "rolloutIdx": 6, + "score": 0.71, + "harnessVersion": "v0.11", + "model": "deepseek-r1", + "dataMix": "mix-c" + }, + { + "ts": 1721260800000, + "rubricId": "svg-gen", + "dimId": "no-regress", + "sessionId": "s-svg-live", + "turnId": "tL", + "rolloutIdx": 6, + "score": false, + "harnessVersion": "v0.11", + "model": "deepseek-r1", + "dataMix": "mix-c" + }, + { + "ts": 1721260800000, + "rubricId": "svg-gen", + "dimId": "shape-correct", + "sessionId": "s-svg-live", + "turnId": "tL", + "rolloutIdx": 7, + "score": 0.8, + "harnessVersion": "v0.11", + "model": "deepseek-r1", + "dataMix": "mix-c" + }, + { + "ts": 1721260800000, + "rubricId": "svg-gen", + "dimId": "style-consistent", + "sessionId": "s-svg-live", + "turnId": "tL", + "rolloutIdx": 7, + "score": 0.83, + "harnessVersion": "v0.11", + "model": "deepseek-r1", + "dataMix": "mix-c" + }, + { + "ts": 1721260800000, + "rubricId": "svg-gen", + "dimId": "no-regress", + "sessionId": "s-svg-live", + "turnId": "tL", + "rolloutIdx": 7, + "score": true, + "harnessVersion": "v0.11", + "model": "deepseek-r1", + "dataMix": "mix-c" + }, + { + "ts": 1721260800000, + "rubricId": "svg-gen", + "dimId": "shape-correct", + "sessionId": "s-svg-live", + "turnId": "tL", + "rolloutIdx": 8, + "score": 0.88, + "harnessVersion": "v0.11", + "model": "deepseek-r1", + "dataMix": "mix-c" + }, + { + "ts": 1721260800000, + "rubricId": "svg-gen", + "dimId": "style-consistent", + "sessionId": "s-svg-live", + "turnId": "tL", + "rolloutIdx": 8, + "score": 0.85, + "harnessVersion": "v0.11", + "model": "deepseek-r1", + "dataMix": "mix-c" + }, + { + "ts": 1721260800000, + "rubricId": "svg-gen", + "dimId": "no-regress", + "sessionId": "s-svg-live", + "turnId": "tL", + "rolloutIdx": 8, + "score": true, + "harnessVersion": "v0.11", + "model": "deepseek-r1", + "dataMix": "mix-c" + }, + { + "ts": 1720915200000, + "rubricId": "bug-fix", + "dimId": "reproduces", + "sessionId": "s-bug-1", + "turnId": "t1", + "rolloutIdx": 1, + "score": false, + "harnessVersion": "v0.9", + "model": "deepseek-r1", + "dataMix": "mix-a" + }, + { + "ts": 1720915200000, + "rubricId": "bug-fix", + "dimId": "minimality", + "sessionId": "s-bug-1", + "turnId": "t1", + "rolloutIdx": 1, + "score": "heavy", + "harnessVersion": "v0.9", + "model": "deepseek-r1", + "dataMix": "mix-a" + }, + { + "ts": 1720915200000, + "rubricId": "bug-fix", + "dimId": "no-regress", + "sessionId": "s-bug-1", + "turnId": "t1", + "rolloutIdx": 1, + "score": 0.4, + "harnessVersion": "v0.9", + "model": "deepseek-r1", + "dataMix": "mix-a" + }, + { + "ts": 1721088000000, + "rubricId": "bug-fix", + "dimId": "reproduces", + "sessionId": "s-bug-2", + "turnId": "t2", + "rolloutIdx": 1, + "score": true, + "harnessVersion": "v0.10", + "model": "deepseek-r1", + "dataMix": "mix-b" + }, + { + "ts": 1721088000000, + "rubricId": "bug-fix", + "dimId": "minimality", + "sessionId": "s-bug-2", + "turnId": "t2", + "rolloutIdx": 1, + "score": "ok", + "harnessVersion": "v0.10", + "model": "deepseek-r1", + "dataMix": "mix-b" + }, + { + "ts": 1721088000000, + "rubricId": "bug-fix", + "dimId": "no-regress", + "sessionId": "s-bug-2", + "turnId": "t2", + "rolloutIdx": 1, + "score": 0.75, + "harnessVersion": "v0.10", + "model": "deepseek-r1", + "dataMix": "mix-b" + }, + { + "ts": 1721260800000, + "rubricId": "bug-fix", + "dimId": "reproduces", + "sessionId": "s-bug-3", + "turnId": "t3", + "rolloutIdx": 1, + "score": true, + "harnessVersion": "v0.11", + "model": "deepseek-r1", + "dataMix": "mix-c" + }, + { + "ts": 1721260800000, + "rubricId": "bug-fix", + "dimId": "minimality", + "sessionId": "s-bug-3", + "turnId": "t3", + "rolloutIdx": 1, + "score": "clean", + "harnessVersion": "v0.11", + "model": "deepseek-r1", + "dataMix": "mix-c" + }, + { + "ts": 1721260800000, + "rubricId": "bug-fix", + "dimId": "no-regress", + "sessionId": "s-bug-3", + "turnId": "t3", + "rolloutIdx": 1, + "score": 0.92, + "harnessVersion": "v0.11", + "model": "deepseek-r1", + "dataMix": "mix-c" + }, + { + "ts": 1721001600000, + "rubricId": "multi-turn-feedback", + "dimId": "feedback-understanding", + "sessionId": "s-mt-1", + "turnId": "t1", + "rolloutIdx": 1, + "score": 3, + "harnessVersion": "v0.10", + "model": "deepseek-r1", + "dataMix": "mix-b" + }, + { + "ts": 1721001600000, + "rubricId": "multi-turn-feedback", + "dimId": "fix-effectiveness", + "sessionId": "s-mt-1", + "turnId": "t1", + "rolloutIdx": 1, + "score": 3, + "harnessVersion": "v0.10", + "model": "deepseek-r1", + "dataMix": "mix-b" + }, + { + "ts": 1721001600000, + "rubricId": "multi-turn-feedback", + "dimId": "convergence", + "sessionId": "s-mt-1", + "turnId": "t1", + "rolloutIdx": 1, + "score": 4, + "harnessVersion": "v0.10", + "model": "deepseek-r1", + "dataMix": "mix-b" + }, + { + "ts": 1721174400000, + "rubricId": "multi-turn-feedback", + "dimId": "feedback-understanding", + "sessionId": "s-mt-2", + "turnId": "t2", + "rolloutIdx": 1, + "score": 4, + "harnessVersion": "v0.11", + "model": "deepseek-r1", + "dataMix": "mix-b" + }, + { + "ts": 1721174400000, + "rubricId": "multi-turn-feedback", + "dimId": "fix-effectiveness", + "sessionId": "s-mt-2", + "turnId": "t2", + "rolloutIdx": 1, + "score": 4, + "harnessVersion": "v0.11", + "model": "deepseek-r1", + "dataMix": "mix-b" + }, + { + "ts": 1721174400000, + "rubricId": "multi-turn-feedback", + "dimId": "convergence", + "sessionId": "s-mt-2", + "turnId": "t2", + "rolloutIdx": 1, + "score": 4, + "harnessVersion": "v0.11", + "model": "deepseek-r1", + "dataMix": "mix-b" + }, + { + "ts": 1721260800000, + "rubricId": "multi-turn-feedback", + "dimId": "feedback-understanding", + "sessionId": "s-mt-3", + "turnId": "t3", + "rolloutIdx": 1, + "score": 5, + "harnessVersion": "v0.11", + "model": "deepseek-r1", + "dataMix": "mix-c" + }, + { + "ts": 1721260800000, + "rubricId": "multi-turn-feedback", + "dimId": "fix-effectiveness", + "sessionId": "s-mt-3", + "turnId": "t3", + "rolloutIdx": 1, + "score": 5, + "harnessVersion": "v0.11", + "model": "deepseek-r1", + "dataMix": "mix-c" + }, + { + "ts": 1721260800000, + "rubricId": "multi-turn-feedback", + "dimId": "convergence", + "sessionId": "s-mt-3", + "turnId": "t3", + "rolloutIdx": 1, + "score": 5, + "harnessVersion": "v0.11", + "model": "deepseek-r1", + "dataMix": "mix-c" + } + ] +}; +})() diff --git a/examples/desktop/src/renderer/rubrics-page.js b/examples/desktop/src/renderer/rubrics-page.js index aad79f9d82..9c93581067 100644 --- a/examples/desktop/src/renderer/rubrics-page.js +++ b/examples/desktop/src/renderer/rubrics-page.js @@ -33,6 +33,19 @@ ? window.__dshRubricsModel : (typeof require === 'function' ? require('./rubrics-model.js') : {}) + // Fusion store — same singleton used by Growth/Runtime; seeded once on + // first render so the three views share the exact same event log. + const fusion = (typeof window !== 'undefined' && window.__dshRubricFusion) + ? window.__dshRubricFusion + : null + let fusionSeeded = false + function seedFusionOnce() { + if (fusionSeeded || !fusion) return + const seed = (typeof window !== 'undefined' && window.__dshRubricFusionSeed) || null + if (seed) fusion.loadFixture(seed) + fusionSeeded = true + } + // Renderer state — kept minimal + re-derivable, per the same pattern as // pr-page.js. Every write triggers a re-render of the affected region. const state = { @@ -43,11 +56,11 @@ createForm: null, // Create-from-scratch draft; null when the form // is closed. Shape: { // name, group, executor, colorDot, - // dimName, dimType, min, max, values, labels + // dimName, dimType, min, max, values, labels, + // hintClassId?, hintPromptSummary? // }. - // Only one draft at a time — the form is a - // modal-in-the-page anchored below the fallback - // CTA, so we don't need a separate id. + dismissedHints: new Set(), // similar-session hint class ids the user + // dismissed this session. } function el(tag, attrs = {}, children = []) { @@ -102,6 +115,7 @@ ]), el('div', { className: 'rubric-tile-desc muted small', text: rubric.description || '' }), el('div', { className: 'rubric-tile-preview', text: preview || 'No checklist items.' }), + renderTileStatsStrip(rubric), el('div', { className: 'rubric-tile-foot' }, [ el('span', { className: 'rubric-tile-executor muted small', text: rubric.executor === 'code' ? 'Code executor' : 'LLM-as-judge' }), el('span', { className: 'rubric-tile-attach chip small', text: 'Attach to Bench' }), @@ -109,6 +123,27 @@ ]) } + // Recent-scores mini strip — total scored + pass-rate% + 8 sparkline + // pass/fail dots. Reads from the fusion event log. Renders an empty + // placeholder when there are no scores yet (keeps tile height stable). + function renderTileStatsStrip(rubric) { + if (!fusion) return el('div', { className: 'rubric-tile-stats empty' }) + const stats = fusion.recentScoresFor(rubric.id, 8) + if (!stats || !stats.total) { + return el('div', { className: 'rubric-tile-stats empty muted small', text: 'No scores yet' }) + } + const pct = Math.round(stats.passRate * 100) + const dots = stats.latest.slice().reverse().map(evt => el('span', { + className: 'rubric-tile-stats-dot ' + (evt.passed ? 'pass' : 'fail'), + title: evt.dimId + ' · ' + (evt.passed ? 'pass' : 'fail'), + })) + return el('div', { className: 'rubric-tile-stats', 'data-testid': 'tile-stats-' + rubric.id }, [ + el('span', { className: 'rubric-tile-stats-total small', text: stats.total + ' scored' }), + el('span', { className: 'rubric-tile-stats-rate small' + (pct >= 50 ? ' pass' : ' fail'), text: pct + '% pass' }), + el('span', { className: 'rubric-tile-stats-spark' }, dots), + ]) + } + function renderGroupSection(group) { const isEmpty = !group.rubrics.length const rows = isEmpty @@ -428,11 +463,57 @@ function renderCatalog() { const host = document.getElementById('rubrics-catalog') if (!host) return + seedFusionOnce() host.replaceChildren() + const hint = renderSimilarSessionsHint() + if (hint) host.appendChild(hint) for (const group of state.catalog) host.appendChild(renderGroupSection(group)) host.appendChild(renderFallbackCTA()) } + // "Detected N similar sessions this week" hint card. Reads similarClasses + // from the fusion store (heuristic detection is a TODO — fixture drives + // the shape today). Dismissable per-class. + function renderSimilarSessionsHint() { + if (!fusion) return null + const classes = fusion.detectSimilarSessions() + const cls = classes.find(c => !state.dismissedHints.has(c.id)) + if (!cls) return null + return el('div', { className: 'rubric-hint-card', 'data-testid': 'rubric-hint-card', role: 'note' }, [ + el('span', { className: 'rubric-hint-icon', text: '✦' }), + el('div', { className: 'rubric-hint-body' }, [ + el('div', { className: 'rubric-hint-title', text: `Detected ${cls.count} similar sessions this week` }), + el('div', { className: 'rubric-hint-sub muted small', text: cls.promptSummary || 'These look like a repeated task class — a rubric would let you track it.' }), + ]), + el('button', { + className: 'primary small rubric-hint-cta', + type: 'button', + text: 'Enable a rubric for this task class', + onclick: () => { + openCreateForm('llm-judge') + if (state.createForm) { + state.createForm.dimName = 'task-class-' + cls.id.replace(/[^a-z0-9-]/gi, '-').toLowerCase() + state.createForm.hintClassId = cls.id + state.createForm.hintPromptSummary = cls.promptSummary || '' + renderCatalog() + } + }, + }), + el('button', { + className: 'ghost small rubric-hint-dismiss', + type: 'button', + text: '×', + title: 'Dismiss for this class', + 'aria-label': 'Dismiss', + onclick: (e) => { + e.stopPropagation() + state.dismissedHints.add(cls.id) + renderCatalog() + }, + }), + ]) + } + function findRubric(id) { return state.rubrics.find(r => r.id === id) || null } diff --git a/examples/desktop/src/renderer/runtimes-page.js b/examples/desktop/src/renderer/runtimes-page.js index 8467f5b3eb..49ee2ded8c 100644 --- a/examples/desktop/src/renderer/runtimes-page.js +++ b/examples/desktop/src/renderer/runtimes-page.js @@ -11,12 +11,13 @@ // initialize handshake (Ticket G) // - window.dsh.playground.list() — isolated daemon(s) currently up // -// The IA design pack §3 flagged this surface as filling the `runtime/list` -// wire gap (G8) with a local composition today: profiles from disk + a -// probe of the mounted runtime + Playground's already-broadcasting isolated -// daemon list. The page renders honest chips saying so — "composed locally -// · G8 gap" — so a researcher inspecting this page knows exactly which -// bits are wire-authoritative vs. locally-inferred. +// This page now has two tabs: +// - Rubric grid (default): the researcher-facing red/green matrix — one +// row per rubric dim, one column per rollout, cells colored by pass/ +// fail. Click a cell → jump to that turn's trace. Data source is the +// rubric-fusion event log. +// - Status: the original composition — profile list + isolated-daemon +// section — for wire/adapter debugging. // // This module is a pure DOM controller: no fetches, no timers other than a // single refresh interval bound to the page's visibility. It exposes @@ -26,6 +27,20 @@ 'use strict' ;(function () { const rootId = 'runtimes-pane' + let currentTab = 'rubric-grid' + let fusionSeeded = false + + function fusion() { + return typeof window !== 'undefined' ? window.__dshRubricFusion : null + } + + function seedFusionOnce() { + const f = fusion() + if (!f || fusionSeeded) return + const seed = typeof window !== 'undefined' ? window.__dshRubricFusionSeed : null + if (seed) f.loadFixture(seed) + fusionSeeded = true + } // Human-readable label for each capability bit. Kept in sync with // src/renderer/capabilities.js CAPABILITY_KEYS — the six bits the shell @@ -304,7 +319,176 @@ async function show() { const root = document.getElementById(rootId) if (!root) return - await refresh(root) + ensureTabStrip(root) + seedFusionOnce() + if (currentTab === 'rubric-grid') { + await renderRubricGrid(root) + } else { + await refresh(root) + } + } + + // Inject a two-tab strip into the header-actions once. Idempotent. + function ensureTabStrip(root) { + if (root.querySelector('[data-runtimes-tabs]')) return + const acts = root.querySelector('.header-actions') + if (!acts) return + const strip = document.createElement('div') + strip.className = 'runtimes-tabs' + strip.setAttribute('data-runtimes-tabs', '') + const gridBtn = document.createElement('button') + gridBtn.type = 'button' + gridBtn.className = 'ghost small runtimes-tab active' + gridBtn.dataset.runtimesTab = 'rubric-grid' + gridBtn.textContent = 'Rubric grid' + gridBtn.title = 'Rollout × rubric-dim red/green matrix (default).' + const statusBtn = document.createElement('button') + statusBtn.type = 'button' + statusBtn.className = 'ghost small runtimes-tab' + statusBtn.dataset.runtimesTab = 'status' + statusBtn.textContent = 'Status' + statusBtn.title = 'Profiles, adapter capabilities, isolated daemons.' + strip.append(gridBtn, statusBtn) + // Insert before the existing legend chip so the tabs sit at the head. + acts.insertBefore(strip, acts.firstChild) + + strip.addEventListener('click', (e) => { + const btn = e.target.closest('[data-runtimes-tab]') + if (!btn) return + currentTab = btn.dataset.runtimesTab + for (const b of strip.querySelectorAll('.runtimes-tab')) b.classList.toggle('active', b === btn) + void show() + }) + } + + // ---------- Rubric grid tab ---------- + + async function renderRubricGrid(root) { + const bodyList = root.querySelector('[data-runtimes-list]') + const bodyIsolated = root.querySelector('[data-runtimes-isolated]') + if (bodyList) bodyList.innerHTML = '' + if (bodyIsolated) bodyIsolated.innerHTML = '' + // We drop grid output into the .runtimes-list slot so no HTML + // template edit is needed — the isolated-daemon slot stays empty on + // this tab (that section only makes sense under Status). + const host = bodyList + if (!host) return + const f = fusion() + if (!f) { + host.appendChild(muted('Rubric fusion store not loaded.')) + return + } + const rubrics = f.listRubrics() + if (!rubrics.length) { + host.appendChild(muted('No rubrics registered. Author one under Rubrics → Create from scratch, or load the fusion fixture.')) + return + } + // Header row: one card per rubric. + for (const rubric of rubrics) { + const grid = f.rolloutGridFor(rubric.id, null) + const card = document.createElement('section') + card.className = 'rubric-grid-card' + card.setAttribute('data-testid', 'rubric-grid-card-' + rubric.id) + const head = document.createElement('header') + head.className = 'rubric-grid-head' + head.appendChild(spanCls('rubric-grid-name', rubric.name)) + head.appendChild(spanCls('rubric-grid-desc muted small', rubric.description || '')) + head.appendChild(spanCls('rubric-grid-rubric-id muted tiny', 'rubric: ' + rubric.id)) + card.appendChild(head) + + if (!grid.rollouts.length) { + card.appendChild(muted('No rollouts scored against this rubric yet.')) + } else { + card.appendChild(renderGridTable(grid)) + } + host.appendChild(card) + } + } + + function renderGridTable(grid) { + const wrap = document.createElement('div') + wrap.className = 'rubric-grid-table-wrap' + const table = document.createElement('table') + table.className = 'rubric-grid-table' + // Header row: rollouts across the top. + const thead = document.createElement('thead') + const hrow = document.createElement('tr') + const corner = document.createElement('th') + corner.className = 'rubric-grid-corner' + corner.textContent = 'dim ╲ rollout' + hrow.appendChild(corner) + for (const r of grid.rollouts) { + const th = document.createElement('th') + th.className = 'rubric-grid-col-head' + th.textContent = String(r) + hrow.appendChild(th) + } + thead.appendChild(hrow) + table.appendChild(thead) + const tbody = document.createElement('tbody') + for (const dim of grid.dims) { + const trow = document.createElement('tr') + const rh = document.createElement('th') + rh.className = 'rubric-grid-row-head' + rh.textContent = dim.label + rh.title = dim.type + trow.appendChild(rh) + for (const r of grid.rollouts) { + const cell = grid.cells.find(c => c.dimId === dim.id && c.rolloutIdx === r) + const td = document.createElement('td') + td.className = 'rubric-grid-cell' + if (!cell || cell.passed == null) { + td.classList.add('rubric-grid-cell--empty') + td.title = 'No score.' + } else if (cell.passed) { + td.classList.add('rubric-grid-cell--pass') + td.title = `pass · session ${cell.sessionId} · turn ${cell.turnId}` + } else { + td.classList.add('rubric-grid-cell--fail') + td.title = `fail · session ${cell.sessionId} · turn ${cell.turnId}` + } + if (cell && cell.sessionId && cell.turnId) { + td.setAttribute('data-session-id', cell.sessionId) + td.setAttribute('data-turn-id', cell.turnId) + td.setAttribute('role', 'button') + td.setAttribute('tabindex', '0') + const jumpTo = () => jumpToTrace(cell.sessionId, cell.turnId) + td.addEventListener('click', jumpTo) + td.addEventListener('keydown', (e) => { if (e.key === 'Enter' || e.key === ' ') { e.preventDefault(); jumpTo() } }) + } + trow.appendChild(td) + } + tbody.appendChild(trow) + } + table.appendChild(tbody) + wrap.appendChild(table) + return wrap + } + + function jumpToTrace(sessionId, turnId) { + // Signal via the existing tab switcher; downstream trace/tracing page + // can pick this event up. We don't hardcode the trace page URL — the + // shell owns navigation. + if (typeof window !== 'undefined' && typeof CustomEvent === 'function') { + window.dispatchEvent(new CustomEvent('dsh:rubric-cell-jump', { detail: { sessionId, turnId } })) + } + if (window.__dshTabs && typeof window.__dshTabs.switchTo === 'function') { + window.__dshTabs.switchTo('tracing') + } + } + + function muted(text) { + const div = document.createElement('div') + div.className = 'muted small' + div.textContent = text + return div + } + + function spanCls(cls, text) { + const s = document.createElement('span') + s.className = cls + s.textContent = text + return s } if (typeof window !== 'undefined') { diff --git a/examples/desktop/src/renderer/style.css b/examples/desktop/src/renderer/style.css index 7f2999857b..f265fe3a38 100644 --- a/examples/desktop/src/renderer/style.css +++ b/examples/desktop/src/renderer/style.css @@ -11850,3 +11850,254 @@ details.devtools-row[open] > .devtools-row-summary::before { transform: rotate(9 /* Give the pane a positioning context so the absolute drawer anchors * inside it, not against the viewport root. */ .pane[data-pane="chat"] { position: relative; } +/* ============================================================ */ +/* D lane: Rubric fusion (Rubrics + Growth + Runtime shared) */ +/* ============================================================ */ + +/* --- Rubrics view: hint card + per-tile stats strip --- */ + +.rubric-hint-card { + display: flex; + align-items: center; + gap: 12px; + padding: 12px 14px; + margin: 0 0 16px 0; + border-radius: 12px; + background: linear-gradient(90deg, rgba(122, 90, 248, 0.10), rgba(225, 170, 255, 0.06)); + border: 1px solid rgba(122, 90, 248, 0.35); +} +.rubric-hint-icon { + font-size: 20px; + flex: 0 0 auto; +} +.rubric-hint-body { + flex: 1 1 auto; + min-width: 0; +} +.rubric-hint-title { + font-weight: 600; + font-size: 14px; + margin-bottom: 2px; +} +.rubric-hint-cta { + flex: 0 0 auto; + white-space: nowrap; +} +.rubric-hint-dismiss { + flex: 0 0 auto; + min-width: 28px; + padding: 2px 8px; +} + +.rubric-tile-stats { + display: flex; + align-items: center; + gap: 8px; + padding: 6px 0 4px 0; + min-height: 20px; +} +.rubric-tile-stats.empty { + opacity: 0.6; +} +.rubric-tile-stats-total { + font-variant-numeric: tabular-nums; +} +.rubric-tile-stats-rate { + padding: 2px 6px; + border-radius: 4px; + font-variant-numeric: tabular-nums; + font-weight: 600; +} +.rubric-tile-stats-rate.pass { + background: rgba(121, 209, 123, 0.18); + color: #1f7a2b; +} +.rubric-tile-stats-rate.fail { + background: rgba(249, 110, 110, 0.18); + color: #a83232; +} +@media (prefers-color-scheme: dark) { + .rubric-tile-stats-rate.pass { color: #79d17b; } + .rubric-tile-stats-rate.fail { color: #f96e6e; } +} +.rubric-tile-stats-spark { + display: inline-flex; + gap: 2px; + align-items: center; +} +.rubric-tile-stats-dot { + display: inline-block; + width: 6px; + height: 10px; + border-radius: 1px; +} +.rubric-tile-stats-dot.pass { background: #79d17b; } +.rubric-tile-stats-dot.fail { background: #f96e6e; } + +/* --- Growth view: time-series chart --- */ + +.growth-fusion-body { + padding: 12px 16px 24px 16px; + display: flex; + flex-direction: column; + gap: 12px; +} +.growth-fusion-filter-row { + display: flex; + flex-wrap: wrap; + gap: 12px; + padding: 8px 0; +} +.growth-fusion-filter-group { + display: flex; + align-items: center; + gap: 4px; +} +.growth-fusion-filter-label { + padding-right: 4px; +} +.growth-fusion-filter-chip { + padding: 2px 8px; + border-radius: 4px; +} +.growth-fusion-filter-chip.active { + background: rgba(122, 90, 248, 0.18); + border-color: rgba(122, 90, 248, 0.6); + color: inherit; +} +.growth-fusion-chart { + width: 100%; + padding: 8px 0; +} +.growth-fusion-chart-svg { + width: 100%; + height: auto; + max-height: 320px; + display: block; +} +.growth-fusion-grid { + stroke: currentColor; + stroke-opacity: 0.15; + stroke-width: 1; + stroke-dasharray: 2 3; +} +.growth-fusion-axis-tick { + fill: currentColor; + fill-opacity: 0.55; + font-size: 10px; +} +.growth-fusion-legend-wrap { + display: flex; + flex-wrap: wrap; + gap: 12px; + padding: 8px 0 0 0; +} +.growth-fusion-legend-item { + display: inline-flex; + align-items: center; + gap: 6px; +} +.growth-fusion-legend-swatch { + display: inline-block; + width: 12px; + height: 12px; + border-radius: 2px; +} +.growth-fusion-empty { + padding: 24px 0; + text-align: center; +} + +/* --- Runtime view: tabs + rubric grid --- */ + +.runtimes-tabs { + display: inline-flex; + gap: 4px; + margin-right: 12px; +} +.runtimes-tab { + padding: 2px 10px; + border-radius: 4px; +} +.runtimes-tab.active { + background: rgba(122, 90, 248, 0.18); + border-color: rgba(122, 90, 248, 0.6); +} + +.rubric-grid-card { + padding: 12px 12px 16px 12px; + border: 1px solid var(--surface-hover, rgba(0, 0, 0, 0.08)); + border-radius: 10px; + margin-bottom: 14px; + background: var(--surface, #fff); +} +.rubric-grid-head { + display: flex; + flex-wrap: wrap; + align-items: baseline; + gap: 8px; + margin-bottom: 8px; +} +.rubric-grid-name { + font-weight: 600; + font-size: 15px; +} +.rubric-grid-rubric-id { + margin-left: auto; +} +.rubric-grid-table-wrap { + overflow-x: auto; +} +.rubric-grid-table { + border-collapse: collapse; + font-size: 12px; +} +.rubric-grid-corner, +.rubric-grid-row-head { + text-align: left; + padding: 4px 8px; + font-weight: 500; + color: rgba(0, 0, 0, 0.6); + background: var(--surface-2, #f7f7f8); + border: 1px solid var(--surface-hover, rgba(0, 0, 0, 0.08)); +} +@media (prefers-color-scheme: dark) { + .rubric-grid-corner, + .rubric-grid-row-head { color: rgba(255, 255, 255, 0.7); } +} +.rubric-grid-col-head { + text-align: center; + padding: 4px 6px; + font-weight: 500; + min-width: 24px; + background: var(--surface-2, #f7f7f8); + border: 1px solid var(--surface-hover, rgba(0, 0, 0, 0.08)); +} +.rubric-grid-cell { + width: 28px; + height: 26px; + border: 1px solid var(--surface-hover, rgba(0, 0, 0, 0.08)); + cursor: default; +} +.rubric-grid-cell--pass { + background: #79d17b; + cursor: pointer; +} +.rubric-grid-cell--fail { + background: #f96e6e; + cursor: pointer; +} +.rubric-grid-cell--empty { + background: repeating-linear-gradient( + 45deg, + var(--surface-2, #f7f7f8), + var(--surface-2, #f7f7f8) 4px, + var(--surface-hover, rgba(0,0,0,0.06)) 4px, + var(--surface-hover, rgba(0,0,0,0.06)) 6px + ); +} +.rubric-grid-cell--pass:hover, +.rubric-grid-cell--fail:hover { + outline: 2px solid rgba(122, 90, 248, 0.8); + outline-offset: -2px; +} diff --git a/examples/desktop/test/rubric-fusion-model.test.js b/examples/desktop/test/rubric-fusion-model.test.js new file mode 100644 index 0000000000..a3e7dd17c1 --- /dev/null +++ b/examples/desktop/test/rubric-fusion-model.test.js @@ -0,0 +1,181 @@ +// Rubric fusion model tests — pure derivations (recent scores, time series, +// rollout grid, similar-session detection). + +'use strict' + +// Load rubrics-model first — fusion-model depends on it via global fallback. +global.window = global.window || {} +if (!global.window.__dshRubricsModel) { + global.window.__dshRubricsModel = require('../src/renderer/rubrics-model.js') +} + +const test = require('node:test') +const assert = require('node:assert') + +// require via a fresh module — the singleton in the fusion model would +// otherwise carry state across tests. +function freshStore() { + delete require.cache[require.resolve('../src/renderer/rubric-fusion-model.js')] + const api = require('../src/renderer/rubric-fusion-model.js') + return api.create() +} + +const SAMPLE_RUBRIC = { + id: 'svg-gen', + name: 'SVG generation', + dims: [ + { id: 'shape', label: 'Shape', type: 'continuous', min: 0, max: 1 }, + { id: 'pass', label: 'Pass', type: 'boolean' }, + { id: 'quality', label: 'Quality', type: 'categorical', values: ['bad', 'ok', 'good'] }, + ], +} + +test('registerRubric normalizes dims and returns a stable id', () => { + const s = freshStore() + const def = s.registerRubric(SAMPLE_RUBRIC) + assert.equal(def.id, 'svg-gen') + assert.equal(def.dims.length, 3) + assert.equal(def.dims[0].type, 'continuous') + assert.equal(def.dims[0].min, 0) + assert.equal(def.dims[0].max, 1) +}) + +test('addEvent derives passed from dim spec', () => { + const s = freshStore() + s.registerRubric(SAMPLE_RUBRIC) + const hi = s.addEvent({ ts: 1000, rubricId: 'svg-gen', dimId: 'shape', sessionId: 's1', turnId: 't1', score: 0.9 }) + const lo = s.addEvent({ ts: 2000, rubricId: 'svg-gen', dimId: 'shape', sessionId: 's2', turnId: 't2', score: 0.2 }) + const boolT = s.addEvent({ ts: 3000, rubricId: 'svg-gen', dimId: 'pass', sessionId: 's3', turnId: 't3', score: true }) + const boolF = s.addEvent({ ts: 4000, rubricId: 'svg-gen', dimId: 'pass', sessionId: 's4', turnId: 't4', score: false }) + const catHi = s.addEvent({ ts: 5000, rubricId: 'svg-gen', dimId: 'quality', sessionId: 's5', turnId: 't5', score: 'good' }) + const catLo = s.addEvent({ ts: 6000, rubricId: 'svg-gen', dimId: 'quality', sessionId: 's6', turnId: 't6', score: 'bad' }) + assert.equal(hi.passed, true) + assert.equal(lo.passed, false) + assert.equal(boolT.passed, true) + assert.equal(boolF.passed, false) + assert.equal(catHi.passed, true) + assert.equal(catLo.passed, false) +}) + +test('addEvent rejects unknown rubric or dim', () => { + const s = freshStore() + s.registerRubric(SAMPLE_RUBRIC) + assert.equal(s.addEvent({ rubricId: 'nope', dimId: 'shape', score: 0.5 }), null) + assert.equal(s.addEvent({ rubricId: 'svg-gen', dimId: 'missing', score: 0.5 }), null) +}) + +test('recentScoresFor computes pass rate + per-dim breakdown', () => { + const s = freshStore() + s.registerRubric(SAMPLE_RUBRIC) + s.addEvent({ ts: 1, rubricId: 'svg-gen', dimId: 'shape', sessionId: 's', turnId: 't', score: 0.9 }) + s.addEvent({ ts: 2, rubricId: 'svg-gen', dimId: 'shape', sessionId: 's', turnId: 't', score: 0.9 }) + s.addEvent({ ts: 3, rubricId: 'svg-gen', dimId: 'shape', sessionId: 's', turnId: 't', score: 0.1 }) + s.addEvent({ ts: 4, rubricId: 'svg-gen', dimId: 'pass', sessionId: 's', turnId: 't', score: true }) + const r = s.recentScoresFor('svg-gen') + assert.equal(r.total, 4) + assert.equal(r.passRate, 0.75) // 3 of 4 passed + assert.equal(r.byDim.shape.n, 3) + assert.equal(r.byDim.shape.passRate, Math.round((2 / 3) * 1000) / 1000) + assert.equal(r.byDim.pass.n, 1) + assert.equal(r.byDim.pass.passRate, 1) + assert.equal(r.latest[0].ts, 4) +}) + +test('timeSeriesFor buckets by day and groups by dim', () => { + const s = freshStore() + s.registerRubric(SAMPLE_RUBRIC) + const t1 = new Date('2026-07-15T12:00:00Z').getTime() + const t2 = new Date('2026-07-16T12:00:00Z').getTime() + s.addEvent({ ts: t1, rubricId: 'svg-gen', dimId: 'shape', sessionId: 'a', turnId: 't', score: 0.5 }) + s.addEvent({ ts: t2, rubricId: 'svg-gen', dimId: 'shape', sessionId: 'b', turnId: 't', score: 0.9 }) + s.addEvent({ ts: t2, rubricId: 'svg-gen', dimId: 'pass', sessionId: 'b', turnId: 't', score: true }) + const ts = s.timeSeriesFor({ by: 'day', groupBy: 'dim' }) + assert.deepEqual(ts.xAxis, ['2026-07-15', '2026-07-16']) + assert.equal(ts.series.length, 2) + const shapeSeries = ts.series.find(x => x.key.endsWith('::shape')) + assert.ok(shapeSeries, 'shape series present') + assert.equal(shapeSeries.points.length, 2) + assert.equal(shapeSeries.points[0].mean01, 0.5) + assert.equal(shapeSeries.points[1].mean01, 0.9) +}) + +test('timeSeriesFor buckets by version', () => { + const s = freshStore() + s.registerRubric(SAMPLE_RUBRIC) + s.addEvent({ ts: 1, rubricId: 'svg-gen', dimId: 'shape', sessionId: 'a', turnId: 't', score: 0.3, harnessVersion: 'v0.9' }) + s.addEvent({ ts: 2, rubricId: 'svg-gen', dimId: 'shape', sessionId: 'b', turnId: 't', score: 0.7, harnessVersion: 'v0.10' }) + s.addEvent({ ts: 3, rubricId: 'svg-gen', dimId: 'shape', sessionId: 'c', turnId: 't', score: 0.9, harnessVersion: 'v0.11' }) + const ts = s.timeSeriesFor({ by: 'version', groupBy: 'dim' }) + assert.deepEqual(ts.xAxis, ['v0.10', 'v0.11', 'v0.9']) // sorted +}) + +test('rolloutGridFor produces one cell per (dim, rollout)', () => { + const s = freshStore() + s.registerRubric(SAMPLE_RUBRIC) + s.addEvent({ ts: 1, rubricId: 'svg-gen', dimId: 'shape', sessionId: 's', turnId: 't', rolloutIdx: 1, score: 0.9 }) + s.addEvent({ ts: 2, rubricId: 'svg-gen', dimId: 'shape', sessionId: 's', turnId: 't', rolloutIdx: 2, score: 0.3 }) + s.addEvent({ ts: 3, rubricId: 'svg-gen', dimId: 'pass', sessionId: 's', turnId: 't', rolloutIdx: 1, score: true }) + s.addEvent({ ts: 4, rubricId: 'svg-gen', dimId: 'pass', sessionId: 's', turnId: 't', rolloutIdx: 2, score: false }) + const grid = s.rolloutGridFor('svg-gen', 's') + assert.equal(grid.rubric.id, 'svg-gen') + assert.deepEqual(grid.rollouts, [1, 2]) + assert.equal(grid.dims.length, 3) + const cellR1Shape = grid.cells.find(c => c.dimId === 'shape' && c.rolloutIdx === 1) + assert.equal(cellR1Shape.passed, true) + const cellR2Shape = grid.cells.find(c => c.dimId === 'shape' && c.rolloutIdx === 2) + assert.equal(cellR2Shape.passed, false) +}) + +test('detectSimilarSessions filters by minCount', () => { + const s = freshStore() + s.loadFixture({ + rubrics: [SAMPLE_RUBRIC], + events: [], + similarClasses: [ + { id: 'a', signature: 'sig-a', count: 5, sessionIds: [], promptSummary: 'a' }, + { id: 'b', signature: 'sig-b', count: 2, sessionIds: [], promptSummary: 'b' }, + ], + }) + const withDefault = s.detectSimilarSessions() + assert.equal(withDefault.length, 1) + assert.equal(withDefault[0].id, 'a') + const relaxed = s.detectSimilarSessions({ minCount: 1 }) + assert.equal(relaxed.length, 2) +}) + +test('subscribe fires on every mutation', () => { + const s = freshStore() + let fires = 0 + const unsub = s.subscribe(() => { fires++ }) + s.registerRubric(SAMPLE_RUBRIC) + s.addEvent({ ts: 1, rubricId: 'svg-gen', dimId: 'shape', sessionId: 's', turnId: 't', score: 0.5 }) + unsub() + s.addEvent({ ts: 2, rubricId: 'svg-gen', dimId: 'shape', sessionId: 's', turnId: 't', score: 0.7 }) + assert.equal(fires, 2) // register + first add; second add is after unsub +}) + +test('loadFixture returns counts', () => { + const s = freshStore() + const res = s.loadFixture({ + rubrics: [SAMPLE_RUBRIC], + events: [ + { rubricId: 'svg-gen', dimId: 'shape', sessionId: 's', turnId: 't', score: 0.5 }, + { rubricId: 'nope', dimId: 'x', score: 0 }, + ], + similarClasses: [], + }) + assert.equal(res.rubrics, 1) + assert.equal(res.events, 1) +}) + +test('timeSeriesFor filter chip: harnessVersion', () => { + const s = freshStore() + s.registerRubric(SAMPLE_RUBRIC) + s.addEvent({ ts: 1, rubricId: 'svg-gen', dimId: 'shape', sessionId: 'a', turnId: 't', score: 0.3, harnessVersion: 'v0.9' }) + s.addEvent({ ts: 2, rubricId: 'svg-gen', dimId: 'shape', sessionId: 'b', turnId: 't', score: 0.9, harnessVersion: 'v0.11' }) + const ts = s.timeSeriesFor({ by: 'day', groupBy: 'dim', filter: { harnessVersion: 'v0.11' } }) + const shape = ts.series.find(x => x.key.endsWith('::shape')) + assert.ok(shape) + assert.equal(shape.points.length, 1) + assert.equal(shape.points[0].mean01, 0.9) +}) diff --git a/examples/desktop/test/rubric-fusion-views.test.js b/examples/desktop/test/rubric-fusion-views.test.js new file mode 100644 index 0000000000..fe8b1694ee --- /dev/null +++ b/examples/desktop/test/rubric-fusion-views.test.js @@ -0,0 +1,189 @@ +// Rubric fusion — cross-view smoke tests. +// +// We can't run the actual DOM controllers under `node --test` (no jsdom), +// but we can: +// 1. Verify the fusion seed JSON parses and drives all three view APIs. +// 2. Verify each of the 3 view scripts loads cleanly with a minimal +// document stub (catching syntax errors early). +// 3. Verify the fusion-model events → view derivations pipeline +// returns the expected shape for each view. + +'use strict' + +const test = require('node:test') +const assert = require('node:assert/strict') +const path = require('node:path') +const fs = require('node:fs') + +// Wire the rubrics-model global before anything requires fusion-model. +global.window = global.window || {} +global.window.__dshRubricsModel = require('../src/renderer/rubrics-model.js') + +// Fresh singleton per test. +function freshFusion() { + delete require.cache[require.resolve('../src/renderer/rubric-fusion-model.js')] + const api = require('../src/renderer/rubric-fusion-model.js') + return api.create() +} + +const FIXTURE_PATH = path.join(__dirname, '..', 'docs', 'rubric-fusion-fixture.json') +const FIXTURE = JSON.parse(fs.readFileSync(FIXTURE_PATH, 'utf8')) + +test('fixture loads with 3 rubrics and > 30 events', () => { + const s = freshFusion() + const res = s.loadFixture(FIXTURE) + assert.equal(res.rubrics, 3) + assert.ok(res.events > 30, 'expected > 30 events, got ' + res.events) +}) + +test('Rubrics view: recentScoresFor returns per-dim breakdown for each rubric', () => { + const s = freshFusion() + s.loadFixture(FIXTURE) + for (const rubric of s.listRubrics()) { + const stats = s.recentScoresFor(rubric.id) + assert.ok(stats.total > 0, `${rubric.id}: expected events > 0`) + assert.ok(stats.passRate >= 0 && stats.passRate <= 1, `${rubric.id}: passRate out of range`) + for (const dim of rubric.dims) { + const byDim = stats.byDim[dim.id] + assert.ok(byDim, `${rubric.id}/${dim.id}: missing per-dim bucket`) + assert.ok(byDim.n > 0, `${rubric.id}/${dim.id}: expected n > 0`) + } + } +}) + +test('Growth view: timeSeriesFor by=day produces sorted xAxis and per-dim series', () => { + const s = freshFusion() + s.loadFixture(FIXTURE) + const ts = s.timeSeriesFor({ by: 'day', groupBy: 'dim' }) + assert.ok(ts.xAxis.length >= 3, 'expected >= 3 days') + // sorted asc + const sorted = ts.xAxis.slice().sort() + assert.deepEqual(ts.xAxis, sorted) + assert.ok(ts.series.length >= 3, 'expected >= 3 dim series') + for (const s2 of ts.series) { + assert.ok(s2.points.length >= 1, 'each series has points: ' + s2.label) + for (const p of s2.points) { + assert.ok(p.mean01 >= 0 && p.mean01 <= 1, 'mean01 in range for ' + s2.label) + assert.ok(p.passRate >= 0 && p.passRate <= 1, 'passRate in range for ' + s2.label) + } + } +}) + +test('Growth view: timeSeriesFor by=version buckets by harness version', () => { + const s = freshFusion() + s.loadFixture(FIXTURE) + const ts = s.timeSeriesFor({ by: 'version', groupBy: 'dim' }) + const versions = new Set(ts.xAxis) + assert.ok(versions.has('v0.9'), 'v0.9 present') + assert.ok(versions.has('v0.11'), 'v0.11 present') +}) + +test('Growth view: filter chip harnessVersion narrows the series', () => { + const s = freshFusion() + s.loadFixture(FIXTURE) + const all = s.timeSeriesFor({ by: 'day', groupBy: 'dim' }) + const v11 = s.timeSeriesFor({ by: 'day', groupBy: 'dim', filter: { harnessVersion: 'v0.11' } }) + const totalAll = all.series.reduce((sum, sr) => sum + sr.points.reduce((a, p) => a + p.n, 0), 0) + const totalV11 = v11.series.reduce((sum, sr) => sum + sr.points.reduce((a, p) => a + p.n, 0), 0) + assert.ok(totalV11 < totalAll, 'filtered total should be strictly smaller') + assert.ok(totalV11 > 0, 'v0.11 filtered total > 0') +}) + +test('Runtime view: rolloutGridFor produces a matrix with cell pass/fail', () => { + const s = freshFusion() + s.loadFixture(FIXTURE) + const grid = s.rolloutGridFor('svg-gen', 's-svg-live') + assert.equal(grid.rubric.id, 'svg-gen') + assert.ok(grid.rollouts.length >= 3, 'expected >= 3 rollouts for the live session') + assert.ok(grid.dims.length === 3, 'svg-gen has 3 dims') + // Assert we have at least one pass AND at least one fail — the fixture + // deliberately spans both. + const passed = grid.cells.filter(c => c.passed === true).length + const failed = grid.cells.filter(c => c.passed === false).length + assert.ok(passed > 0, 'at least one pass cell') + assert.ok(failed > 0, 'at least one fail cell') +}) + +test('Runtime view: rolloutGridFor sessionId=null aggregates all rollouts', () => { + const s = freshFusion() + s.loadFixture(FIXTURE) + const grid = s.rolloutGridFor('svg-gen', null) + // The fixture has rollouts 1..8 for svg-gen (5 seed sessions + 3 on + // the live session). + assert.deepEqual(grid.rollouts, [1, 2, 3, 4, 5, 6, 7, 8]) +}) + +test('Rubrics view: similar-sessions hint fires with count >= 3', () => { + const s = freshFusion() + s.loadFixture(FIXTURE) + const classes = s.detectSimilarSessions() + assert.equal(classes.length, 1) + assert.equal(classes[0].id, 'similar-svg-gen') + assert.ok(classes[0].count >= 3) +}) + +// --- Script-load smoke tests: the 3 view scripts must load without +// throwing under a minimal document stub. Catches syntax errors before +// they hit the browser. --- + +function stubDocument() { + return { + addEventListener() {}, + readyState: 'complete', + querySelector() { return null }, + querySelectorAll() { return [] }, + createElement() { + const node = { + style: {}, + classList: { add() {}, remove() {}, toggle() {}, contains() { return false } }, + dataset: {}, + setAttribute() {}, + getAttribute() { return null }, + appendChild(c) { return c }, + replaceChildren() {}, + removeChild() {}, + addEventListener() {}, + set textContent(v) {}, + set innerHTML(v) {}, + set hidden(v) {}, + } + return node + }, + createElementNS() { return this.createElement() }, + getElementById() { return null }, + } +} + +test('rubrics-page.js loads under minimal document stub', () => { + global.window = { __dshRubricsModel: require('../src/renderer/rubrics-model.js') } + global.document = stubDocument() + global.requestAnimationFrame = () => {} + delete require.cache[require.resolve('../src/renderer/rubrics-page.js')] + const page = require('../src/renderer/rubrics-page.js') + assert.ok(page._internal, 'exposes _internal') +}) + +test('growth-v2.js loads under minimal document stub', () => { + global.window = { + __dshRubricsModel: require('../src/renderer/rubrics-model.js'), + __dshRubricFusion: require('../src/renderer/rubric-fusion-model.js'), + } + global.document = stubDocument() + delete require.cache[require.resolve('../src/renderer/growth-v2.js')] + require('../src/renderer/growth-v2.js') + assert.ok(global.window.__dshGrowthV2, 'exposes __dshGrowthV2') + assert.equal(typeof global.window.__dshGrowthV2.show, 'function') + assert.equal(typeof global.window.__dshGrowthV2.render, 'function') +}) + +test('runtimes-page.js loads under minimal document stub', () => { + global.window = { + __dshRubricsModel: require('../src/renderer/rubrics-model.js'), + __dshRubricFusion: require('../src/renderer/rubric-fusion-model.js'), + } + global.document = stubDocument() + delete require.cache[require.resolve('../src/renderer/runtimes-page.js')] + require('../src/renderer/runtimes-page.js') + assert.ok(global.window.__dshRuntimes, 'exposes __dshRuntimes') + assert.equal(typeof global.window.__dshRuntimes.show, 'function') +})