feat(desktop): trace signal detection + tri-view badges
Adds a small signal-detection layer over trace payloads and surfaces
three kinds of semantic signals inline in the trace tri-view (Timeline
/ Graph / turn footer chips) so a researcher can spot repetitive
tool-call loops, redundant identical calls, and plan-completion beats
without hand-scanning the raw log:
- loop-detected — three or more repeats of the same tool.method
signature inside a window, badge in Timeline gutter
+ colored ring on the Graph node
- redundant-call — identical (tool, arg-hash) call twice without any
state change in between, badge + subdued styling
so it stays advisory not alarmist
- plan-* chips — plan/subplan step-complete beats surfaced above the
assistant body in the turn footer, wired through
the finishTurnContainer tail
Pure rendering; no wire-format changes needed on the runtime side —
signal detection is fixture-driven off the same payload the tri-view
already consumes. Companion RFC L-2 in docs/upstream-ledger.md asks
the runtime to emit these signals natively rather than deriving them
in the shell, so once L-2 lands the shell will consume upstream signals
and this detector becomes a fallback.
Files:
src/renderer/trace-signal-detect.js new (loop/redundant/plan)
src/renderer/trace-tri-view.js +24 (wire detector output)
src/renderer/trace-timeline.js +39 (badge in gutter)
src/renderer/trace-graph.js +30 (colored ring)
src/renderer/renderer.js +81 (finishTurnContainer chips)
src/renderer/index.html +1 (one script tag)
src/renderer/style.css +58 (badge/ring/chip rules)
docs/upstream-ledger.md +121 (append L-2 RFC)
scripts/qa-trace-signals-fixture.mjs new (headless SVG proof)
scripts/qa-trace-signals-shoot.mjs new (CDP live shoot)
docs/trace-signals-shoot/*.html new (3 fixture-driven shots)
test/trace-signal-detect.test.js new (detector unit tests)
test/trace-signal-overlay.test.js new (renderer overlay tests)
Test suite: 1668/1668 pass (17 new). Fixture-driven signals-01-timeline-
loop.html regenerated byte-identical from the merged HEAD.
This commit is contained in:
ZiyaZhang
@@ -0,0 +1,50 @@
|
||||
<!doctype html>
|
||||
<html><head><meta charset="utf-8"><title>Signals 01 — Timeline: loop-detected badge</title>
|
||||
<style>
|
||||
body { font: 13px system-ui, sans-serif; margin: 24px; background: #f9fafb; color: #1d1d1f; }
|
||||
h1 { font-size: 15px; margin: 0 0 8px 0; }
|
||||
.note { color: #6b6b70; font-size: 12px; margin: 0 0 16px 0; max-width: 720px; line-height: 1.4; }
|
||||
.stage { background: #fff; padding: 16px; border: 1px solid #e5e5ea; border-radius: 8px; overflow-x: auto; }
|
||||
/* Palette matches src/renderer/style.css tail block for signal badges/rings/chips. */
|
||||
.trace-timeline-svg { font: 11px system-ui, sans-serif; }
|
||||
.trace-timeline-tick { stroke: #d0d0d5; stroke-width: 0.5; stroke-dasharray: 2 3; }
|
||||
.trace-timeline-tick-label { fill: #6b6b70; font-size: 10px; }
|
||||
.trace-timeline-label { fill: #1d1d1f; font-size: 11px; }
|
||||
.trace-timeline-bar.family-step { fill: #b0b4bd; }
|
||||
.trace-timeline-bar.family-llm { fill: #7c3aed; }
|
||||
.trace-timeline-bar.family-tool { fill: #b45309; }
|
||||
.trace-timeline-bar.family-input { fill: #6b6b70; }
|
||||
.trace-timeline-signal-badge { stroke: rgba(0,0,0,0.15); stroke-width: 0.75; }
|
||||
.trace-timeline-signal-badge.sig-error { fill: #dc2626; }
|
||||
.trace-timeline-signal-badge.sig-loop { fill: #ef4444; }
|
||||
.trace-timeline-signal-badge.sig-redundant{ fill: #f59e0b; }
|
||||
.trace-timeline-signal-badge.sig-plan { fill: #2563eb; }
|
||||
.trace-graph-svg { background: #fff; }
|
||||
.trace-graph-node-body { fill: #f3f4f6; stroke: #9ca3af; stroke-width: 1; }
|
||||
.trace-graph-node-body.family-tool { fill: #fde68a; stroke: #b45309; }
|
||||
.trace-graph-node-body.family-step { fill: #e5e7eb; stroke: #6b7280; }
|
||||
.trace-graph-node-body.family-llm { fill: #ddd6fe; stroke: #7c3aed; }
|
||||
.trace-graph-node-glyph { font: 12px system-ui, sans-serif; fill: #1d1d1f; }
|
||||
.trace-graph-node-label { font: 10px system-ui, sans-serif; fill: #1d1d1f; }
|
||||
.trace-graph-edge { stroke: #9ca3af; stroke-width: 1.2; }
|
||||
.trace-graph-signal-ring { stroke-width: 2.5; fill: none; }
|
||||
.trace-graph-signal-ring.sig-error { stroke: #dc2626; }
|
||||
.trace-graph-signal-ring.sig-loop { stroke: #ef4444; }
|
||||
.trace-graph-signal-ring.sig-plan { stroke: #2563eb; }
|
||||
.assistant-turn { border-left: 3px solid #d0d0d5; padding: 12px; margin: 12px 0; background: #fff; border-radius: 6px; }
|
||||
.turn-body { display: flex; flex-direction: column; gap: 6px; }
|
||||
.text-block { font-size: 13px; color: #1d1d1f; }
|
||||
.tool-row { font-family: ui-monospace, monospace; font-size: 12px; color: #4b5563; }
|
||||
.turn-glyph { display: inline-block; width: 14px; }
|
||||
.turn-signal-chip-row { display: flex; flex-wrap: wrap; gap: 6px; padding: 4px 0 6px 0; align-items: center; }
|
||||
.turn-signal-chip { font-size: 11px; line-height: 1; padding: 3px 8px; border-radius: 10px; border: 1px solid transparent; background: rgba(0,0,0,0.03); color: #1d1d1f; cursor: pointer; font-family: inherit; }
|
||||
.turn-signal-chip::before { content: ''; display: inline-block; width: 6px; height: 6px; border-radius: 50%; margin-right: 6px; vertical-align: 1px; background: currentColor; }
|
||||
.turn-signal-chip.sig-error, .turn-signal-chip.sig-loop { color: #b91c1c; border-color: rgba(239,68,68,0.35); background: rgba(239,68,68,0.06); }
|
||||
.turn-signal-chip.sig-redundant { color: #92400e; border-color: rgba(245,158,11,0.4); background: rgba(245,158,11,0.08); }
|
||||
.turn-signal-chip.sig-plan { color: #1d4ed8; border-color: rgba(37,99,235,0.35); background: rgba(37,99,235,0.06); }
|
||||
</style>
|
||||
</head><body>
|
||||
<h1>Signals 01 — Timeline: loop-detected badge</h1>
|
||||
<p class="note">Three consecutive <code>fs.read</code> calls with identical args. The detector fires <code>loop-detected</code> on the third call; its badge sits in the left gutter, colored red. Signals detected: <code>loop-detected</code>. See docs/upstream-ledger.md L-2.</p>
|
||||
<div class="stage"><div data-total-ms="500" class="trace-timeline"><svg viewBox="0 0 820 110" width="820" height="110" role="img" aria-label="Timeline of 4 rows over 500ms" class="trace-timeline-svg"><g class="trace-timeline-axis"><line x1="220" y1="22" x2="220" y2="106" class="trace-timeline-tick"></line><text x="223" y="12" class="trace-timeline-tick-label">0ms</text><line x1="367" y1="22" x2="367" y2="106" class="trace-timeline-tick"></line><text x="370" y="12" class="trace-timeline-tick-label">125ms</text><line x1="514" y1="22" x2="514" y2="106" class="trace-timeline-tick"></line><text x="517" y="12" class="trace-timeline-tick-label">250ms</text><line x1="661" y1="22" x2="661" y2="106" class="trace-timeline-tick"></line><text x="664" y="12" class="trace-timeline-tick-label">375ms</text><line x1="808" y1="22" x2="808" y2="106" class="trace-timeline-tick"></line><text x="811" y="12" class="trace-timeline-tick-label">500ms</text></g><g data-seq="10" data-kind="step" class="trace-timeline-row family-step depth-0"><text x="8" y="38.96" class="trace-timeline-label"><title>step 1.0 — read main.ts · 500ms</title></text><rect x="220" y="30" width="588" height="10" rx="2" ry="2" class="trace-timeline-bar family-step kind-step"><title>step 1.0 — read main.ts · 500ms · 500ms</title></rect></g><g data-seq="11" data-kind="event" class="trace-timeline-row family-tool depth-1"><text x="20" y="56.96" class="trace-timeline-label"><title>tool: fs.read</title></text><rect x="278.8" y="48" width="35.27999999999997" height="10" rx="2" ry="2" class="trace-timeline-bar family-tool kind-event"><title>tool: fs.read · 30ms</title></rect></g><g data-seq="12" data-kind="event" class="trace-timeline-row family-tool depth-1"><text x="20" y="74.96" class="trace-timeline-label"><title>tool: fs.read</title></text><rect x="396.4" y="66" width="35.27999999999997" height="10" rx="2" ry="2" class="trace-timeline-bar family-tool kind-event"><title>tool: fs.read · 30ms</title></rect></g><g data-seq="13" data-kind="event" class="trace-timeline-row family-tool depth-1"><text x="20" y="92.96" class="trace-timeline-label"><title>tool: fs.read</title></text><rect x="514" y="84" width="35.27999999999997" height="10" rx="2" ry="2" class="trace-timeline-bar family-tool kind-event"><title>tool: fs.read · 30ms</title></rect></g><g class="trace-timeline-badges"><circle cx="210" cy="89" r="4" class="trace-timeline-signal-badge sig-loop"><title>Loop detected · 3 × fs.read · prior seq 11, 12 · (heuristic)</title></circle></g></svg></div></div>
|
||||
</body></html>
|
||||
@@ -0,0 +1,50 @@
|
||||
<!doctype html>
|
||||
<html><head><meta charset="utf-8"><title>Signals 02 — Graph: tool-error ring</title>
|
||||
<style>
|
||||
body { font: 13px system-ui, sans-serif; margin: 24px; background: #f9fafb; color: #1d1d1f; }
|
||||
h1 { font-size: 15px; margin: 0 0 8px 0; }
|
||||
.note { color: #6b6b70; font-size: 12px; margin: 0 0 16px 0; max-width: 720px; line-height: 1.4; }
|
||||
.stage { background: #fff; padding: 16px; border: 1px solid #e5e5ea; border-radius: 8px; overflow-x: auto; }
|
||||
/* Palette matches src/renderer/style.css tail block for signal badges/rings/chips. */
|
||||
.trace-timeline-svg { font: 11px system-ui, sans-serif; }
|
||||
.trace-timeline-tick { stroke: #d0d0d5; stroke-width: 0.5; stroke-dasharray: 2 3; }
|
||||
.trace-timeline-tick-label { fill: #6b6b70; font-size: 10px; }
|
||||
.trace-timeline-label { fill: #1d1d1f; font-size: 11px; }
|
||||
.trace-timeline-bar.family-step { fill: #b0b4bd; }
|
||||
.trace-timeline-bar.family-llm { fill: #7c3aed; }
|
||||
.trace-timeline-bar.family-tool { fill: #b45309; }
|
||||
.trace-timeline-bar.family-input { fill: #6b6b70; }
|
||||
.trace-timeline-signal-badge { stroke: rgba(0,0,0,0.15); stroke-width: 0.75; }
|
||||
.trace-timeline-signal-badge.sig-error { fill: #dc2626; }
|
||||
.trace-timeline-signal-badge.sig-loop { fill: #ef4444; }
|
||||
.trace-timeline-signal-badge.sig-redundant{ fill: #f59e0b; }
|
||||
.trace-timeline-signal-badge.sig-plan { fill: #2563eb; }
|
||||
.trace-graph-svg { background: #fff; }
|
||||
.trace-graph-node-body { fill: #f3f4f6; stroke: #9ca3af; stroke-width: 1; }
|
||||
.trace-graph-node-body.family-tool { fill: #fde68a; stroke: #b45309; }
|
||||
.trace-graph-node-body.family-step { fill: #e5e7eb; stroke: #6b7280; }
|
||||
.trace-graph-node-body.family-llm { fill: #ddd6fe; stroke: #7c3aed; }
|
||||
.trace-graph-node-glyph { font: 12px system-ui, sans-serif; fill: #1d1d1f; }
|
||||
.trace-graph-node-label { font: 10px system-ui, sans-serif; fill: #1d1d1f; }
|
||||
.trace-graph-edge { stroke: #9ca3af; stroke-width: 1.2; }
|
||||
.trace-graph-signal-ring { stroke-width: 2.5; fill: none; }
|
||||
.trace-graph-signal-ring.sig-error { stroke: #dc2626; }
|
||||
.trace-graph-signal-ring.sig-loop { stroke: #ef4444; }
|
||||
.trace-graph-signal-ring.sig-plan { stroke: #2563eb; }
|
||||
.assistant-turn { border-left: 3px solid #d0d0d5; padding: 12px; margin: 12px 0; background: #fff; border-radius: 6px; }
|
||||
.turn-body { display: flex; flex-direction: column; gap: 6px; }
|
||||
.text-block { font-size: 13px; color: #1d1d1f; }
|
||||
.tool-row { font-family: ui-monospace, monospace; font-size: 12px; color: #4b5563; }
|
||||
.turn-glyph { display: inline-block; width: 14px; }
|
||||
.turn-signal-chip-row { display: flex; flex-wrap: wrap; gap: 6px; padding: 4px 0 6px 0; align-items: center; }
|
||||
.turn-signal-chip { font-size: 11px; line-height: 1; padding: 3px 8px; border-radius: 10px; border: 1px solid transparent; background: rgba(0,0,0,0.03); color: #1d1d1f; cursor: pointer; font-family: inherit; }
|
||||
.turn-signal-chip::before { content: ''; display: inline-block; width: 6px; height: 6px; border-radius: 50%; margin-right: 6px; vertical-align: 1px; background: currentColor; }
|
||||
.turn-signal-chip.sig-error, .turn-signal-chip.sig-loop { color: #b91c1c; border-color: rgba(239,68,68,0.35); background: rgba(239,68,68,0.06); }
|
||||
.turn-signal-chip.sig-redundant { color: #92400e; border-color: rgba(245,158,11,0.4); background: rgba(245,158,11,0.08); }
|
||||
.turn-signal-chip.sig-plan { color: #1d4ed8; border-color: rgba(37,99,235,0.35); background: rgba(37,99,235,0.06); }
|
||||
</style>
|
||||
</head><body>
|
||||
<h1>Signals 02 — Graph: tool-error ring</h1>
|
||||
<p class="note">A failing <code>bash</code> call followed by a successful retry. The detector emits <code>tool-error</code> on the failing call's seq and <code>plan-restart</code> on the retry. The Graph node for the failing call gets a red outer ring; the retry node gets a blue ring. Signals detected: <code>tool-error, tool-error, plan-restart</code>.</p>
|
||||
<div class="stage"><div data-layout-mode="bfs-layered-fallback" class="trace-graph"><svg viewBox="0 0 494 200" width="494" height="200" role="img" aria-label="Agent graph: 3 nodes, 2 edges" class="trace-graph-svg"><g class="trace-graph-edges"><path d="M44,44 C107,44 121,44 184,44" fill="none" class="trace-graph-edge kind-seq"></path><path d="M184,44 C247,44 261,44 324,44" fill="none" class="trace-graph-edge kind-seq"></path></g><g class="trace-graph-nodes"><g data-seq="10" transform="translate(44, 44)" class="trace-graph-node family-step status-ok"><circle cx="0" cy="0" r="14" class="trace-graph-node-body family-step"></circle><text x="0" y="4" text-anchor="middle" class="trace-graph-node-glyph">●</text><text x="0" y="28" text-anchor="middle" class="trace-graph-node-label">step 1.0</text><title>step 1.0 · step · 500ms · seq 10</title></g><g data-seq="11" transform="translate(184, 44)" class="trace-graph-node family-tool status-ok"><circle cx="0" cy="0" r="14" class="trace-graph-node-body family-tool"></circle><circle cx="0" cy="0" r="18" fill="none" class="trace-graph-signal-ring sig-error"><title>Tool error · bash: ENOENT</title></circle><text x="0" y="4" text-anchor="middle" class="trace-graph-node-glyph">▲</text><text x="0" y="28" text-anchor="middle" class="trace-graph-node-label">bash</text><title>bash · tool · 30ms · seq 11</title></g><g data-seq="13" transform="translate(324, 44)" class="trace-graph-node family-tool status-ok"><circle cx="0" cy="0" r="14" class="trace-graph-node-body family-tool"></circle><circle cx="0" cy="0" r="18" fill="none" class="trace-graph-signal-ring sig-plan-restart"><title>Plan restart · retry bash after seq 12 · (heuristic)</title></circle><text x="0" y="4" text-anchor="middle" class="trace-graph-node-glyph">▲</text><text x="0" y="28" text-anchor="middle" class="trace-graph-node-label">bash</text><title>bash · tool · 30ms · seq 13</title></g></g></svg></div></div>
|
||||
</body></html>
|
||||
@@ -0,0 +1,63 @@
|
||||
<!doctype html>
|
||||
<html><head><meta charset="utf-8"><title>Signals 03 — Turn container: signal chip row</title>
|
||||
<style>
|
||||
body { font: 13px system-ui, sans-serif; margin: 24px; background: #f9fafb; color: #1d1d1f; }
|
||||
h1 { font-size: 15px; margin: 0 0 8px 0; }
|
||||
.note { color: #6b6b70; font-size: 12px; margin: 0 0 16px 0; max-width: 720px; line-height: 1.4; }
|
||||
.stage { background: #fff; padding: 16px; border: 1px solid #e5e5ea; border-radius: 8px; overflow-x: auto; }
|
||||
/* Palette matches src/renderer/style.css tail block for signal badges/rings/chips. */
|
||||
.trace-timeline-svg { font: 11px system-ui, sans-serif; }
|
||||
.trace-timeline-tick { stroke: #d0d0d5; stroke-width: 0.5; stroke-dasharray: 2 3; }
|
||||
.trace-timeline-tick-label { fill: #6b6b70; font-size: 10px; }
|
||||
.trace-timeline-label { fill: #1d1d1f; font-size: 11px; }
|
||||
.trace-timeline-bar.family-step { fill: #b0b4bd; }
|
||||
.trace-timeline-bar.family-llm { fill: #7c3aed; }
|
||||
.trace-timeline-bar.family-tool { fill: #b45309; }
|
||||
.trace-timeline-bar.family-input { fill: #6b6b70; }
|
||||
.trace-timeline-signal-badge { stroke: rgba(0,0,0,0.15); stroke-width: 0.75; }
|
||||
.trace-timeline-signal-badge.sig-error { fill: #dc2626; }
|
||||
.trace-timeline-signal-badge.sig-loop { fill: #ef4444; }
|
||||
.trace-timeline-signal-badge.sig-redundant{ fill: #f59e0b; }
|
||||
.trace-timeline-signal-badge.sig-plan { fill: #2563eb; }
|
||||
.trace-graph-svg { background: #fff; }
|
||||
.trace-graph-node-body { fill: #f3f4f6; stroke: #9ca3af; stroke-width: 1; }
|
||||
.trace-graph-node-body.family-tool { fill: #fde68a; stroke: #b45309; }
|
||||
.trace-graph-node-body.family-step { fill: #e5e7eb; stroke: #6b7280; }
|
||||
.trace-graph-node-body.family-llm { fill: #ddd6fe; stroke: #7c3aed; }
|
||||
.trace-graph-node-glyph { font: 12px system-ui, sans-serif; fill: #1d1d1f; }
|
||||
.trace-graph-node-label { font: 10px system-ui, sans-serif; fill: #1d1d1f; }
|
||||
.trace-graph-edge { stroke: #9ca3af; stroke-width: 1.2; }
|
||||
.trace-graph-signal-ring { stroke-width: 2.5; fill: none; }
|
||||
.trace-graph-signal-ring.sig-error { stroke: #dc2626; }
|
||||
.trace-graph-signal-ring.sig-loop { stroke: #ef4444; }
|
||||
.trace-graph-signal-ring.sig-plan { stroke: #2563eb; }
|
||||
.assistant-turn { border-left: 3px solid #d0d0d5; padding: 12px; margin: 12px 0; background: #fff; border-radius: 6px; }
|
||||
.turn-body { display: flex; flex-direction: column; gap: 6px; }
|
||||
.text-block { font-size: 13px; color: #1d1d1f; }
|
||||
.tool-row { font-family: ui-monospace, monospace; font-size: 12px; color: #4b5563; }
|
||||
.turn-glyph { display: inline-block; width: 14px; }
|
||||
.turn-signal-chip-row { display: flex; flex-wrap: wrap; gap: 6px; padding: 4px 0 6px 0; align-items: center; }
|
||||
.turn-signal-chip { font-size: 11px; line-height: 1; padding: 3px 8px; border-radius: 10px; border: 1px solid transparent; background: rgba(0,0,0,0.03); color: #1d1d1f; cursor: pointer; font-family: inherit; }
|
||||
.turn-signal-chip::before { content: ''; display: inline-block; width: 6px; height: 6px; border-radius: 50%; margin-right: 6px; vertical-align: 1px; background: currentColor; }
|
||||
.turn-signal-chip.sig-error, .turn-signal-chip.sig-loop { color: #b91c1c; border-color: rgba(239,68,68,0.35); background: rgba(239,68,68,0.06); }
|
||||
.turn-signal-chip.sig-redundant { color: #92400e; border-color: rgba(245,158,11,0.4); background: rgba(245,158,11,0.08); }
|
||||
.turn-signal-chip.sig-plan { color: #1d4ed8; border-color: rgba(37,99,235,0.35); background: rgba(37,99,235,0.06); }
|
||||
</style>
|
||||
</head><body>
|
||||
<h1>Signals 03 — Turn container: signal chip row</h1>
|
||||
<p class="note">The chip row sits above the assistant body. Each chip covers one signal kind detected in this turn (loop / redundant / plan-update). Clicking a chip auto-opens the trace drawer for a drill-in. Signals detected in this turn: <code>plan-update, loop-detected, redundant-call</code>.</p>
|
||||
<div class="stage">
|
||||
<section class="assistant-turn">
|
||||
<div class="turn-body">
|
||||
<div class="turn-signal-chip-row"><button class="turn-signal-chip sig-plan" title="Plan update · “Here is the new plan: 1. read main.ts
|
||||
2. edit imports
|
||||
3. verify.” · (heuristic)">Plan update</button><button class="turn-signal-chip sig-loop" title="Loop detected · 3 × fs.read · prior seq 3, 4 · (heuristic)">Loop detected</button><button class="turn-signal-chip sig-redundant" title="Redundant call · fs.read, matches seq 5 · (heuristic)">Redundant call</button></div>
|
||||
<div class="text-block">Here is the new plan: 1. read main.ts, 2. edit imports, 3. verify.</div>
|
||||
<div class="tool-row"><span class="turn-glyph">▸</span>fs.read(main.ts)</div>
|
||||
<div class="tool-row"><span class="turn-glyph">▸</span>fs.read(main.ts)</div>
|
||||
<div class="tool-row"><span class="turn-glyph">▸</span>fs.read(main.ts)</div>
|
||||
<div class="tool-row"><span class="turn-glyph">▸</span>bash(ls)</div>
|
||||
<div class="tool-row"><span class="turn-glyph">▸</span>fs.read(main.ts)</div>
|
||||
</div>
|
||||
</section></div>
|
||||
</body></html>
|
||||
@@ -79,3 +79,124 @@ so there's no coupling between land order.
|
||||
Future entries append below. Keep the numbering monotone (L-2, L-3…) so a
|
||||
cross-repo reference like "see upstream ledger L-1" stays stable.
|
||||
-->
|
||||
|
||||
## L-2 Runtime should emit semantic trace signals (loop / redundant / plan-*)
|
||||
|
||||
**Symptom.** A researcher watching a session can't see at a glance where
|
||||
the interesting things happened: the agent got stuck in a tool-call loop,
|
||||
called the same tool with the same args twice within a few turns,
|
||||
mid-turn rewrote its plan, or restarted after a tool error. These are the
|
||||
first four things a debugger wants highlighted, and today the trace tri-view
|
||||
+ main assistant flow show every step at equal visual weight. The Tree
|
||||
column has an ✗ glyph for tool errors — that's the only pre-existing signal.
|
||||
|
||||
**Root cause.** The runtime wire has no dedicated "signal" event type. All
|
||||
diagnostic annotations that today's UI could show — loop detection,
|
||||
redundant-call detection, plan updates, plan restarts, ordinary tool errors
|
||||
raised to signal status — are inferable from the flat event stream but not
|
||||
themselves emitted. Concretely:
|
||||
|
||||
- `packages/core/agent-loop/src/loop.ts` — the loop sees every tool/call
|
||||
and every tool/result but never emits a `trace/signal` derived from them.
|
||||
- `packages/core/tools/src/index.ts` — tool descriptors don't declare
|
||||
loop/retry guardrails that could feed a signal emission.
|
||||
- `packages/core/planner/` (or the equivalent — grep for `plan` in
|
||||
`deepseek-harness-dev/packages/core/`) — plan updates are internal state,
|
||||
not observable on the wire.
|
||||
|
||||
The desktop renderer therefore has no signal to render.
|
||||
|
||||
**Local workaround.** New `src/renderer/trace-signal-detect.js` is a
|
||||
heuristic detector run over `meta.cachedEvents`. It emits five signal kinds:
|
||||
|
||||
- `loop-detected` — ≥ N consecutive same-tool + same-args-prefix calls (N=3)
|
||||
- `redundant-call` — same (name, args-prefix) reappearing within an 8-seq
|
||||
window with at least one different call in between
|
||||
- `plan-update` — assistant/message text matching "new/revised/updated
|
||||
plan", "here's the plan", or a two-line numbered-list intro
|
||||
- `plan-restart` — same tool re-invoked after a `tool/result` `ok:false`
|
||||
- `tool-error` — surfaced from the already-visible `ok:false` result, but
|
||||
also stamped on the matching call seq so the Graph node (which absorbs
|
||||
the result into the call) has a place to hang the badge
|
||||
|
||||
Signals are then rendered as:
|
||||
|
||||
- Timeline bars: colored dots to the left of each affected row
|
||||
(`.trace-timeline-signal-badge` in `style.css`)
|
||||
- Graph nodes: outer ring around the node
|
||||
(`.trace-graph-signal-ring`, highest-priority signal wins the color)
|
||||
- Assistant turn container: a chip row above the body
|
||||
(`.turn-signal-chip-row` / `.turn-signal-chip`), one chip per signal
|
||||
kind observed in that turn, clicking a chip auto-opens the trace drawer
|
||||
|
||||
The detector already special-cases wire-emitted signals: any event whose
|
||||
`type === 'trace/signal'` is passed through verbatim (marked `source: 'wire'`)
|
||||
and the heuristic scan skips seqs already covered by wire signals. This
|
||||
means the shape is forward-compatible: once upstream lands the fix, the
|
||||
runtime's signals win and the heuristic scan becomes dead code without
|
||||
requiring a renderer change.
|
||||
|
||||
Spot the workaround in code review by:
|
||||
|
||||
- `src/renderer/trace-signal-detect.js` — the entire file
|
||||
- `src/renderer/trace-timeline.js` — the `options.signals` badge loop
|
||||
(around the "Signal badges" comment)
|
||||
- `src/renderer/trace-graph.js` — the "Signal ring" block inside
|
||||
`renderGraph`'s node loop
|
||||
- `src/renderer/trace-tri-view.js` — the `_computeSignals(records)` call
|
||||
passing a `bySeq` map to both Timeline and Graph
|
||||
- `src/renderer/renderer.js` — `applyTurnSignalChips(sessionId, section)`
|
||||
called at the end of `finishTurnContainer`
|
||||
- `src/renderer/style.css` — the `.trace-timeline-signal-badge` /
|
||||
`.trace-graph-signal-ring` / `.turn-signal-chip*` blocks at the tail
|
||||
|
||||
**Upstream fix (needed).** Emit `trace/signal` events from the runtime with
|
||||
this shape:
|
||||
|
||||
```
|
||||
{
|
||||
type: 'trace/signal',
|
||||
seq: <number>, // seq the signal decorates (often the tool/call seq)
|
||||
time: <number>,
|
||||
data: {
|
||||
signal: 'loop-detected'|'redundant-call'|'plan-update'|'plan-restart'|'tool-error',
|
||||
// signal-specific fields; the renderer uses these for tooltip content:
|
||||
name?: string, // for tool-family signals: which tool
|
||||
argsKey?: string, // 80-char args prefix
|
||||
run?: number, // loop-detected: number of consecutive matches
|
||||
priorSeqs?: number[], // loop-detected: earlier calls in the run
|
||||
priorSeq?: number, // redundant-call / plan-restart pointer
|
||||
priorErrorSeq?: number,
|
||||
snippet?: string, // plan-update: first 80 chars of the plan text
|
||||
error?: string, // tool-error
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Suggested seams (grep upstream to confirm exact `packages/…` paths — the
|
||||
comments below cite the same file family as L-1):
|
||||
|
||||
1. **Loop/redundant detection** — add a small ring buffer of recent
|
||||
`tool/call` (name, argsKey) pairs inside `agent-loop`'s emit path. When
|
||||
the buffer trips the threshold, emit `trace/signal` before the offending
|
||||
`tool/call` reaches the wire so the signal precedes the call in seq order.
|
||||
Threshold defaults (loopN=3, window=8) can be config-flagged.
|
||||
2. **Plan updates** — plumbed from the planner / plan-summary side. Ideal
|
||||
shape: a `plan/updated` event upstream, with `trace-signal` derived from
|
||||
it. If the planner state is internal, at minimum emit the "assistant
|
||||
drafted a new plan" fact when it happens (the heuristic detector proves
|
||||
the text is recoverable, but the wire truth is upstream).
|
||||
3. **Plan restarts / tool errors** — pair `tool/result` `ok:false` with the
|
||||
next same-tool `tool/call` and emit the paired signal at emit time.
|
||||
|
||||
The detector-side dedup already covers the "signal already came from the
|
||||
wire" case, so upstream can ship these one at a time without a big-bang
|
||||
change: each signal kind lands, its heuristic branch becomes dead code, and
|
||||
eventually `trace-signal-detect.js` reduces to a pass-through for the wire
|
||||
signals.
|
||||
|
||||
Once all five signal kinds land upstream, the renderer-side detector
|
||||
becomes pure pass-through (roughly 20 lines) — a small doc-only PR at
|
||||
that point removes the heuristic scan entirely. Until then, the tri-view
|
||||
tabs read as "here's what the runtime is showing you" (wire signals) and
|
||||
"here's what the shell inferred" (heuristic, tooltipped as such).
|
||||
|
||||
248
examples/desktop/scripts/qa-trace-signals-fixture.mjs
Normal file
248
examples/desktop/scripts/qa-trace-signals-fixture.mjs
Normal file
@@ -0,0 +1,248 @@
|
||||
// qa-trace-signals-fixture.mjs — headless SVG proof for lane-trace-signals.
|
||||
//
|
||||
// The Electron shoot (qa-trace-signals-shoot.mjs) can't run in the CI env
|
||||
// (electron isn't installed there — that's the same story as the
|
||||
// pre-existing artifact-server.test.js failure). This script drives the
|
||||
// same rendering code paths through node's require system, then writes
|
||||
// three standalone HTML files that show the exact SVG output the Timeline
|
||||
// and Graph produce. Open them in a browser to see the badges/rings/chips
|
||||
// exactly as the real app would render them.
|
||||
//
|
||||
// Output (default: docs/trace-signals-shoot/):
|
||||
// signals-01-timeline-loop.html
|
||||
// signals-02-graph-error.html
|
||||
// signals-03-chips-plan.html
|
||||
|
||||
import { writeFileSync, mkdirSync } from 'node:fs'
|
||||
import { resolve, dirname } from 'node:path'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import { createRequire } from 'node:module'
|
||||
|
||||
const require = createRequire(import.meta.url)
|
||||
const __filename = fileURLToPath(import.meta.url)
|
||||
const __dirname = dirname(__filename)
|
||||
|
||||
const outdir = process.argv[2] || resolve(__dirname, '..', 'docs', 'trace-signals-shoot')
|
||||
mkdirSync(outdir, { recursive: true })
|
||||
|
||||
const T = require(resolve(__dirname, '..', 'src', 'renderer', 'trace-timeline.js'))
|
||||
const G = require(resolve(__dirname, '..', 'src', 'renderer', 'trace-graph.js'))
|
||||
const SD = require(resolve(__dirname, '..', 'src', 'renderer', 'trace-signal-detect.js'))
|
||||
|
||||
// Node has no document; hand-roll a minimal serializer over the same shape
|
||||
// the renderer builds. This mirrors the test-shim in test/trace-signal-*.
|
||||
function makeDoc() {
|
||||
function makeEl(tagOrNS, tag) {
|
||||
const cls = new Set()
|
||||
const el = {
|
||||
tagName: (tag || tagOrNS).toLowerCase(),
|
||||
isSvg: !!tag && tagOrNS === 'http://www.w3.org/2000/svg',
|
||||
_children: [], _attrs: {}, _cls: cls,
|
||||
textContent: '',
|
||||
dataset: {},
|
||||
style: {},
|
||||
get className() { return Array.from(cls).join(' ') },
|
||||
set className(v) { cls.clear(); String(v || '').split(/\s+/).forEach(x => x && cls.add(x)) },
|
||||
classList: {
|
||||
add(c) { cls.add(c) }, remove(c) { cls.delete(c) },
|
||||
toggle(c, on) { if (on) cls.add(c); else cls.delete(c) },
|
||||
contains(c) { return cls.has(c) },
|
||||
},
|
||||
appendChild(c) { this._children.push(c); return c },
|
||||
append(...cs) { for (const c of cs) this._children.push(c) },
|
||||
insertBefore(node) { this._children.unshift(node); return node },
|
||||
setAttribute(k, v) {
|
||||
this._attrs[k] = String(v)
|
||||
if (k === 'class') {
|
||||
cls.clear()
|
||||
String(v || '').split(/\s+/).forEach(x => x && cls.add(x))
|
||||
}
|
||||
},
|
||||
getAttribute(k) { return this._attrs[k] },
|
||||
addEventListener() {}, removeEventListener() {},
|
||||
querySelector() { return null }, querySelectorAll() { return [] },
|
||||
}
|
||||
return el
|
||||
}
|
||||
return {
|
||||
createElement(t) { return makeEl(t) },
|
||||
createElementNS(ns, t) { return makeEl(ns, t) },
|
||||
body: makeEl('body'),
|
||||
}
|
||||
}
|
||||
|
||||
function serialize(el) {
|
||||
if (!el) return ''
|
||||
if (typeof el === 'string') return el
|
||||
const tag = el.tagName
|
||||
const cls = el.className
|
||||
const attrs = []
|
||||
for (const [k, v] of Object.entries(el._attrs || {})) {
|
||||
if (k === 'class') continue
|
||||
attrs.push(`${k}="${escapeHtml(v)}"`)
|
||||
}
|
||||
if (cls) attrs.push(`class="${escapeHtml(cls)}"`)
|
||||
const attrStr = attrs.length ? ' ' + attrs.join(' ') : ''
|
||||
const kids = (el._children || []).map(serialize).join('')
|
||||
const text = el.textContent && (!el._children || !el._children.length) ? escapeHtml(el.textContent) : ''
|
||||
return `<${tag}${attrStr}>${text}${kids}</${tag}>`
|
||||
}
|
||||
|
||||
function escapeHtml(s) {
|
||||
return String(s == null ? '' : s)
|
||||
.replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>')
|
||||
.replace(/"/g, '"').replace(/'/g, ''')
|
||||
}
|
||||
|
||||
function pageWrap(title, body, note) {
|
||||
return `<!doctype html>
|
||||
<html><head><meta charset="utf-8"><title>${escapeHtml(title)}</title>
|
||||
<style>
|
||||
body { font: 13px system-ui, sans-serif; margin: 24px; background: #f9fafb; color: #1d1d1f; }
|
||||
h1 { font-size: 15px; margin: 0 0 8px 0; }
|
||||
.note { color: #6b6b70; font-size: 12px; margin: 0 0 16px 0; max-width: 720px; line-height: 1.4; }
|
||||
.stage { background: #fff; padding: 16px; border: 1px solid #e5e5ea; border-radius: 8px; overflow-x: auto; }
|
||||
/* Palette matches src/renderer/style.css tail block for signal badges/rings/chips. */
|
||||
.trace-timeline-svg { font: 11px system-ui, sans-serif; }
|
||||
.trace-timeline-tick { stroke: #d0d0d5; stroke-width: 0.5; stroke-dasharray: 2 3; }
|
||||
.trace-timeline-tick-label { fill: #6b6b70; font-size: 10px; }
|
||||
.trace-timeline-label { fill: #1d1d1f; font-size: 11px; }
|
||||
.trace-timeline-bar.family-step { fill: #b0b4bd; }
|
||||
.trace-timeline-bar.family-llm { fill: #7c3aed; }
|
||||
.trace-timeline-bar.family-tool { fill: #b45309; }
|
||||
.trace-timeline-bar.family-input { fill: #6b6b70; }
|
||||
.trace-timeline-signal-badge { stroke: rgba(0,0,0,0.15); stroke-width: 0.75; }
|
||||
.trace-timeline-signal-badge.sig-error { fill: #dc2626; }
|
||||
.trace-timeline-signal-badge.sig-loop { fill: #ef4444; }
|
||||
.trace-timeline-signal-badge.sig-redundant{ fill: #f59e0b; }
|
||||
.trace-timeline-signal-badge.sig-plan { fill: #2563eb; }
|
||||
.trace-graph-svg { background: #fff; }
|
||||
.trace-graph-node-body { fill: #f3f4f6; stroke: #9ca3af; stroke-width: 1; }
|
||||
.trace-graph-node-body.family-tool { fill: #fde68a; stroke: #b45309; }
|
||||
.trace-graph-node-body.family-step { fill: #e5e7eb; stroke: #6b7280; }
|
||||
.trace-graph-node-body.family-llm { fill: #ddd6fe; stroke: #7c3aed; }
|
||||
.trace-graph-node-glyph { font: 12px system-ui, sans-serif; fill: #1d1d1f; }
|
||||
.trace-graph-node-label { font: 10px system-ui, sans-serif; fill: #1d1d1f; }
|
||||
.trace-graph-edge { stroke: #9ca3af; stroke-width: 1.2; }
|
||||
.trace-graph-signal-ring { stroke-width: 2.5; fill: none; }
|
||||
.trace-graph-signal-ring.sig-error { stroke: #dc2626; }
|
||||
.trace-graph-signal-ring.sig-loop { stroke: #ef4444; }
|
||||
.trace-graph-signal-ring.sig-plan { stroke: #2563eb; }
|
||||
.assistant-turn { border-left: 3px solid #d0d0d5; padding: 12px; margin: 12px 0; background: #fff; border-radius: 6px; }
|
||||
.turn-body { display: flex; flex-direction: column; gap: 6px; }
|
||||
.text-block { font-size: 13px; color: #1d1d1f; }
|
||||
.tool-row { font-family: ui-monospace, monospace; font-size: 12px; color: #4b5563; }
|
||||
.turn-glyph { display: inline-block; width: 14px; }
|
||||
.turn-signal-chip-row { display: flex; flex-wrap: wrap; gap: 6px; padding: 4px 0 6px 0; align-items: center; }
|
||||
.turn-signal-chip { font-size: 11px; line-height: 1; padding: 3px 8px; border-radius: 10px; border: 1px solid transparent; background: rgba(0,0,0,0.03); color: #1d1d1f; cursor: pointer; font-family: inherit; }
|
||||
.turn-signal-chip::before { content: ''; display: inline-block; width: 6px; height: 6px; border-radius: 50%; margin-right: 6px; vertical-align: 1px; background: currentColor; }
|
||||
.turn-signal-chip.sig-error, .turn-signal-chip.sig-loop { color: #b91c1c; border-color: rgba(239,68,68,0.35); background: rgba(239,68,68,0.06); }
|
||||
.turn-signal-chip.sig-redundant { color: #92400e; border-color: rgba(245,158,11,0.4); background: rgba(245,158,11,0.08); }
|
||||
.turn-signal-chip.sig-plan { color: #1d4ed8; border-color: rgba(37,99,235,0.35); background: rgba(37,99,235,0.06); }
|
||||
</style>
|
||||
</head><body>
|
||||
<h1>${escapeHtml(title)}</h1>
|
||||
<p class="note">${note}</p>
|
||||
<div class="stage">${body}</div>
|
||||
</body></html>`
|
||||
}
|
||||
|
||||
// ─── shot 1 — Timeline with a loop-detected badge on seq 12 ──────────
|
||||
{
|
||||
const doc = makeDoc()
|
||||
const rec = {
|
||||
turn: 1, step: 0, startSeq: 10, endSeq: 15,
|
||||
startTime: 1000, endTime: 1500, durationMs: 500,
|
||||
summary: 'read main.ts',
|
||||
inputs: [], outputs: [],
|
||||
events: [
|
||||
{ type: 'tool/call', seq: 11, time: 1050, data: { name: 'fs.read', arguments: '{"path":"main.ts"}', callId: 'c1' } },
|
||||
{ type: 'tool/result', seq: 11.5, time: 1080, data: { callId: 'c1', ok: true } },
|
||||
{ type: 'tool/call', seq: 12, time: 1150, data: { name: 'fs.read', arguments: '{"path":"main.ts"}', callId: 'c2' } },
|
||||
{ type: 'tool/result', seq: 12.5, time: 1180, data: { callId: 'c2', ok: true } },
|
||||
{ type: 'tool/call', seq: 13, time: 1250, data: { name: 'fs.read', arguments: '{"path":"main.ts"}', callId: 'c3' } },
|
||||
{ type: 'tool/result', seq: 13.5, time: 1280, data: { callId: 'c3', ok: true } },
|
||||
],
|
||||
}
|
||||
const { bySeq, all } = SD.detectSignalsFromRecords(rec, { loopN: 3 })
|
||||
const el = T.renderTimeline(doc, rec, { signals: bySeq, width: 820 })
|
||||
const html = pageWrap(
|
||||
'Signals 01 — Timeline: loop-detected badge',
|
||||
serialize(el),
|
||||
'Three consecutive <code>fs.read</code> calls with identical args. The detector fires <code>loop-detected</code> on the third call; its badge sits in the left gutter, colored red. Signals detected: <code>' + all.map(s => s.signal).join(', ') + '</code>. See docs/upstream-ledger.md L-2.',
|
||||
)
|
||||
writeFileSync(resolve(outdir, 'signals-01-timeline-loop.html'), html)
|
||||
console.log('wrote', resolve(outdir, 'signals-01-timeline-loop.html'))
|
||||
}
|
||||
|
||||
// ─── shot 2 — Graph with a tool-error ring around the failing call ───
|
||||
{
|
||||
const doc = makeDoc()
|
||||
const rec = {
|
||||
turn: 1, step: 0, startSeq: 10, endSeq: 14,
|
||||
startTime: 1000, endTime: 1500, durationMs: 500,
|
||||
summary: 'bash checks',
|
||||
inputs: [], outputs: [],
|
||||
events: [
|
||||
{ type: 'tool/call', seq: 11, time: 1050, data: { name: 'bash', arguments: 'ls /nope', callId: 'c1' } },
|
||||
{ type: 'tool/result', seq: 12, time: 1080, data: { callId: 'c1', ok: false, error: 'ENOENT' } },
|
||||
{ type: 'tool/call', seq: 13, time: 1150, data: { name: 'bash', arguments: 'ls /tmp', callId: 'c2' } },
|
||||
{ type: 'tool/result', seq: 14, time: 1180, data: { callId: 'c2', ok: true } },
|
||||
],
|
||||
}
|
||||
const { bySeq, all } = SD.detectSignalsFromRecords(rec)
|
||||
const el = G.renderGraph(doc, rec, { signals: bySeq })
|
||||
const html = pageWrap(
|
||||
'Signals 02 — Graph: tool-error ring',
|
||||
serialize(el),
|
||||
'A failing <code>bash</code> call followed by a successful retry. The detector emits <code>tool-error</code> on the failing call\'s seq and <code>plan-restart</code> on the retry. The Graph node for the failing call gets a red outer ring; the retry node gets a blue ring. Signals detected: <code>' + all.map(s => s.signal).join(', ') + '</code>.',
|
||||
)
|
||||
writeFileSync(resolve(outdir, 'signals-02-graph-error.html'), html)
|
||||
console.log('wrote', resolve(outdir, 'signals-02-graph-error.html'))
|
||||
}
|
||||
|
||||
// ─── shot 3 — Assistant turn with signal chip row ────────────────────
|
||||
{
|
||||
const events = [
|
||||
{ type: 'turn/start', seq: 1, time: 1000, data: { turn: 1 } },
|
||||
{ type: 'assistant/message', seq: 2, time: 1050, data: { content: [{ type: 'text', text: 'Here is the new plan: 1. read main.ts\n2. edit imports\n3. verify.' }] } },
|
||||
{ type: 'tool/call', seq: 3, time: 1100, data: { name: 'fs.read', arguments: '{"path":"main.ts"}', callId: 'c1' } },
|
||||
{ type: 'tool/call', seq: 4, time: 1150, data: { name: 'fs.read', arguments: '{"path":"main.ts"}', callId: 'c2' } },
|
||||
{ type: 'tool/call', seq: 5, time: 1200, data: { name: 'fs.read', arguments: '{"path":"main.ts"}', callId: 'c3' } },
|
||||
{ type: 'tool/call', seq: 6, time: 1250, data: { name: 'bash', arguments: 'ls', callId: 'c4' } },
|
||||
{ type: 'tool/call', seq: 7, time: 1300, data: { name: 'fs.read', arguments: '{"path":"main.ts"}', callId: 'c5' } },
|
||||
{ type: 'turn/end', seq: 8, time: 1350 },
|
||||
]
|
||||
const { all } = SD.detectSignals(events)
|
||||
// Group by signal kind (matches renderer.js applyTurnSignalChips)
|
||||
const seen = new Map()
|
||||
for (const s of all) {
|
||||
if (!seen.has(s.signal)) seen.set(s.signal, { signal: s.signal, count: 1, first: s })
|
||||
else seen.get(s.signal).count++
|
||||
}
|
||||
const chips = Array.from(seen.values()).map(e => {
|
||||
const cls = SD.classFor(e.signal)
|
||||
const label = e.count > 1 ? `${SD.labelFor(e.signal)} × ${e.count}` : SD.labelFor(e.signal)
|
||||
const title = SD.tooltipFor(e.first)
|
||||
return `<button class="turn-signal-chip ${cls}" title="${escapeHtml(title)}">${escapeHtml(label)}</button>`
|
||||
}).join('')
|
||||
const body = `
|
||||
<section class="assistant-turn">
|
||||
<div class="turn-body">
|
||||
<div class="turn-signal-chip-row">${chips}</div>
|
||||
<div class="text-block">Here is the new plan: 1. read main.ts, 2. edit imports, 3. verify.</div>
|
||||
<div class="tool-row"><span class="turn-glyph">▸</span>fs.read(main.ts)</div>
|
||||
<div class="tool-row"><span class="turn-glyph">▸</span>fs.read(main.ts)</div>
|
||||
<div class="tool-row"><span class="turn-glyph">▸</span>fs.read(main.ts)</div>
|
||||
<div class="tool-row"><span class="turn-glyph">▸</span>bash(ls)</div>
|
||||
<div class="tool-row"><span class="turn-glyph">▸</span>fs.read(main.ts)</div>
|
||||
</div>
|
||||
</section>`
|
||||
const html = pageWrap(
|
||||
'Signals 03 — Turn container: signal chip row',
|
||||
body,
|
||||
'The chip row sits above the assistant body. Each chip covers one signal kind detected in this turn (loop / redundant / plan-update). Clicking a chip auto-opens the trace drawer for a drill-in. Signals detected in this turn: <code>' + all.map(s => s.signal).join(', ') + '</code>.',
|
||||
)
|
||||
writeFileSync(resolve(outdir, 'signals-03-chips-plan.html'), html)
|
||||
console.log('wrote', resolve(outdir, 'signals-03-chips-plan.html'))
|
||||
}
|
||||
159
examples/desktop/scripts/qa-trace-signals-shoot.mjs
Normal file
159
examples/desktop/scripts/qa-trace-signals-shoot.mjs
Normal file
@@ -0,0 +1,159 @@
|
||||
// qa-trace-signals-shoot.mjs — lane-trace-signals selfie driver.
|
||||
//
|
||||
// Three shots proving the trace signal overlays (feat/trace-signals):
|
||||
// signals-01 Timeline view — loop-detected + redundant-call badges
|
||||
// in the left gutter of a loop-heavy step-record
|
||||
// signals-02 Graph view — colored ring around a tool-error node
|
||||
// signals-03 Turn footer — signal chip row above the assistant body
|
||||
// with three signal kinds (loop/redundant/plan) chips
|
||||
//
|
||||
// Requires the desktop shell running via `pnpm --dir examples/desktop start`
|
||||
// with --remote-debugging-port=<port> exposed. In an env without electron,
|
||||
// see scripts/qa-trace-signals-fixture.mjs for a headless SVG render that
|
||||
// exercises the exact rendering code paths — enough for a diff-review.
|
||||
//
|
||||
// Usage:
|
||||
// node scripts/qa-trace-signals-shoot.mjs <port> <outdir>
|
||||
|
||||
import { writeFileSync, mkdirSync } from 'node:fs'
|
||||
import { resolve } from 'node:path'
|
||||
|
||||
const [,, portArg, outdir] = process.argv
|
||||
const port = portArg || '9241'
|
||||
if (!outdir) {
|
||||
console.error('usage: node scripts/qa-trace-signals-shoot.mjs <port> <outdir>')
|
||||
process.exit(1)
|
||||
}
|
||||
mkdirSync(outdir, { recursive: true })
|
||||
|
||||
async function cdp() {
|
||||
const listRes = await fetch(`http://localhost:${port}/json/list`)
|
||||
const targets = await listRes.json()
|
||||
const target = targets.find((t) => t.type === 'page')
|
||||
if (!target) throw new Error('no page target on port ' + port)
|
||||
const ws = new WebSocket(target.webSocketDebuggerUrl)
|
||||
await new Promise((r, x) => { ws.onopen = r; ws.onerror = (e) => x(e) })
|
||||
let id = 1
|
||||
const pending = new Map()
|
||||
ws.onmessage = (ev) => {
|
||||
const data = typeof ev.data === 'string' ? ev.data : String(ev.data)
|
||||
let msg; try { msg = JSON.parse(data) } catch { return }
|
||||
if (msg.id != null && pending.has(msg.id)) {
|
||||
const [ok, err] = pending.get(msg.id); pending.delete(msg.id)
|
||||
if (msg.error) err(new Error(msg.error.message)); else ok(msg.result)
|
||||
}
|
||||
}
|
||||
const call = (m, p = {}, timeoutMs = 60000) => new Promise((ok, err) => {
|
||||
const _id = id++
|
||||
const t = setTimeout(() => { pending.delete(_id); err(new Error(`cdp timeout: ${m}`)) }, timeoutMs)
|
||||
pending.set(_id, [(v) => { clearTimeout(t); ok(v) }, (e) => { clearTimeout(t); err(e) }])
|
||||
ws.send(JSON.stringify({ id: _id, method: m, params: p }))
|
||||
})
|
||||
const evjs = async (js) => {
|
||||
const r = await call('Runtime.evaluate', { expression: js, returnByValue: true, awaitPromise: true })
|
||||
if (r.exceptionDetails) throw new Error(r.exceptionDetails.exception?.description || r.exceptionDetails.text)
|
||||
return r.result?.value
|
||||
}
|
||||
const sleep = (ms) => new Promise((r) => setTimeout(r, ms))
|
||||
return { call, evjs, sleep, close: () => ws.close() }
|
||||
}
|
||||
|
||||
async function shoot(c, name, opts) {
|
||||
const { fixture, prep, wait = 500 } = opts
|
||||
await c.evjs(`(function () {
|
||||
const fx = ${JSON.stringify(fixture)};
|
||||
// Inject fixture into a fresh session and force a re-render.
|
||||
const chat = window.__dshChat; if (!chat) throw new Error('__dshChat missing');
|
||||
const id = chat.newSession ? chat.newSession('trace-signals-' + '${name}') : 'trace-signals-${name}';
|
||||
const meta = window.__dshState && window.__dshState.sessions
|
||||
? window.__dshState.sessions.get(id) : null;
|
||||
if (meta) { meta.cachedEvents = fx.events.slice(); }
|
||||
if (typeof window.__dshQaReplayFixture === 'function') {
|
||||
window.__dshQaReplayFixture(id, fx.events);
|
||||
}
|
||||
return id;
|
||||
})()`)
|
||||
await c.sleep(wait)
|
||||
if (typeof prep === 'function') await prep(c)
|
||||
const shot = await c.call('Page.captureScreenshot', { format: 'png' })
|
||||
const buf = Buffer.from(shot.data, 'base64')
|
||||
const out = resolve(outdir, `${name}.png`)
|
||||
writeFileSync(out, buf)
|
||||
console.log('wrote', out)
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const c = await cdp()
|
||||
try {
|
||||
// Fixture 1: three identical fs.read calls → loop-detected + redundant
|
||||
await shoot(c, 'signals-01-timeline-loop', {
|
||||
fixture: {
|
||||
events: [
|
||||
{ type: 'turn/start', seq: 1, time: 1000, data: { turn: 1 } },
|
||||
{ type: 'step/start', seq: 2, time: 1010, data: { turn: 1, step: 0 } },
|
||||
{ type: 'tool/call', seq: 3, time: 1050, data: { name: 'fs.read', arguments: '{"path":"main.ts"}', callId: 'c1' } },
|
||||
{ type: 'tool/result', seq: 4, time: 1080, data: { callId: 'c1', ok: true } },
|
||||
{ type: 'tool/call', seq: 5, time: 1100, data: { name: 'fs.read', arguments: '{"path":"main.ts"}', callId: 'c2' } },
|
||||
{ type: 'tool/result', seq: 6, time: 1130, data: { callId: 'c2', ok: true } },
|
||||
{ type: 'tool/call', seq: 7, time: 1150, data: { name: 'fs.read', arguments: '{"path":"main.ts"}', callId: 'c3' } },
|
||||
{ type: 'tool/result', seq: 8, time: 1180, data: { callId: 'c3', ok: true } },
|
||||
{ type: 'step/end', seq: 9, time: 1200, data: { turn: 1, step: 0 } },
|
||||
{ type: 'turn/end', seq: 10, time: 1210 },
|
||||
],
|
||||
},
|
||||
// Open the trace drawer and switch to Timeline
|
||||
prep: async (c) => {
|
||||
await c.evjs(`(function () {
|
||||
const drawer = document.querySelector('.turn-trace-drawer');
|
||||
if (drawer) drawer.open = true;
|
||||
const btn = document.querySelector('.trace-tri-chip.chip-timeline');
|
||||
if (btn) btn.click();
|
||||
})()`)
|
||||
await c.sleep(300)
|
||||
},
|
||||
})
|
||||
|
||||
// Fixture 2: tool error → red ring on the graph node
|
||||
await shoot(c, 'signals-02-graph-error', {
|
||||
fixture: {
|
||||
events: [
|
||||
{ type: 'turn/start', seq: 1, time: 1000, data: { turn: 1 } },
|
||||
{ type: 'step/start', seq: 2, time: 1010, data: { turn: 1, step: 0 } },
|
||||
{ type: 'tool/call', seq: 3, time: 1050, data: { name: 'bash', arguments: 'ls /nope', callId: 'c1' } },
|
||||
{ type: 'tool/result', seq: 4, time: 1080, data: { callId: 'c1', ok: false, error: 'ENOENT: no such file or directory' } },
|
||||
{ type: 'tool/call', seq: 5, time: 1100, data: { name: 'bash', arguments: 'ls /tmp', callId: 'c2' } },
|
||||
{ type: 'tool/result', seq: 6, time: 1130, data: { callId: 'c2', ok: true } },
|
||||
{ type: 'step/end', seq: 7, time: 1200, data: { turn: 1, step: 0 } },
|
||||
{ type: 'turn/end', seq: 8, time: 1210 },
|
||||
],
|
||||
},
|
||||
prep: async (c) => {
|
||||
await c.evjs(`(function () {
|
||||
const drawer = document.querySelector('.turn-trace-drawer');
|
||||
if (drawer) drawer.open = true;
|
||||
const btn = document.querySelector('.trace-tri-chip.chip-graph');
|
||||
if (btn) btn.click();
|
||||
})()`)
|
||||
await c.sleep(300)
|
||||
},
|
||||
})
|
||||
|
||||
// Fixture 3: plan-update + loop → chips above turn body
|
||||
await shoot(c, 'signals-03-chips-plan', {
|
||||
fixture: {
|
||||
events: [
|
||||
{ type: 'turn/start', seq: 1, time: 1000, data: { turn: 1 } },
|
||||
{ type: 'assistant/message', seq: 2, time: 1050, data: { content: [{ type: 'text', text: 'Here is the new plan: 1. read main.ts\n2. edit imports\n3. verify' }] } },
|
||||
{ type: 'tool/call', seq: 3, time: 1100, data: { name: 'fs.read', arguments: '{"path":"main.ts"}', callId: 'c1' } },
|
||||
{ type: 'tool/call', seq: 4, time: 1150, data: { name: 'fs.read', arguments: '{"path":"main.ts"}', callId: 'c2' } },
|
||||
{ type: 'tool/call', seq: 5, time: 1200, data: { name: 'fs.read', arguments: '{"path":"main.ts"}', callId: 'c3' } },
|
||||
{ type: 'turn/end', seq: 6, time: 1210 },
|
||||
],
|
||||
},
|
||||
})
|
||||
} finally {
|
||||
c.close()
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((e) => { console.error(e); process.exit(1) })
|
||||
@@ -1323,6 +1323,7 @@
|
||||
<script src="./inject-family.js"></script>
|
||||
<script src="./raw-inject.js"></script><!-- ticket #15 B: envelope:'raw' classifier -->
|
||||
<script src="./trace-aggregator.js"></script>
|
||||
<script src="./trace-signal-detect.js"></script><!-- lane-trace-signals: heuristic signal detector (see docs/upstream-ledger.md L-2) -->
|
||||
<script src="./trace-timeline.js"></script><!-- task #203: view B Gantt/waterfall -->
|
||||
<script src="./trace-graph.js"></script><!-- task #203: view C agent node graph -->
|
||||
<script src="./trace-detail-pane.js"></script><!-- task #205: right-side detail pane (4 tabs) -->
|
||||
|
||||
@@ -1553,9 +1553,90 @@ function finishTurnContainer(sessionId, { footerSpec, traceCard, traceSummaryTex
|
||||
}
|
||||
ct.section.appendChild(footer)
|
||||
ct.section.dataset.turnStatus = 'sealed'
|
||||
// Signal marker chips: overlay any loop/redundant/plan/error signals
|
||||
// detected in this turn's cached events onto the top of the turn
|
||||
// section. The chip row sits above the assistant body so a reader
|
||||
// scanning the stream sees "this turn had a loop" before deciding
|
||||
// whether to open the trace drawer. See trace-signal-detect.js and
|
||||
// docs/upstream-ledger.md L-2.
|
||||
applyTurnSignalChips(sessionId, ct.section)
|
||||
state.currentTurn = null
|
||||
}
|
||||
|
||||
// Compute+attach the signal chip row for a just-sealed turn. Reads
|
||||
// meta.cachedEvents (already populated) and detects signals whose seq
|
||||
// falls inside this turn's range. When no signals fire, no chip row is
|
||||
// added.
|
||||
function applyTurnSignalChips(sessionId, section) {
|
||||
try {
|
||||
const SD = window.__dshTraceSignalDetect
|
||||
if (!SD || typeof SD.detectSignals !== 'function') return
|
||||
const meta = state.sessions.get(sessionId)
|
||||
if (!meta || !Array.isArray(meta.cachedEvents) || !meta.cachedEvents.length) return
|
||||
// Restrict to events whose seq falls inside this turn's range so the
|
||||
// chip row reflects THIS turn, not the whole session. We use the last
|
||||
// `turn/start`→`turn/end` bracket in the cache. When no bracket is
|
||||
// findable, fall back to detecting on the whole cache (which will still
|
||||
// produce meaningful chips at the session scope).
|
||||
const range = _lastTurnSeqRange(meta.cachedEvents)
|
||||
const scope = range
|
||||
? meta.cachedEvents.filter(ev => typeof ev.seq === 'number'
|
||||
&& ev.seq >= range.start && ev.seq <= range.end)
|
||||
: meta.cachedEvents
|
||||
const { all } = SD.detectSignals(scope)
|
||||
if (!all.length) return
|
||||
// Dedup by signal kind for the chip row: the row is a "kinds seen"
|
||||
// summary; the badges in the drawer show the specific seqs.
|
||||
const seen = new Map()
|
||||
for (const sig of all) {
|
||||
const key = sig.signal
|
||||
if (!seen.has(key)) seen.set(key, { signal: sig.signal, count: 1, first: sig })
|
||||
else seen.get(key).count++
|
||||
}
|
||||
const row = document.createElement('div')
|
||||
row.className = 'turn-signal-chip-row'
|
||||
for (const entry of seen.values()) {
|
||||
const chip = document.createElement('button')
|
||||
chip.type = 'button'
|
||||
chip.className = `turn-signal-chip ${SD.classFor(entry.signal)}`
|
||||
chip.dataset.signal = entry.signal
|
||||
chip.textContent = entry.count > 1
|
||||
? `${SD.labelFor(entry.signal)} × ${entry.count}`
|
||||
: SD.labelFor(entry.signal)
|
||||
chip.title = SD.tooltipFor(entry.first)
|
||||
// Clicking a chip opens the trace drawer so the reader can drill in.
|
||||
chip.addEventListener('click', function () {
|
||||
const drawer = section.querySelector('.turn-trace-drawer')
|
||||
if (drawer) {
|
||||
drawer.open = true
|
||||
if (typeof drawer.scrollIntoView === 'function') {
|
||||
try { drawer.scrollIntoView({ block: 'nearest' }) } catch (_) {}
|
||||
}
|
||||
}
|
||||
})
|
||||
row.appendChild(chip)
|
||||
}
|
||||
// Insert as the first body-child so it sits above assistant text/tool
|
||||
// rows without breaking the turn-rule up top.
|
||||
const body = section.querySelector('.turn-body')
|
||||
if (body && body.firstChild) body.insertBefore(row, body.firstChild)
|
||||
else if (body) body.appendChild(row)
|
||||
else section.appendChild(row)
|
||||
} catch (_) { /* chip row is a visual enhancement — never crash the stream */ }
|
||||
}
|
||||
|
||||
function _lastTurnSeqRange(events) {
|
||||
let start = null, end = null
|
||||
for (let i = events.length - 1; i >= 0; i--) {
|
||||
const ev = events[i]
|
||||
if (!ev || typeof ev.seq !== 'number') continue
|
||||
if (end === null && ev.type === 'turn/end') end = ev.seq
|
||||
if (ev.type === 'turn/start') { start = ev.seq; break }
|
||||
}
|
||||
if (start === null || end === null) return null
|
||||
return { start, end }
|
||||
}
|
||||
|
||||
function ensureStreamingBubble(sessionId) {
|
||||
if (state.streaming && state.streaming.sessionId === sessionId) return state.streaming.el
|
||||
// Ensure the turn container is open before the bubble drops in so
|
||||
|
||||
@@ -11510,6 +11510,64 @@ details.devtools-row[open] > .devtools-row-summary::before { transform: rotate(9
|
||||
.trace-detail-field-block[open] > .trace-detail-field-block-head::before { transform: rotate(90deg); }
|
||||
.trace-detail-field-block > .trace-detail-field-block-head:hover { background: var(--surface-hover); }
|
||||
|
||||
/* --- trace signal badges (lane-trace-signals; see docs/upstream-ledger.md L-2) --
|
||||
* The detector runs renderer-side until upstream emits `trace/signal` events.
|
||||
* Colors follow the palette: red=error/loop, amber=redundant, blue=plan.
|
||||
* Multiple signals stack horizontally on the Timeline; the Graph uses a single
|
||||
* outer ring per node (highest-priority signal wins the color).
|
||||
*/
|
||||
.trace-timeline-signal-badge { stroke: rgba(0,0,0,0.15); stroke-width: 0.75; }
|
||||
.trace-timeline-signal-badge.sig-error { fill: #dc2626; }
|
||||
.trace-timeline-signal-badge.sig-loop { fill: #ef4444; }
|
||||
.trace-timeline-signal-badge.sig-redundant{ fill: #f59e0b; }
|
||||
.trace-timeline-signal-badge.sig-plan { fill: #2563eb; }
|
||||
.trace-timeline-signal-badge.sig-plan-restart { fill: #1d4ed8; }
|
||||
.trace-timeline-signal-badge.sig-generic { fill: #6b7280; }
|
||||
|
||||
.trace-graph-signal-ring { stroke-width: 2.5; }
|
||||
.trace-graph-signal-ring.sig-error { stroke: #dc2626; }
|
||||
.trace-graph-signal-ring.sig-loop { stroke: #ef4444; }
|
||||
.trace-graph-signal-ring.sig-redundant{ stroke: #f59e0b; }
|
||||
.trace-graph-signal-ring.sig-plan { stroke: #2563eb; }
|
||||
.trace-graph-signal-ring.sig-plan-restart { stroke: #1d4ed8; }
|
||||
.trace-graph-signal-ring.sig-generic { stroke: #6b7280; }
|
||||
|
||||
/* Main-flow marker chips — small pill row above the assistant turn body */
|
||||
.turn-signal-chip-row {
|
||||
display: flex; flex-wrap: wrap; gap: 6px;
|
||||
padding: 4px 0 6px 0;
|
||||
align-items: center;
|
||||
}
|
||||
.turn-signal-chip {
|
||||
font-size: 11px; line-height: 1;
|
||||
padding: 3px 8px; border-radius: 10px;
|
||||
border: 1px solid transparent;
|
||||
background: rgba(0,0,0,0.03);
|
||||
color: #1d1d1f;
|
||||
cursor: pointer;
|
||||
font-family: inherit;
|
||||
}
|
||||
.turn-signal-chip:hover { filter: brightness(0.96); }
|
||||
.turn-signal-chip::before {
|
||||
content: ''; display: inline-block; width: 6px; height: 6px;
|
||||
border-radius: 50%; margin-right: 6px; vertical-align: 1px;
|
||||
background: currentColor;
|
||||
}
|
||||
.turn-signal-chip.sig-error { color: #b91c1c; border-color: rgba(220,38,38,0.35); background: rgba(220,38,38,0.06); }
|
||||
.turn-signal-chip.sig-loop { color: #b91c1c; border-color: rgba(239,68,68,0.35); background: rgba(239,68,68,0.06); }
|
||||
.turn-signal-chip.sig-redundant { color: #92400e; border-color: rgba(245,158,11,0.4); background: rgba(245,158,11,0.08); }
|
||||
.turn-signal-chip.sig-plan { color: #1d4ed8; border-color: rgba(37,99,235,0.35); background: rgba(37,99,235,0.06); }
|
||||
.turn-signal-chip.sig-plan-restart{ color: #1e3a8a; border-color: rgba(29,78,216,0.4); background: rgba(29,78,216,0.08); }
|
||||
|
||||
/* Dark theme reads the badges/chips against the dark surface, so bump the
|
||||
* saturation a touch — the palette is designed for both. */
|
||||
@media (prefers-color-scheme: dark) {
|
||||
.turn-signal-chip { background: rgba(255,255,255,0.05); color: #f3f4f6; }
|
||||
.turn-signal-chip.sig-error, .turn-signal-chip.sig-loop { color: #fca5a5; }
|
||||
.turn-signal-chip.sig-redundant { color: #fcd34d; }
|
||||
.turn-signal-chip.sig-plan, .turn-signal-chip.sig-plan-restart { color: #93c5fd; }
|
||||
}
|
||||
|
||||
/* --- onboarding overlay: first-frame FOUC guard ------------------------ */
|
||||
/* index.html declares <div id="onboarding" class="onboarding" hidden> and */
|
||||
/* onboarding-ui.js clears the [hidden] attribute only after a first-run */
|
||||
|
||||
@@ -372,6 +372,36 @@ function renderGraph(doc, input, opts) {
|
||||
class: `trace-graph-node-body family-${node.family}`,
|
||||
})
|
||||
nodeG.appendChild(circle)
|
||||
|
||||
// Signal ring — a colored outer stroke when this node's seq shows up
|
||||
// in the passed signals map. Multiple signals share one ring but the
|
||||
// tooltip enumerates them. See trace-signal-detect.js + L-2 RFC.
|
||||
if (options.signals && typeof options.signals.get === 'function' && node.seq !== null) {
|
||||
const sigs = options.signals.get(node.seq)
|
||||
if (sigs && sigs.length) {
|
||||
// Highest-priority signal wins the ring class (error > loop > redundant > plan).
|
||||
const priority = ['tool-error', 'loop-detected', 'plan-restart', 'redundant-call', 'plan-update']
|
||||
let winner = sigs[0]
|
||||
let winnerRank = 999
|
||||
for (const s of sigs) {
|
||||
const r = priority.indexOf(s.signal)
|
||||
if (r >= 0 && r < winnerRank) { winner = s; winnerRank = r }
|
||||
}
|
||||
const SD = (typeof window !== 'undefined' && window.__dshTraceSignalDetect)
|
||||
|| (typeof require !== 'undefined' ? require('./trace-signal-detect.js') : null)
|
||||
const cls = SD ? SD.classFor(winner.signal) : 'sig-generic'
|
||||
const ring = svgEl(doc, 'circle', {
|
||||
cx: 0, cy: 0, r: NODE_R + 4,
|
||||
class: `trace-graph-signal-ring ${cls}`,
|
||||
fill: 'none',
|
||||
})
|
||||
const title = svgEl(doc, 'title', {})
|
||||
title.textContent = SD ? sigs.map(s => SD.tooltipFor(s)).join('\n') : sigs.map(s => s.signal).join(', ')
|
||||
ring.appendChild(title)
|
||||
nodeG.appendChild(ring)
|
||||
}
|
||||
}
|
||||
|
||||
const glyph = svgEl(doc, 'text', {
|
||||
x: 0, y: 4, class: 'trace-graph-node-glyph',
|
||||
'text-anchor': 'middle',
|
||||
|
||||
381
examples/desktop/src/renderer/trace-signal-detect.js
Normal file
381
examples/desktop/src/renderer/trace-signal-detect.js
Normal file
@@ -0,0 +1,381 @@
|
||||
// trace-signal-detect.js — heuristic detector for "trace signals" that a
|
||||
// researcher wants highlighted in the trace tri-view + the main assistant
|
||||
// stream. This is a renderer-side workaround for a gap in the runtime wire:
|
||||
// the SDK does not (yet) emit `type: 'trace/signal'` events, so we scan the
|
||||
// event stream ourselves and mark four kinds of interesting nodes.
|
||||
//
|
||||
// See docs/upstream-ledger.md L-2 for the RFC that would remove this file
|
||||
// from the critical path — once upstream emits real signal events, this
|
||||
// module becomes dead code (the DOM overlay path already checks `_wireSignal`
|
||||
// first and only falls back to detected signals when the wire is silent).
|
||||
//
|
||||
// The four signal kinds:
|
||||
//
|
||||
// loop-detected — >= N consecutive tool/call events with the same tool
|
||||
// name AND the same args-prefix. Default N = 3.
|
||||
// Reported on the SECOND repeat's seq (so a reader sees
|
||||
// the badge as soon as the loop becomes suspicious).
|
||||
// redundant-call — a tool/call whose (name, args-prefix) exactly matches
|
||||
// a call within the last WINDOW seqs. Default WINDOW = 8.
|
||||
// A true duplicate is often benign (retries, polling),
|
||||
// but a researcher wants it flagged because it's the
|
||||
// shape of accidental re-work. Not reported when the
|
||||
// loop-detected badge already covers the same seq.
|
||||
// plan-update — assistant/message text mentions a new plan (keywords:
|
||||
// "new plan", "revised plan", "updated plan", "here's the
|
||||
// plan", "plan:", numbered "1." at line start). Heuristic
|
||||
// — never claimed to be authoritative. The DOM overlay
|
||||
// surfaces the badge with an "heuristic" tooltip.
|
||||
// plan-restart — the same tool retried after a tool/result error
|
||||
// (ok:false). Signals "the plan is being restarted after
|
||||
// a failure" — often the first place a reader wants to
|
||||
// look when debugging a stuck agent.
|
||||
// tool-error — tool/result with ok:false. Already visible via the ✗
|
||||
// glyph, but we surface a red badge on Timeline/Graph so
|
||||
// a full-session view has the error nodes stand out.
|
||||
//
|
||||
// The detector is pure: it takes `events` (a flat array from
|
||||
// session.cachedEvents), returns `{ bySeq: Map<seq, Signal[]>, all: Signal[] }`.
|
||||
// Each Signal is { signal, seq, meta } where `meta` names the offending
|
||||
// tools / prior seqs / snippet so the tooltip can be informative.
|
||||
//
|
||||
// Deliberately lightweight — no dep on trace-aggregator so the detector
|
||||
// can run on raw wire without step boundaries.
|
||||
|
||||
'use strict'
|
||||
|
||||
;(function () {
|
||||
const DEFAULT_LOOP_N = 3
|
||||
const DEFAULT_REDUNDANT_WINDOW = 8
|
||||
const PLAN_KEYWORDS = [
|
||||
'new plan',
|
||||
'revised plan',
|
||||
'updated plan',
|
||||
'here\'s the plan',
|
||||
'here is the plan',
|
||||
'the plan is now',
|
||||
'let me revise',
|
||||
'let me update the plan',
|
||||
]
|
||||
|
||||
function detectSignals(events, opts) {
|
||||
const options = opts || {}
|
||||
const loopN = Number.isFinite(options.loopN) && options.loopN >= 2 ? options.loopN : DEFAULT_LOOP_N
|
||||
const window = Number.isFinite(options.window) && options.window >= 2 ? options.window : DEFAULT_REDUNDANT_WINDOW
|
||||
const list = Array.isArray(events) ? events : []
|
||||
const all = []
|
||||
const bySeq = new Map()
|
||||
|
||||
// Wire-pass first: if any event is already a signal from the runtime
|
||||
// (post-RFC L-2), consume it verbatim and skip heuristic detection for
|
||||
// that seq. Keeps this module a no-op once upstream lands the fix.
|
||||
for (const ev of list) {
|
||||
if (ev && ev.type === 'trace/signal' && ev.data && typeof ev.data.signal === 'string') {
|
||||
const sig = {
|
||||
signal: ev.data.signal,
|
||||
seq: typeof ev.seq === 'number' ? ev.seq : null,
|
||||
meta: Object.assign({ source: 'wire' }, ev.data),
|
||||
}
|
||||
emit(all, bySeq, sig)
|
||||
}
|
||||
}
|
||||
|
||||
// Rolling window over tool/call events for loop + redundant detection.
|
||||
const recentCalls = [] // { seq, name, argsKey }
|
||||
let lastToolError = null // { seq, name, callId }
|
||||
|
||||
for (let i = 0; i < list.length; i++) {
|
||||
const ev = list[i]
|
||||
if (!ev || typeof ev !== 'object') continue
|
||||
|
||||
// Loop / redundant — tool/call ordering
|
||||
if (ev.type === 'tool/call') {
|
||||
const name = ev.data && typeof ev.data.name === 'string' ? ev.data.name : ''
|
||||
const argsKey = _argsKey(ev.data)
|
||||
const seq = typeof ev.seq === 'number' ? ev.seq : null
|
||||
const cur = { seq, name, argsKey, callId: ev.data && ev.data.callId }
|
||||
|
||||
// loop-detected: look back at recentCalls tail for a run of
|
||||
// (name, argsKey) matches. When length >= loopN including cur,
|
||||
// flag cur's seq. Use `argsPrefix` (first 80 chars) so a call that
|
||||
// varies only in a trailing timestamp still matches.
|
||||
let run = 1
|
||||
for (let j = recentCalls.length - 1; j >= 0; j--) {
|
||||
const prev = recentCalls[j]
|
||||
if (prev.name === name && prev.argsKey === argsKey) run++
|
||||
else break
|
||||
}
|
||||
if (run >= loopN && seq !== null) {
|
||||
emit(all, bySeq, {
|
||||
signal: 'loop-detected',
|
||||
seq,
|
||||
meta: { source: 'heuristic', name, argsKey, run, priorSeqs: _lastNPriorSeqs(recentCalls, run - 1) },
|
||||
})
|
||||
}
|
||||
|
||||
// redundant-call: exact (name, argsKey) match within the last N calls,
|
||||
// NOT counting the immediate consecutive run (that's loop-detected).
|
||||
// Only flag when there is at least one intervening different call, so
|
||||
// we don't double-flag a pure loop.
|
||||
if (seq !== null && run < loopN) {
|
||||
for (let j = recentCalls.length - 1; j >= 0 && recentCalls.length - j <= window; j--) {
|
||||
const prev = recentCalls[j]
|
||||
if (prev.name === name && prev.argsKey === argsKey && (recentCalls.length - 1 - j) >= 1) {
|
||||
// Ensure at least one call between prev and cur has a different key.
|
||||
let interleaved = false
|
||||
for (let k = j + 1; k < recentCalls.length; k++) {
|
||||
if (recentCalls[k].name !== name || recentCalls[k].argsKey !== argsKey) { interleaved = true; break }
|
||||
}
|
||||
if (interleaved) {
|
||||
emit(all, bySeq, {
|
||||
signal: 'redundant-call',
|
||||
seq,
|
||||
meta: { source: 'heuristic', name, argsKey, priorSeq: prev.seq },
|
||||
})
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// plan-restart: same tool re-invoked after a recent tool/result error.
|
||||
// Guard: current callId must differ from the errored one (a wire replay
|
||||
// of the same call shouldn't count as a restart).
|
||||
if (lastToolError && lastToolError.name === name && seq !== null
|
||||
&& cur.callId !== lastToolError.callId) {
|
||||
emit(all, bySeq, {
|
||||
signal: 'plan-restart',
|
||||
seq,
|
||||
meta: { source: 'heuristic', name, priorErrorSeq: lastToolError.seq },
|
||||
})
|
||||
lastToolError = null
|
||||
}
|
||||
|
||||
recentCalls.push(cur)
|
||||
// Cap window to keep the scan O(N) — recentCalls only needs to hold
|
||||
// the last `window` entries.
|
||||
if (recentCalls.length > window * 2) recentCalls.splice(0, recentCalls.length - window * 2)
|
||||
continue
|
||||
}
|
||||
|
||||
// Tool errors
|
||||
if (ev.type === 'tool/result' && ev.data && ev.data.ok === false) {
|
||||
const seq = typeof ev.seq === 'number' ? ev.seq : null
|
||||
const name = ev.data && typeof ev.data.name === 'string' ? ev.data.name
|
||||
: _findCallName(list, ev.data && ev.data.callId)
|
||||
// Emit on the result seq AND the matching call seq. Timeline pairs
|
||||
// call+result into one bar keyed by the call's seq, and the Graph
|
||||
// absorbs the result into the call node — so the call seq is the
|
||||
// seq a reader actually sees. Result seq is kept for tree/detail
|
||||
// rendering that lists events individually.
|
||||
const callSeq = _findCallSeq(list, ev.data && ev.data.callId)
|
||||
const errorMeta = { source: 'wire', name, callId: ev.data.callId, error: ev.data.error || null }
|
||||
if (seq !== null) {
|
||||
emit(all, bySeq, { signal: 'tool-error', seq, meta: errorMeta })
|
||||
}
|
||||
if (callSeq !== null && callSeq !== seq) {
|
||||
emit(all, bySeq, { signal: 'tool-error', seq: callSeq, meta: errorMeta })
|
||||
}
|
||||
lastToolError = { seq, name, callId: ev.data && ev.data.callId }
|
||||
continue
|
||||
}
|
||||
|
||||
// plan-update: assistant text mentioning "new plan" / "revised plan".
|
||||
if (ev.type === 'assistant/message') {
|
||||
const seq = typeof ev.seq === 'number' ? ev.seq : null
|
||||
const text = _assistantText(ev.data)
|
||||
if (seq !== null && _looksLikePlanUpdate(text)) {
|
||||
emit(all, bySeq, {
|
||||
signal: 'plan-update',
|
||||
seq,
|
||||
meta: { source: 'heuristic', snippet: _trim(text, 80) },
|
||||
})
|
||||
}
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
return { bySeq, all }
|
||||
}
|
||||
|
||||
function emit(all, bySeq, sig) {
|
||||
// Dedup: don't emit the same signal kind on the same seq twice.
|
||||
const list = bySeq.get(sig.seq) || []
|
||||
for (const prev of list) if (prev.signal === sig.signal) return
|
||||
list.push(sig)
|
||||
bySeq.set(sig.seq, list)
|
||||
all.push(sig)
|
||||
}
|
||||
|
||||
function _argsKey(data) {
|
||||
if (!data) return ''
|
||||
// Prefer a stable JSON of the arguments; fall back to string coercion.
|
||||
// Truncate to 80 chars so we don't burn memory on huge blobs, and so
|
||||
// small trailing-timestamp diffs still coalesce.
|
||||
try {
|
||||
const raw = data.arguments != null ? data.arguments
|
||||
: data.args != null ? data.args : ''
|
||||
const s = typeof raw === 'string' ? raw : JSON.stringify(raw)
|
||||
return _trim(String(s || ''), 80)
|
||||
} catch (_) {
|
||||
return _trim(String(data.arguments || data.args || ''), 80)
|
||||
}
|
||||
}
|
||||
|
||||
function _assistantText(data) {
|
||||
if (!data) return ''
|
||||
if (Array.isArray(data.content)) {
|
||||
const parts = []
|
||||
for (const block of data.content) {
|
||||
if (block && typeof block === 'object' && block.type === 'text' && typeof block.text === 'string') {
|
||||
parts.push(block.text)
|
||||
}
|
||||
}
|
||||
return parts.join('\n')
|
||||
}
|
||||
if (typeof data.text === 'string') return data.text
|
||||
return ''
|
||||
}
|
||||
|
||||
function _looksLikePlanUpdate(text) {
|
||||
if (typeof text !== 'string' || !text) return false
|
||||
const lower = text.toLowerCase()
|
||||
for (const kw of PLAN_KEYWORDS) if (lower.includes(kw)) return true
|
||||
// Numbered plan intro: "1. …\n2. …" occurring in the first 200 chars.
|
||||
// Requires two adjacent numbered lines to reduce false positives from
|
||||
// enumerations inside prose.
|
||||
const head = text.slice(0, 400)
|
||||
const m = head.match(/(^|\n)\s*1\.\s.+\n\s*2\.\s/)
|
||||
if (m) return true
|
||||
return false
|
||||
}
|
||||
|
||||
function _findCallName(events, callId) {
|
||||
if (!callId) return ''
|
||||
for (let i = events.length - 1; i >= 0; i--) {
|
||||
const ev = events[i]
|
||||
if (ev && ev.type === 'tool/call' && ev.data && ev.data.callId === callId) {
|
||||
return typeof ev.data.name === 'string' ? ev.data.name : ''
|
||||
}
|
||||
}
|
||||
return ''
|
||||
}
|
||||
|
||||
function _findCallSeq(events, callId) {
|
||||
if (!callId) return null
|
||||
for (let i = events.length - 1; i >= 0; i--) {
|
||||
const ev = events[i]
|
||||
if (ev && ev.type === 'tool/call' && ev.data && ev.data.callId === callId
|
||||
&& typeof ev.seq === 'number') return ev.seq
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
function _lastNPriorSeqs(list, n) {
|
||||
const out = []
|
||||
for (let i = list.length - 1; i >= 0 && out.length < n; i--) {
|
||||
if (typeof list[i].seq === 'number') out.unshift(list[i].seq)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
function _trim(s, n) {
|
||||
if (typeof s !== 'string') return ''
|
||||
if (s.length <= n) return s
|
||||
return s.slice(0, n - 1) + '…'
|
||||
}
|
||||
|
||||
// Given a step-record (or array of them from the aggregator), flatten
|
||||
// the outputs+inputs+events into one seq-ordered event list and detect.
|
||||
// Consumers on the tri-view side prefer this shape because they hold
|
||||
// records, not raw events.
|
||||
function detectSignalsFromRecords(records, opts) {
|
||||
const list = Array.isArray(records) ? records : (records ? [records] : [])
|
||||
const flat = []
|
||||
const seen = new Set()
|
||||
for (const rec of list) {
|
||||
if (!rec) continue
|
||||
for (const bucket of ['inputs', 'outputs', 'events']) {
|
||||
const arr = rec[bucket]
|
||||
if (!Array.isArray(arr)) continue
|
||||
for (const ev of arr) {
|
||||
if (!ev || typeof ev !== 'object') continue
|
||||
const key = typeof ev.seq === 'number' ? `s${ev.seq}` : `t${ev.type}|${flat.length}`
|
||||
if (seen.has(key)) continue
|
||||
seen.add(key)
|
||||
flat.push(ev)
|
||||
}
|
||||
}
|
||||
}
|
||||
flat.sort(function (a, b) {
|
||||
const sa = typeof a.seq === 'number' ? a.seq : Infinity
|
||||
const sb = typeof b.seq === 'number' ? b.seq : Infinity
|
||||
return sa - sb
|
||||
})
|
||||
return detectSignals(flat, opts)
|
||||
}
|
||||
|
||||
// Human-readable label for a signal — used by tooltip helpers on both
|
||||
// Timeline/Graph badges and the main-flow chip.
|
||||
function labelFor(signal) {
|
||||
switch (signal) {
|
||||
case 'loop-detected': return 'Loop detected'
|
||||
case 'redundant-call': return 'Redundant call'
|
||||
case 'plan-update': return 'Plan update'
|
||||
case 'plan-restart': return 'Plan restart'
|
||||
case 'tool-error': return 'Tool error'
|
||||
default: return signal || 'Signal'
|
||||
}
|
||||
}
|
||||
|
||||
// CSS class-name suffix — kept aligned with the badge styles in style.css.
|
||||
function classFor(signal) {
|
||||
switch (signal) {
|
||||
case 'loop-detected': return 'sig-loop'
|
||||
case 'redundant-call': return 'sig-redundant'
|
||||
case 'plan-update': return 'sig-plan'
|
||||
case 'plan-restart': return 'sig-plan-restart'
|
||||
case 'tool-error': return 'sig-error'
|
||||
default: return 'sig-generic'
|
||||
}
|
||||
}
|
||||
|
||||
function tooltipFor(sig) {
|
||||
if (!sig) return ''
|
||||
const parts = [labelFor(sig.signal)]
|
||||
const m = sig.meta || {}
|
||||
if (sig.signal === 'loop-detected' && m.name) {
|
||||
parts.push(`${m.run || '?'} × ${m.name}`)
|
||||
if (Array.isArray(m.priorSeqs) && m.priorSeqs.length) {
|
||||
parts.push(`prior seq ${m.priorSeqs.join(', ')}`)
|
||||
}
|
||||
} else if (sig.signal === 'redundant-call' && m.name) {
|
||||
parts.push(`${m.name}, matches seq ${m.priorSeq}`)
|
||||
} else if (sig.signal === 'plan-update' && m.snippet) {
|
||||
parts.push(`“${m.snippet}”`)
|
||||
} else if (sig.signal === 'plan-restart' && m.name) {
|
||||
parts.push(`retry ${m.name} after seq ${m.priorErrorSeq}`)
|
||||
} else if (sig.signal === 'tool-error' && m.name) {
|
||||
parts.push(m.name + (m.error ? `: ${_trim(String(m.error), 60)}` : ''))
|
||||
}
|
||||
if (m.source === 'heuristic') parts.push('(heuristic)')
|
||||
return parts.join(' · ')
|
||||
}
|
||||
|
||||
const api = {
|
||||
detectSignals,
|
||||
detectSignalsFromRecords,
|
||||
labelFor,
|
||||
classFor,
|
||||
tooltipFor,
|
||||
// Exposed for tests / diagnostics only.
|
||||
_internals: { _argsKey, _looksLikePlanUpdate },
|
||||
}
|
||||
|
||||
if (typeof module !== 'undefined' && module.exports) {
|
||||
module.exports = api
|
||||
}
|
||||
if (typeof window !== 'undefined') {
|
||||
window.__dshTraceSignalDetect = api
|
||||
}
|
||||
})()
|
||||
@@ -338,6 +338,45 @@ function renderTimeline(doc, input, opts) {
|
||||
svg.appendChild(g)
|
||||
}
|
||||
|
||||
// Signal badges — small filled circles hanging off the right edge of the
|
||||
// label column for any row whose seq matches a detected/emitted signal.
|
||||
// The `signals` option is a bySeq Map from trace-signal-detect.js. We keep
|
||||
// this render entirely optional so tests + non-signal callers stay
|
||||
// untouched; when no map is supplied nothing is drawn. See L-2 in
|
||||
// docs/upstream-ledger.md for the RFC that would move detection upstream.
|
||||
if (options.signals && typeof options.signals.get === 'function') {
|
||||
const badgesG = svgGroup(doc, 'trace-timeline-badges')
|
||||
for (let i = 0; i < rows.length; i++) {
|
||||
const row = rows[i]
|
||||
if (row.seq === null) continue
|
||||
const sigs = options.signals.get(row.seq)
|
||||
if (!sigs || !sigs.length) continue
|
||||
const y = topPad + i * rowH + rowH * 0.5
|
||||
// Stack badges horizontally so multiple signals on one seq stay
|
||||
// readable. The seed x sits just inside the label column so the
|
||||
// badge visually "labels" the row without eating bar space.
|
||||
let bx = labelW - 10
|
||||
for (const sig of sigs) {
|
||||
const cls = (typeof window !== 'undefined' && window.__dshTraceSignalDetect)
|
||||
? window.__dshTraceSignalDetect.classFor(sig.signal)
|
||||
: (typeof require !== 'undefined' ? require('./trace-signal-detect.js').classFor(sig.signal) : 'sig-generic')
|
||||
const tip = (typeof window !== 'undefined' && window.__dshTraceSignalDetect)
|
||||
? window.__dshTraceSignalDetect.tooltipFor(sig)
|
||||
: (typeof require !== 'undefined' ? require('./trace-signal-detect.js').tooltipFor(sig) : sig.signal)
|
||||
const badge = svgEl(doc, 'circle', {
|
||||
cx: bx, cy: y, r: 4,
|
||||
class: `trace-timeline-signal-badge ${cls}`,
|
||||
})
|
||||
const title = svgEl(doc, 'title', {})
|
||||
title.textContent = tip
|
||||
badge.appendChild(title)
|
||||
badgesG.appendChild(badge)
|
||||
bx -= 10
|
||||
}
|
||||
}
|
||||
svg.appendChild(badgesG)
|
||||
}
|
||||
|
||||
// Live cursor: hairline at nowMs during streaming.
|
||||
if (typeof options.nowMs === 'number' && Number.isFinite(options.nowMs)) {
|
||||
const nx = labelW + xScale(options.nowMs)
|
||||
|
||||
@@ -117,6 +117,12 @@
|
||||
}
|
||||
openDetailForSeq(seq, meta)
|
||||
}
|
||||
// Signals map (bySeq) is derived once from the tri-view's records and
|
||||
// passed to both Timeline + Graph so badge/ring placement stays
|
||||
// consistent across view switches. See trace-signal-detect.js and
|
||||
// docs/upstream-ledger.md L-2 for the RFC that would replace this
|
||||
// renderer-side detection with wire-emitted `trace/signal` events.
|
||||
const signalMap = _computeSignals(spec.records)
|
||||
function ensureTimeline() {
|
||||
if (builtTimeline) return
|
||||
builtTimeline = true
|
||||
@@ -126,6 +132,7 @@
|
||||
onSeqClick: handleNodeClick,
|
||||
nowMs: typeof spec.nowMs === 'number' ? spec.nowMs : undefined,
|
||||
width: spec.scope === 'session' ? 860 : 720,
|
||||
signals: signalMap,
|
||||
})
|
||||
panelEls.timeline.appendChild(el)
|
||||
}
|
||||
@@ -134,7 +141,10 @@
|
||||
builtGraph = true
|
||||
const G = (typeof window !== 'undefined' && window.__dshTraceGraph) || null
|
||||
if (!G) { panelEls.graph.textContent = 'trace-graph.js not loaded'; return }
|
||||
const el = G.renderGraph(doc, spec.records || [], { onSeqClick: handleNodeClick })
|
||||
const el = G.renderGraph(doc, spec.records || [], {
|
||||
onSeqClick: handleNodeClick,
|
||||
signals: signalMap,
|
||||
})
|
||||
panelEls.graph.appendChild(el)
|
||||
}
|
||||
|
||||
@@ -254,6 +264,18 @@
|
||||
return agg.aggregateSteps(Array.isArray(events) ? events : [])
|
||||
}
|
||||
|
||||
// Derive the signals bySeq map from a set of step-records. Falls back to
|
||||
// an empty Map when the detector module isn't loaded, so a lean test env
|
||||
// that only pulls tri-view keeps working.
|
||||
function _computeSignals(records) {
|
||||
const SD = (typeof window !== 'undefined' && window.__dshTraceSignalDetect) || null
|
||||
if (!SD || typeof SD.detectSignalsFromRecords !== 'function') return new Map()
|
||||
try {
|
||||
const res = SD.detectSignalsFromRecords(records)
|
||||
return (res && res.bySeq) || new Map()
|
||||
} catch (_) { return new Map() }
|
||||
}
|
||||
|
||||
// Pick the step-record whose seq range contains `seq`, preferring an
|
||||
// exact startSeq match. Falls back to the closest containing range, then
|
||||
// to the last record before `seq`. Returns null when nothing matches.
|
||||
|
||||
194
examples/desktop/test/trace-signal-detect.test.js
Normal file
194
examples/desktop/test/trace-signal-detect.test.js
Normal file
@@ -0,0 +1,194 @@
|
||||
// trace-signal-detect.test.js — heuristic detector unit tests.
|
||||
//
|
||||
// Detector is pure and used by both the tri-view SVG overlays and the main
|
||||
// flow marker chips (see renderer.js `applyTurnSignalChips`). Every branch
|
||||
// has a fixture-driven assertion so a future refactor breaks visibly.
|
||||
|
||||
'use strict'
|
||||
|
||||
const test = require('node:test')
|
||||
const assert = require('node:assert')
|
||||
|
||||
const SD = require('../src/renderer/trace-signal-detect.js')
|
||||
|
||||
test('loop-detected fires after N consecutive same-tool same-args calls', () => {
|
||||
const events = []
|
||||
for (let i = 0; i < 3; i++) {
|
||||
events.push({
|
||||
type: 'tool/call', seq: 10 + i,
|
||||
data: { name: 'fs.read', arguments: '{"path":"a.ts"}', callId: `c${i}` },
|
||||
})
|
||||
}
|
||||
const { bySeq, all } = SD.detectSignals(events, { loopN: 3 })
|
||||
const loops = all.filter(s => s.signal === 'loop-detected')
|
||||
assert.strictEqual(loops.length, 1, 'exactly one loop-detected on the 3rd call')
|
||||
assert.strictEqual(loops[0].seq, 12)
|
||||
assert.strictEqual(loops[0].meta.run, 3)
|
||||
assert.strictEqual(loops[0].meta.name, 'fs.read')
|
||||
assert.ok(bySeq.get(12).find(s => s.signal === 'loop-detected'))
|
||||
})
|
||||
|
||||
test('loop-detected does not fire when args differ', () => {
|
||||
const events = [
|
||||
{ type: 'tool/call', seq: 1, data: { name: 'fs.read', arguments: '{"path":"a"}', callId: '1' } },
|
||||
{ type: 'tool/call', seq: 2, data: { name: 'fs.read', arguments: '{"path":"b"}', callId: '2' } },
|
||||
{ type: 'tool/call', seq: 3, data: { name: 'fs.read', arguments: '{"path":"c"}', callId: '3' } },
|
||||
]
|
||||
const { all } = SD.detectSignals(events, { loopN: 3 })
|
||||
const loops = all.filter(s => s.signal === 'loop-detected')
|
||||
assert.strictEqual(loops.length, 0, 'name matches but args differ — not a loop')
|
||||
})
|
||||
|
||||
test('redundant-call fires when a call repeats with an interleaved other call between', () => {
|
||||
const events = [
|
||||
{ type: 'tool/call', seq: 1, data: { name: 'fs.read', arguments: '{"path":"a"}', callId: '1' } },
|
||||
{ type: 'tool/call', seq: 2, data: { name: 'bash', arguments: '{"cmd":"ls"}', callId: '2' } },
|
||||
{ type: 'tool/call', seq: 3, data: { name: 'fs.read', arguments: '{"path":"a"}', callId: '3' } },
|
||||
]
|
||||
const { all } = SD.detectSignals(events)
|
||||
const red = all.filter(s => s.signal === 'redundant-call')
|
||||
assert.strictEqual(red.length, 1)
|
||||
assert.strictEqual(red[0].seq, 3)
|
||||
assert.strictEqual(red[0].meta.priorSeq, 1)
|
||||
})
|
||||
|
||||
test('redundant-call does NOT fire when the repeat is consecutive (that\'s a loop, not redundancy)', () => {
|
||||
const events = [
|
||||
{ type: 'tool/call', seq: 1, data: { name: 'fs.read', arguments: '{"p":"a"}', callId: '1' } },
|
||||
{ type: 'tool/call', seq: 2, data: { name: 'fs.read', arguments: '{"p":"a"}', callId: '2' } },
|
||||
]
|
||||
const { all } = SD.detectSignals(events, { loopN: 3 })
|
||||
const red = all.filter(s => s.signal === 'redundant-call')
|
||||
assert.strictEqual(red.length, 0, '2 same-in-a-row is not yet a loop and not yet redundant')
|
||||
})
|
||||
|
||||
test('plan-update fires on assistant text with "new plan" keyword', () => {
|
||||
const events = [
|
||||
{
|
||||
type: 'assistant/message', seq: 20,
|
||||
data: { content: [{ type: 'text', text: 'Okay, here is the new plan: first read the file, then edit it.' }] },
|
||||
},
|
||||
]
|
||||
const { all } = SD.detectSignals(events)
|
||||
const plan = all.filter(s => s.signal === 'plan-update')
|
||||
assert.strictEqual(plan.length, 1)
|
||||
assert.strictEqual(plan[0].seq, 20)
|
||||
assert.match(plan[0].meta.snippet, /new plan/i)
|
||||
assert.strictEqual(plan[0].meta.source, 'heuristic')
|
||||
})
|
||||
|
||||
test('plan-update fires on a numbered-plan intro (two adjacent numbered lines)', () => {
|
||||
const events = [
|
||||
{
|
||||
type: 'assistant/message', seq: 30,
|
||||
data: { content: [{ type: 'text', text: '1. Read main.ts\n2. Edit imports\n3. Verify.' }] },
|
||||
},
|
||||
]
|
||||
const { all } = SD.detectSignals(events)
|
||||
const plan = all.filter(s => s.signal === 'plan-update')
|
||||
assert.strictEqual(plan.length, 1)
|
||||
assert.strictEqual(plan[0].seq, 30)
|
||||
})
|
||||
|
||||
test('plan-update does NOT fire on plain prose without plan keywords', () => {
|
||||
const events = [
|
||||
{
|
||||
type: 'assistant/message', seq: 40,
|
||||
data: { content: [{ type: 'text', text: 'The file looks fine. No changes needed.' }] },
|
||||
},
|
||||
]
|
||||
const { all } = SD.detectSignals(events)
|
||||
assert.strictEqual(all.filter(s => s.signal === 'plan-update').length, 0)
|
||||
})
|
||||
|
||||
test('tool-error signal fires on tool/result with ok:false — on BOTH the result seq and the matching call seq', () => {
|
||||
const events = [
|
||||
{ type: 'tool/call', seq: 5, data: { name: 'bash', arguments: 'ls /nope', callId: 'c1' } },
|
||||
{ type: 'tool/result', seq: 6, data: { callId: 'c1', ok: false, error: 'ENOENT: /nope' } },
|
||||
]
|
||||
const { all, bySeq } = SD.detectSignals(events)
|
||||
const err = all.filter(s => s.signal === 'tool-error')
|
||||
assert.strictEqual(err.length, 2, 'one badge on the call seq, one on the result seq')
|
||||
const seqs = err.map(e => e.seq).sort()
|
||||
assert.deepStrictEqual(seqs, [5, 6])
|
||||
assert.ok(bySeq.get(5).find(s => s.signal === 'tool-error'), 'call seq carries the tool-error signal')
|
||||
assert.ok(bySeq.get(6).find(s => s.signal === 'tool-error'), 'result seq carries the tool-error signal')
|
||||
for (const e of err) assert.strictEqual(e.meta.name, 'bash')
|
||||
})
|
||||
|
||||
test('plan-restart fires when the same tool is re-invoked after an error', () => {
|
||||
const events = [
|
||||
{ type: 'tool/call', seq: 1, data: { name: 'bash', arguments: 'ls /nope', callId: 'c1' } },
|
||||
{ type: 'tool/result', seq: 2, data: { callId: 'c1', ok: false, error: 'ENOENT' } },
|
||||
{ type: 'tool/call', seq: 3, data: { name: 'bash', arguments: 'ls /tmp', callId: 'c2' } },
|
||||
]
|
||||
const { all } = SD.detectSignals(events)
|
||||
const restart = all.filter(s => s.signal === 'plan-restart')
|
||||
assert.strictEqual(restart.length, 1)
|
||||
assert.strictEqual(restart[0].seq, 3)
|
||||
assert.strictEqual(restart[0].meta.priorErrorSeq, 2)
|
||||
})
|
||||
|
||||
test('wire-side signals (trace/signal events) are consumed verbatim and marked source:wire', () => {
|
||||
const events = [
|
||||
{
|
||||
type: 'trace/signal', seq: 100,
|
||||
data: { signal: 'loop-detected', name: 'fs.read', run: 5 },
|
||||
},
|
||||
]
|
||||
const { all } = SD.detectSignals(events)
|
||||
assert.strictEqual(all.length, 1)
|
||||
assert.strictEqual(all[0].signal, 'loop-detected')
|
||||
assert.strictEqual(all[0].seq, 100)
|
||||
assert.strictEqual(all[0].meta.source, 'wire')
|
||||
})
|
||||
|
||||
test('detectSignalsFromRecords flattens step records back to a seq-ordered event list', () => {
|
||||
const rec = {
|
||||
turn: 1, step: 0, startSeq: 10, endSeq: 15,
|
||||
inputs: [],
|
||||
outputs: [
|
||||
{ type: 'tool/call', seq: 12, data: { name: 'fs.read', arguments: '{"p":"a"}', callId: 'c1' } },
|
||||
],
|
||||
events: [
|
||||
{ type: 'tool/call', seq: 12, data: { name: 'fs.read', arguments: '{"p":"a"}', callId: 'c1' } },
|
||||
{ type: 'tool/call', seq: 13, data: { name: 'fs.read', arguments: '{"p":"a"}', callId: 'c2' } },
|
||||
{ type: 'tool/call', seq: 14, data: { name: 'fs.read', arguments: '{"p":"a"}', callId: 'c3' } },
|
||||
],
|
||||
}
|
||||
const { all } = SD.detectSignalsFromRecords(rec, { loopN: 3 })
|
||||
const loops = all.filter(s => s.signal === 'loop-detected')
|
||||
assert.strictEqual(loops.length, 1, 'dedup on seq means only one entry per event')
|
||||
assert.strictEqual(loops[0].seq, 14)
|
||||
})
|
||||
|
||||
test('labelFor and classFor return the expected mappings', () => {
|
||||
assert.strictEqual(SD.labelFor('loop-detected'), 'Loop detected')
|
||||
assert.strictEqual(SD.labelFor('redundant-call'), 'Redundant call')
|
||||
assert.strictEqual(SD.labelFor('plan-update'), 'Plan update')
|
||||
assert.strictEqual(SD.labelFor('plan-restart'), 'Plan restart')
|
||||
assert.strictEqual(SD.labelFor('tool-error'), 'Tool error')
|
||||
|
||||
assert.strictEqual(SD.classFor('loop-detected'), 'sig-loop')
|
||||
assert.strictEqual(SD.classFor('redundant-call'), 'sig-redundant')
|
||||
assert.strictEqual(SD.classFor('plan-update'), 'sig-plan')
|
||||
assert.strictEqual(SD.classFor('plan-restart'), 'sig-plan-restart')
|
||||
assert.strictEqual(SD.classFor('tool-error'), 'sig-error')
|
||||
})
|
||||
|
||||
test('tooltipFor produces a readable tooltip for each signal kind', () => {
|
||||
const tip1 = SD.tooltipFor({
|
||||
signal: 'loop-detected', seq: 12,
|
||||
meta: { source: 'heuristic', name: 'fs.read', run: 3, priorSeqs: [10, 11] },
|
||||
})
|
||||
assert.match(tip1, /Loop detected/)
|
||||
assert.match(tip1, /fs\.read/)
|
||||
assert.match(tip1, /heuristic/)
|
||||
|
||||
const tip2 = SD.tooltipFor({
|
||||
signal: 'plan-update', seq: 20,
|
||||
meta: { source: 'heuristic', snippet: 'here is the new plan…' },
|
||||
})
|
||||
assert.match(tip2, /Plan update/)
|
||||
assert.match(tip2, /new plan/i)
|
||||
})
|
||||
141
examples/desktop/test/trace-signal-overlay.test.js
Normal file
141
examples/desktop/test/trace-signal-overlay.test.js
Normal file
@@ -0,0 +1,141 @@
|
||||
// trace-signal-overlay.test.js — Timeline + Graph rendering with a
|
||||
// `signals` bySeq Map should drop badges/rings and expose the tooltip.
|
||||
|
||||
'use strict'
|
||||
|
||||
const test = require('node:test')
|
||||
const assert = require('node:assert')
|
||||
|
||||
const T = require('../src/renderer/trace-timeline.js')
|
||||
const G = require('../src/renderer/trace-graph.js')
|
||||
const SD = require('../src/renderer/trace-signal-detect.js')
|
||||
|
||||
function makeDoc() {
|
||||
function makeEl(nsOrTag, tag) {
|
||||
const cls = { _s: new Set(),
|
||||
add(c) { this._s.add(c) }, remove(c) { this._s.delete(c) },
|
||||
toggle(c, on) { if (on) this._s.add(c); else this._s.delete(c) },
|
||||
contains(c) { return this._s.has(c) },
|
||||
}
|
||||
const el = {
|
||||
tagName: (tag || nsOrTag).toUpperCase(),
|
||||
_children: [],
|
||||
_attrs: {},
|
||||
_listeners: {},
|
||||
dataset: {},
|
||||
style: {},
|
||||
textContent: '',
|
||||
hidden: false,
|
||||
get className() { return Array.from(cls._s).join(' ') },
|
||||
set className(v) {
|
||||
cls._s.clear()
|
||||
String(v || '').split(/\s+/).forEach(x => x && cls._s.add(x))
|
||||
},
|
||||
classList: cls,
|
||||
appendChild(c) { this._children.push(c); return c },
|
||||
append(...cs) { for (const c of cs) this._children.push(c) },
|
||||
setAttribute(k, v) {
|
||||
this._attrs[k] = String(v)
|
||||
if (k === 'class') {
|
||||
cls._s.clear()
|
||||
String(v || '').split(/\s+/).forEach(x => x && cls._s.add(x))
|
||||
}
|
||||
},
|
||||
getAttribute(k) { return this._attrs[k] },
|
||||
addEventListener(name, fn) { (this._listeners[name] = this._listeners[name] || []).push(fn) },
|
||||
querySelector(sel) { return null },
|
||||
querySelectorAll() { return [] },
|
||||
firstChild: null,
|
||||
insertBefore(node) { this._children.unshift(node); return node },
|
||||
outerHTML: '',
|
||||
}
|
||||
return el
|
||||
}
|
||||
return {
|
||||
createElement(tag) { return makeEl(tag) },
|
||||
createElementNS(ns, tag) { return makeEl(ns, tag) },
|
||||
body: makeEl('body'),
|
||||
}
|
||||
}
|
||||
|
||||
function collectByClass(root, cls) {
|
||||
const found = []
|
||||
const stack = [root]
|
||||
while (stack.length) {
|
||||
const cur = stack.pop()
|
||||
if (cur && cur.classList && cur.classList.contains(cls)) found.push(cur)
|
||||
if (cur && cur._children) for (const c of cur._children) stack.push(c)
|
||||
}
|
||||
return found
|
||||
}
|
||||
|
||||
test('renderTimeline drops one signal badge per row whose seq appears in the map', () => {
|
||||
const doc = makeDoc()
|
||||
const rec = {
|
||||
turn: 1, step: 0, startSeq: 10, endSeq: 14,
|
||||
startTime: 1000, endTime: 1500, durationMs: 500,
|
||||
summary: 'read', inputs: [], outputs: [],
|
||||
events: [
|
||||
{ type: 'tool/call', seq: 11, time: 1050, data: { name: 'fs.read', arguments: '{"p":"a"}', callId: 'c1' } },
|
||||
{ type: 'tool/call', seq: 12, time: 1100, data: { name: 'fs.read', arguments: '{"p":"a"}', callId: 'c2' } },
|
||||
{ type: 'tool/call', seq: 13, time: 1150, data: { name: 'fs.read', arguments: '{"p":"a"}', callId: 'c3' } },
|
||||
],
|
||||
}
|
||||
const { bySeq } = SD.detectSignalsFromRecords(rec, { loopN: 3 })
|
||||
const el = T.renderTimeline(doc, rec, { signals: bySeq })
|
||||
const badges = collectByClass(el, 'trace-timeline-signal-badge')
|
||||
assert.ok(badges.length >= 1, 'at least one signal badge drawn for the loop-detected seq (12)')
|
||||
// The badge for the loop-detected signal on seq 12 should carry the sig-loop class.
|
||||
const loopBadges = badges.filter(b => b.classList.contains('sig-loop'))
|
||||
assert.strictEqual(loopBadges.length, 1)
|
||||
// Tooltip <title> child is present
|
||||
const title = loopBadges[0]._children.find(c => c.tagName === 'TITLE')
|
||||
assert.ok(title, 'badge has a <title> tooltip child')
|
||||
assert.match(title.textContent, /Loop/)
|
||||
})
|
||||
|
||||
test('renderTimeline draws no badges when signals map is empty', () => {
|
||||
const doc = makeDoc()
|
||||
const rec = {
|
||||
turn: 1, step: 0, startSeq: 10, endSeq: 12,
|
||||
startTime: 1000, endTime: 1200,
|
||||
inputs: [], outputs: [],
|
||||
events: [
|
||||
{ type: 'tool/call', seq: 11, time: 1050, data: { name: 'fs.read', arguments: '{}', callId: 'c1' } },
|
||||
],
|
||||
}
|
||||
const el = T.renderTimeline(doc, rec, { signals: new Map() })
|
||||
const badges = collectByClass(el, 'trace-timeline-signal-badge')
|
||||
assert.strictEqual(badges.length, 0)
|
||||
})
|
||||
|
||||
test('renderGraph puts a colored ring around a node whose seq matches a signal', () => {
|
||||
const doc = makeDoc()
|
||||
const rec = {
|
||||
turn: 1, step: 0, startSeq: 10, endSeq: 14,
|
||||
startTime: 1000, endTime: 1500,
|
||||
inputs: [], outputs: [],
|
||||
events: [
|
||||
{ type: 'tool/call', seq: 11, time: 1050, data: { name: 'fs.read', arguments: '{"p":"a"}', callId: 'c1' } },
|
||||
{ type: 'tool/result', seq: 12, time: 1080, data: { callId: 'c1', ok: false, error: 'ENOENT' } },
|
||||
],
|
||||
}
|
||||
const { bySeq } = SD.detectSignalsFromRecords(rec)
|
||||
const el = G.renderGraph(doc, rec, { signals: bySeq })
|
||||
const rings = collectByClass(el, 'trace-graph-signal-ring')
|
||||
assert.ok(rings.length >= 1, 'at least one signal ring around the tool-error seq')
|
||||
const errRings = rings.filter(r => r.classList.contains('sig-error'))
|
||||
assert.strictEqual(errRings.length, 1, 'ring wears the sig-error class')
|
||||
})
|
||||
|
||||
test('renderGraph draws no rings without a signals map', () => {
|
||||
const doc = makeDoc()
|
||||
const rec = {
|
||||
turn: 1, step: 0, startSeq: 10, endSeq: 12,
|
||||
startTime: 1000, endTime: 1100,
|
||||
inputs: [], outputs: [],
|
||||
events: [{ type: 'tool/call', seq: 11, time: 1050, data: { name: 'x', arguments: '{}', callId: 'c' } }],
|
||||
}
|
||||
const el = G.renderGraph(doc, rec, {})
|
||||
assert.strictEqual(collectByClass(el, 'trace-graph-signal-ring').length, 0)
|
||||
})
|
||||
Reference in New Issue
Block a user