feat(web): turn speed metrics and composer context meter
Assistant footers and the stats line gain TTFT/tok-per-second readings folded from step timings; context occupancy moves off the stats line onto a composer ring whose panel shows a heuristic system/tools/messages breakdown from the new token-meter contextBreakdown session projection.
This commit is contained in:
87
packages/llm/token-meter/src/breakdown-projection.ts
Normal file
87
packages/llm/token-meter/src/breakdown-projection.ts
Normal file
@@ -0,0 +1,87 @@
|
||||
/**
|
||||
* Pure fold for the heuristic context-composition projection: system prompt
|
||||
* and tool schemas from the newest request envelope, conversation from the
|
||||
* live surface. Prices with the same shared estimator as the meter service,
|
||||
* so the three figures match `measure()`'s heuristic vocabulary exactly.
|
||||
*/
|
||||
|
||||
import { z } from 'zod'
|
||||
import { canonicalHeader, deriveEventMessage, isSurfaceEvent } from '@deepseek-ai/dsh-session'
|
||||
import type { ProjectionDefinition } from '@deepseek-ai/dsh-session-projection'
|
||||
import { estimateMessage, estimateSystemTokens, estimateToolsTokens } from './estimate.ts'
|
||||
// Import for the `contextBreakdown` SessionProjectionMap key merge.
|
||||
import type {} from './projection.ts'
|
||||
|
||||
/** One priced surface node (plain JSON for the persisted projection cache). */
|
||||
interface BreakdownSurfaceNode {
|
||||
seq: number
|
||||
tokens: number
|
||||
}
|
||||
|
||||
interface ContextBreakdownState {
|
||||
systemTokens: number
|
||||
toolsTokens: number
|
||||
messageTokens: number
|
||||
surface: BreakdownSurfaceNode[]
|
||||
}
|
||||
|
||||
const breakdownSchema = z.object({
|
||||
systemTokens: z.number().int().nonnegative(),
|
||||
toolsTokens: z.number().int().nonnegative(),
|
||||
messageTokens: z.number().int().nonnegative(),
|
||||
}).strict()
|
||||
|
||||
/**
|
||||
* Token-meter's context-composition projection unit.
|
||||
*
|
||||
* Envelope figures are last-wins per `request/header`; the message figure
|
||||
* folds surface appends and positional replacements, so compaction shrinks it
|
||||
* the same way it shrinks the next request. Committed logs are
|
||||
* surface-validated at append time, so an unresolvable replace range here is
|
||||
* log corruption and fails loud rather than skipping the event.
|
||||
*/
|
||||
export const contextBreakdownProjectionDefinition:
|
||||
ProjectionDefinition<'contextBreakdown', ContextBreakdownState> = {
|
||||
key: 'contextBreakdown',
|
||||
schema: breakdownSchema,
|
||||
init: () => ({ systemTokens: 0, toolsTokens: 0, messageTokens: 0, surface: [] }),
|
||||
apply: (state, event) => {
|
||||
if (event.type === 'request/header') {
|
||||
const header = canonicalHeader(event.data.header)
|
||||
const systemTokens = estimateSystemTokens(header)
|
||||
const toolsTokens = estimateToolsTokens(header)
|
||||
if (systemTokens === state.systemTokens && toolsTokens === state.toolsTokens) return state
|
||||
return { ...state, systemTokens, toolsTokens }
|
||||
}
|
||||
if (!isSurfaceEvent(event)) return state
|
||||
const message = deriveEventMessage(event)
|
||||
const tokens = message === null ? 0 : estimateMessage(message)
|
||||
const op = event.surfaceOp
|
||||
if (op === 'append') {
|
||||
return {
|
||||
...state,
|
||||
messageTokens: state.messageTokens + tokens,
|
||||
surface: [...state.surface, { seq: event.seq, tokens }],
|
||||
}
|
||||
}
|
||||
const startIdx = state.surface.findIndex(node => node.seq === op.start)
|
||||
const endIdx = state.surface.findIndex(node => node.seq === op.end)
|
||||
if (startIdx === -1 || endIdx === -1 || startIdx > endIdx) {
|
||||
throw new Error(
|
||||
`context breakdown: replace at seq ${event.seq} has invalid current range ${op.start}-${op.end}`,
|
||||
)
|
||||
}
|
||||
const removed = state.surface
|
||||
.slice(startIdx, endIdx + 1)
|
||||
.reduce((total, node) => total + node.tokens, 0)
|
||||
const surface = [...state.surface]
|
||||
surface.splice(startIdx, endIdx - startIdx + 1, { seq: event.seq, tokens })
|
||||
return {
|
||||
...state,
|
||||
messageTokens: state.messageTokens + tokens - removed,
|
||||
surface,
|
||||
}
|
||||
},
|
||||
view: ({ systemTokens, toolsTokens, messageTokens }) => ({ systemTokens, toolsTokens, messageTokens }),
|
||||
stateVersion: 1,
|
||||
}
|
||||
87
packages/llm/token-meter/src/estimate.ts
Normal file
87
packages/llm/token-meter/src/estimate.ts
Normal file
@@ -0,0 +1,87 @@
|
||||
/**
|
||||
* Fixed-density heuristic token pricing shared by the meter service and the
|
||||
* pure context-breakdown projection, so both surfaces price identical content
|
||||
* to identical numbers.
|
||||
*
|
||||
* @module @deepseek-ai/dsh-token-meter/estimate
|
||||
*/
|
||||
|
||||
import type { ContentBlock, Message } from '@deepseek-ai/dsh-llm'
|
||||
import type { EpochHeader } from '@deepseek-ai/dsh-session'
|
||||
|
||||
/** Fixed text-density estimate used until exact tokenization is needed. */
|
||||
const CHARS_PER_TOKEN = 4
|
||||
|
||||
/** Per-block structural overhead for JSON framing and type tags. */
|
||||
const BLOCK_OVERHEAD = 4
|
||||
|
||||
/** Role-field framing overhead added to every priced message. */
|
||||
export const ROLE_OVERHEAD = 4
|
||||
|
||||
/**
|
||||
* Price content blocks recursively under the fixed density heuristic.
|
||||
* @param blocks - content blocks to price without mutation.
|
||||
* @returns heuristic tokens including per-block structural overhead.
|
||||
*/
|
||||
export function estimateContent(blocks: readonly ContentBlock[]): number {
|
||||
let tokens = 0
|
||||
for (const block of blocks) {
|
||||
switch (block.type) {
|
||||
case 'text':
|
||||
case 'reasoning':
|
||||
tokens += Math.ceil(block.text.length / CHARS_PER_TOKEN) + BLOCK_OVERHEAD
|
||||
break
|
||||
case 'tool-call':
|
||||
tokens += Math.ceil(block.name.length / CHARS_PER_TOKEN)
|
||||
+ Math.ceil(block.arguments.length / CHARS_PER_TOKEN)
|
||||
+ BLOCK_OVERHEAD
|
||||
break
|
||||
case 'tool-result':
|
||||
tokens += estimateContent(block.content) + BLOCK_OVERHEAD
|
||||
break
|
||||
default:
|
||||
// ContentBlockMap is merge-extensible; unknown blocks retain a
|
||||
// conservative structural JSON price under the fixed heuristic.
|
||||
tokens += BLOCK_OVERHEAD + Math.ceil(JSON.stringify(block).length / CHARS_PER_TOKEN)
|
||||
}
|
||||
}
|
||||
return tokens
|
||||
}
|
||||
|
||||
/**
|
||||
* Heuristically price one model-visible message.
|
||||
* @param message - message to price without mutation.
|
||||
* @returns content and role-framing tokens under the fixed heuristic.
|
||||
*/
|
||||
export function estimateMessage(message: Message): number {
|
||||
return estimateContent(message.content) + ROLE_OVERHEAD
|
||||
}
|
||||
|
||||
/**
|
||||
* Price the system-prompt part of a canonical request envelope.
|
||||
* @param header - canonical envelope, or undefined before any request.
|
||||
* @returns heuristic system-prompt tokens; 0 when absent.
|
||||
*/
|
||||
export function estimateSystemTokens(header: EpochHeader | undefined): number {
|
||||
if (header?.system === undefined) return 0
|
||||
return Math.ceil(header.system.length / CHARS_PER_TOKEN) + ROLE_OVERHEAD
|
||||
}
|
||||
|
||||
/**
|
||||
* Price the tool-schema part of a canonical request envelope.
|
||||
* @param header - canonical envelope, or undefined before any request.
|
||||
* @returns heuristic tool-schema tokens; 0 when absent or empty.
|
||||
*/
|
||||
export function estimateToolsTokens(header: EpochHeader | undefined): number {
|
||||
if (header?.tools === undefined || header.tools.length === 0) return 0
|
||||
return Math.ceil(JSON.stringify(header.tools).length / CHARS_PER_TOKEN) + BLOCK_OVERHEAD
|
||||
}
|
||||
|
||||
/**
|
||||
* Price the complete non-surface request envelope.
|
||||
* @param header - canonical envelope, or undefined before any request.
|
||||
* @returns heuristic system plus tool tokens.
|
||||
*/
|
||||
export function estimateHeader(header: EpochHeader | undefined): number {
|
||||
return estimateSystemTokens(header) + estimateToolsTokens(header)
|
||||
}
|
||||
@@ -7,7 +7,7 @@
|
||||
import { Context, Service } from 'cordis'
|
||||
import z from 'schemastery'
|
||||
import { BlockAssembler, deepFreeze } from '@deepseek-ai/dsh-llm'
|
||||
import type { ContentBlock, Message, TokenUsage } from '@deepseek-ai/dsh-llm'
|
||||
import type { Message, TokenUsage } from '@deepseek-ai/dsh-llm'
|
||||
import type { EpochHeader, Session, SessionEvent, SurfaceEvent } from '@deepseek-ai/dsh-session'
|
||||
import { canonicalHeader, headerEquals, isSurfaceEvent } from '@deepseek-ai/dsh-session'
|
||||
// Type-only: resolves the optional projection registry Context seam.
|
||||
@@ -18,19 +18,12 @@ import type {
|
||||
TokenMeterConfig,
|
||||
TokenSurfaceNode,
|
||||
} from './types.ts'
|
||||
import { contextBreakdownProjectionDefinition } from './breakdown-projection.ts'
|
||||
import { contextPressureProjectionDefinition, tokenUsageProjectionDefinition } from './usage-projection.ts'
|
||||
import { estimateContent, estimateHeader, estimateMessage, ROLE_OVERHEAD } from './estimate.ts'
|
||||
|
||||
export type * from './types.ts'
|
||||
|
||||
/** Fixed text-density estimate used until exact tokenization is needed. */
|
||||
const CHARS_PER_TOKEN = 4
|
||||
|
||||
/** Per-block structural overhead for JSON framing and type tags. */
|
||||
const BLOCK_OVERHEAD = 4
|
||||
|
||||
/** Role-field framing overhead added to every priced message. */
|
||||
const ROLE_OVERHEAD = 4
|
||||
|
||||
interface MeasurementAnchor {
|
||||
readonly header: EpochHeader | undefined
|
||||
readonly surfaceTokens: number
|
||||
@@ -98,6 +91,7 @@ export class TokenMeterService extends Service {
|
||||
ctx.inject(['sessionProjections'], (projectionCtx) => {
|
||||
projectionCtx.sessionProjections.register(tokenUsageProjectionDefinition)
|
||||
projectionCtx.sessionProjections.register(contextPressureProjectionDefinition)
|
||||
projectionCtx.sessionProjections.register(contextBreakdownProjectionDefinition)
|
||||
})
|
||||
|
||||
// Readers catch up independently, while eager observation bounds ordinary
|
||||
@@ -141,7 +135,7 @@ export class TokenMeterService extends Service {
|
||||
} else {
|
||||
baseline = {
|
||||
kind: 'estimated',
|
||||
tokens: this._estimateHeader(header) + state.surfaceTokens,
|
||||
tokens: estimateHeader(header) + state.surfaceTokens,
|
||||
}
|
||||
surfaceDeltaTokens = 0
|
||||
}
|
||||
@@ -157,12 +151,13 @@ export class TokenMeterService extends Service {
|
||||
}
|
||||
|
||||
/**
|
||||
* Heuristically price one model-visible message.
|
||||
* Heuristically price one model-visible message (instance face of the pure
|
||||
* {@link estimateMessage}).
|
||||
* @param message - message to price without mutation.
|
||||
* @returns content and role-framing tokens under the fixed service heuristic.
|
||||
*/
|
||||
estimateMessage(message: Message): number {
|
||||
return this._estimateContent(message.content) + ROLE_OVERHEAD
|
||||
return estimateMessage(message)
|
||||
}
|
||||
|
||||
/** Catch one session's fold up to the current durable tail. */
|
||||
@@ -246,7 +241,7 @@ export class TokenMeterService extends Service {
|
||||
)
|
||||
const anchorSurfaceTokens = stepStart.surfaceTokens + providerAssistantTokens
|
||||
const providerTokens = usageTokens(event.data.usage)
|
||||
const estimatedAnchorTokens = this._estimateHeader(nextHeader) + anchorSurfaceTokens
|
||||
const estimatedAnchorTokens = estimateHeader(nextHeader) + anchorSurfaceTokens
|
||||
nextAnchor = {
|
||||
header: nextHeader,
|
||||
surfaceTokens: anchorSurfaceTokens,
|
||||
@@ -263,7 +258,7 @@ export class TokenMeterService extends Service {
|
||||
surfaceTokens: anchorSurfaceTokens,
|
||||
baseline: {
|
||||
kind: 'estimated',
|
||||
tokens: this._estimateHeader(nextHeader) + anchorSurfaceTokens,
|
||||
tokens: estimateHeader(nextHeader) + anchorSurfaceTokens,
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -355,46 +350,7 @@ export class TokenMeterService extends Service {
|
||||
assembler.push(sourceEvent.data.chunk)
|
||||
}
|
||||
const providerContent = assembler.blocks()
|
||||
return providerContent.length === 0 ? 0 : this._estimateContent(providerContent) + ROLE_OVERHEAD
|
||||
}
|
||||
|
||||
/** Price content blocks recursively under the fixed density heuristic. */
|
||||
private _estimateContent(blocks: readonly ContentBlock[]): number {
|
||||
let tokens = 0
|
||||
for (const block of blocks) {
|
||||
switch (block.type) {
|
||||
case 'text':
|
||||
case 'reasoning':
|
||||
tokens += Math.ceil(block.text.length / CHARS_PER_TOKEN) + BLOCK_OVERHEAD
|
||||
break
|
||||
case 'tool-call':
|
||||
tokens += Math.ceil(block.name.length / CHARS_PER_TOKEN)
|
||||
+ Math.ceil(block.arguments.length / CHARS_PER_TOKEN)
|
||||
+ BLOCK_OVERHEAD
|
||||
break
|
||||
case 'tool-result':
|
||||
tokens += this._estimateContent(block.content) + BLOCK_OVERHEAD
|
||||
break
|
||||
default:
|
||||
// ContentBlockMap is merge-extensible; unknown blocks retain a
|
||||
// conservative structural JSON price under the fixed heuristic.
|
||||
tokens += BLOCK_OVERHEAD + Math.ceil(JSON.stringify(block).length / CHARS_PER_TOKEN)
|
||||
}
|
||||
}
|
||||
return tokens
|
||||
}
|
||||
|
||||
/** Price the canonical non-surface request envelope. */
|
||||
private _estimateHeader(header: EpochHeader | undefined): number {
|
||||
if (header === undefined) return 0
|
||||
let tokens = 0
|
||||
if (header.system !== undefined) {
|
||||
tokens += Math.ceil(header.system.length / CHARS_PER_TOKEN) + ROLE_OVERHEAD
|
||||
}
|
||||
if (header.tools !== undefined && header.tools.length > 0) {
|
||||
tokens += Math.ceil(JSON.stringify(header.tools).length / CHARS_PER_TOKEN) + BLOCK_OVERHEAD
|
||||
}
|
||||
return tokens
|
||||
return providerContent.length === 0 ? 0 : estimateContent(providerContent) + ROLE_OVERHEAD
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -40,11 +40,30 @@ export interface ContextPressureProjection {
|
||||
contextWindow?: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Heuristic composition of the next request's context: what the prompt is
|
||||
* made of, not what it costs. All three figures use the meter's fixed
|
||||
* density estimate (they will not sum exactly to the provider-reported
|
||||
* `pressureTokens`, which is billing-grade and one request behind), and the
|
||||
* message figure tracks the live surface, so it moves as content is appended
|
||||
* or compacted while the provider number holds still.
|
||||
*/
|
||||
export interface ContextBreakdownProjection {
|
||||
/** Heuristic tokens of the newest request envelope's system prompt; 0 before any request. */
|
||||
systemTokens: number
|
||||
/** Heuristic tokens of the newest request envelope's tool schemas; 0 before any request. */
|
||||
toolsTokens: number
|
||||
/** Heuristic tokens of the current model-visible conversation surface. */
|
||||
messageTokens: number
|
||||
}
|
||||
|
||||
declare module '@deepseek-ai/dsh-session-projection/types' {
|
||||
interface SessionProjectionMap {
|
||||
/** Provider-reported usage accumulated across the complete durable log. */
|
||||
tokenUsage: TokenUsageProjection
|
||||
/** Newest request pressure paired with the newest known route capacity. */
|
||||
contextPressure: ContextPressureProjection
|
||||
/** Heuristic system/tools/message composition of the next request. */
|
||||
contextBreakdown: ContextBreakdownProjection
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
|
||||
import type { TokenUsage } from '@deepseek-ai/dsh-llm'
|
||||
|
||||
export type { ContextPressureProjection, TokenUsageProjection } from './projection.ts'
|
||||
export type { ContextBreakdownProjection, ContextPressureProjection, TokenUsageProjection } from './projection.ts'
|
||||
|
||||
/** Token-meter plugin configuration; the fixed estimator has no settings. */
|
||||
export type TokenMeterConfig = Record<string, never>
|
||||
|
||||
Reference in New Issue
Block a user