refactor(web): project usage and snapshot request context
This commit is contained in:
@@ -1,4 +1,4 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import { Context } from 'cordis'
|
||||
import AgentRegistry, { agentEvents } from '@deepseek-ai/dsh-agent'
|
||||
import type { Agent } from '@deepseek-ai/dsh-agent'
|
||||
@@ -22,7 +22,7 @@ async function nextFrame<K extends MuxFrame['type']>(
|
||||
}
|
||||
|
||||
describe('ApiProxy model-request telemetry', () => {
|
||||
it('forwards only to open mux connections and never backfills history or reconnect baselines', async () => {
|
||||
it('atomically measures the observed request, forwards only live, and degrades per field', async () => {
|
||||
const ctx = new Context()
|
||||
await ctx.plugin(SessionStore)
|
||||
await ctx.plugin(UserInteractionService)
|
||||
@@ -35,6 +35,8 @@ describe('ApiProxy model-request telemetry', () => {
|
||||
ctx,
|
||||
} as Agent
|
||||
ctx.agents.register(agent)
|
||||
const measure = vi.fn(() => ({ totalTokens: 321 }))
|
||||
const removeTokenMeter = ctx.provide('tokenMeter' as never, { measure } as never)
|
||||
const api = createApiProxy(ctx, {
|
||||
provider: 'test',
|
||||
model: 'alpha',
|
||||
@@ -48,13 +50,13 @@ describe('ApiProxy model-request telemetry', () => {
|
||||
primaryAbort.signal,
|
||||
)[Symbol.asyncIterator]()
|
||||
expect((await nextFrame(primary, 'session/subscribed')).sessionId).toBe(session.id)
|
||||
expect((await nextFrame(primary, 'session/metrics')).metrics).not.toHaveProperty('contextWindow')
|
||||
|
||||
agentEvents(ctx, agent).emit('agent/model-request', 1, 2, {
|
||||
provider: 'test',
|
||||
model: 'alpha',
|
||||
contextWindow: 128_000,
|
||||
})
|
||||
expect(measure).toHaveBeenCalledWith(session)
|
||||
expect(await nextFrame(primary, 'session/model-request')).toEqual({
|
||||
type: 'session/model-request',
|
||||
sessionId: session.id,
|
||||
@@ -62,6 +64,7 @@ describe('ApiProxy model-request telemetry', () => {
|
||||
step: 2,
|
||||
provider: 'test',
|
||||
model: 'alpha',
|
||||
contextTokens: 321,
|
||||
contextWindow: 128_000,
|
||||
})
|
||||
|
||||
@@ -70,7 +73,8 @@ describe('ApiProxy model-request telemetry', () => {
|
||||
payload: { sessionId: session.id },
|
||||
})
|
||||
if (!history.result.ok) throw new Error('history failed')
|
||||
expect(history.result.value.metrics).not.toHaveProperty('contextWindow')
|
||||
expect(history.result.value).not.toHaveProperty('metrics')
|
||||
expect(history.result.value).not.toHaveProperty('modelRequest')
|
||||
|
||||
const reconnectAbort = new AbortController()
|
||||
const reconnect = api.events.mux(
|
||||
@@ -78,8 +82,8 @@ describe('ApiProxy model-request telemetry', () => {
|
||||
reconnectAbort.signal,
|
||||
)[Symbol.asyncIterator]()
|
||||
expect((await nextFrame(reconnect, 'session/subscribed')).sessionId).toBe(session.id)
|
||||
expect((await nextFrame(reconnect, 'session/metrics')).metrics).not.toHaveProperty('contextWindow')
|
||||
|
||||
measure.mockImplementation(() => { throw new Error('unmeasurable replay') })
|
||||
agentEvents(ctx, agent).emit('agent/model-request', 2, 1, {
|
||||
provider: 'test',
|
||||
model: 'without-capacity',
|
||||
@@ -95,6 +99,22 @@ describe('ApiProxy model-request telemetry', () => {
|
||||
})
|
||||
}
|
||||
|
||||
removeTokenMeter()
|
||||
agentEvents(ctx, agent).emit('agent/model-request', 3, 1, {
|
||||
provider: 'test',
|
||||
model: 'without-meter',
|
||||
contextWindow: 64_000,
|
||||
})
|
||||
expect(await nextFrame(primary, 'session/model-request')).toEqual({
|
||||
type: 'session/model-request',
|
||||
sessionId: session.id,
|
||||
turn: 3,
|
||||
step: 1,
|
||||
provider: 'test',
|
||||
model: 'without-meter',
|
||||
contextWindow: 64_000,
|
||||
})
|
||||
|
||||
primaryAbort.abort()
|
||||
reconnectAbort.abort()
|
||||
await primary.return?.()
|
||||
|
||||
@@ -10,7 +10,7 @@ import {
|
||||
sessionCreateValueSchema, sessionEventSchema, sessionHistoryRequestSchema, sessionHistoryValueSchema,
|
||||
sessionIdSchema, sessionListRequestSchema, sessionListValueSchema, sessionModelsRequestSchema,
|
||||
sessionModelsValueSchema, sessionPromptRequestSchema, sessionPromptValueSchema,
|
||||
sessionSelectModelRequestSchema, sessionSelectModelValueSchema, sessionSummarySchema, sessionMetricsSchema,
|
||||
sessionSelectModelRequestSchema, sessionSelectModelValueSchema, sessionSummarySchema,
|
||||
} from '../src/api/sessions.schema.ts'
|
||||
import {
|
||||
hostCreateDirectoryRequestSchema, hostCreateDirectoryValueSchema,
|
||||
@@ -153,29 +153,13 @@ describe('sessions domain schemas', () => {
|
||||
expect(sessionCreateValueSchema.parse({ sessionId: 's1' }).sessionId).toBe('s1')
|
||||
expect(sessionHistoryRequestSchema.parse({ sessionId: 's1', beforeSeq: 3, maxMessages: 5 }).beforeSeq).toBe(3)
|
||||
expect(() => sessionHistoryRequestSchema.parse({ sessionId: 's1', maxMessages: 0 })).toThrow()
|
||||
expect(sessionHistoryValueSchema.parse({
|
||||
const history = sessionHistoryValueSchema.parse({
|
||||
events: [],
|
||||
hasMore: false,
|
||||
projections: { asOfSeq: 11, values: { todos: [] } },
|
||||
metrics: {
|
||||
logRevision: 12,
|
||||
projectionRevision: 4,
|
||||
uncachedInputTokens: 1_000,
|
||||
outputTokens: 200,
|
||||
cacheReadTokens: 4_000,
|
||||
cacheWriteTokens: 500,
|
||||
contextTokens: 8_000,
|
||||
},
|
||||
modelTarget: { provider: 'deepseek', model: 'deepseek-v4-flash' },
|
||||
}).metrics?.contextTokens).toBe(8_000)
|
||||
expect(() => sessionMetricsSchema.parse({
|
||||
logRevision: 1,
|
||||
projectionRevision: 0,
|
||||
uncachedInputTokens: -1,
|
||||
outputTokens: 0,
|
||||
cacheReadTokens: 0,
|
||||
cacheWriteTokens: 0,
|
||||
})).toThrow()
|
||||
})
|
||||
expect(history.projections).toEqual({ asOfSeq: 11, values: { todos: [] } })
|
||||
expect(sessionModelsRequestSchema.parse({ sessionId: 's1' }).sessionId).toBe('s1')
|
||||
expect(sessionModelsValueSchema.parse({
|
||||
current: { provider: 'deepseek', model: 'deepseek-v4-flash', reasoningEffort: 'max' },
|
||||
@@ -377,18 +361,6 @@ describe('events frame schemas', () => {
|
||||
const frames = [
|
||||
{ type: 'session/event', sessionId: 's', event: { type: 't', seq: 0, time: 1, data: null } },
|
||||
{ type: 'session/subscribed', sessionId: 's', lastSeq: -1 },
|
||||
{
|
||||
type: 'session/metrics',
|
||||
sessionId: 's',
|
||||
metrics: {
|
||||
logRevision: 3,
|
||||
projectionRevision: 1,
|
||||
uncachedInputTokens: 100,
|
||||
outputTokens: 20,
|
||||
cacheReadTokens: 300,
|
||||
cacheWriteTokens: 40,
|
||||
},
|
||||
},
|
||||
{
|
||||
type: 'session/model-request',
|
||||
sessionId: 's',
|
||||
@@ -396,6 +368,7 @@ describe('events frame schemas', () => {
|
||||
step: 1,
|
||||
provider: 'deepseek',
|
||||
model: 'deepseek-chat',
|
||||
contextTokens: 8_000,
|
||||
contextWindow: 128_000,
|
||||
},
|
||||
{
|
||||
@@ -419,6 +392,7 @@ describe('events frame schemas', () => {
|
||||
expect(() => muxFrameSchema.parse({ type: 'unknown/frame' })).toThrow()
|
||||
for (const invalid of [
|
||||
{ type: 'session/model-request', sessionId: 's', turn: 0, step: 1, provider: 'p', model: 'm' },
|
||||
{ type: 'session/model-request', sessionId: 's', turn: 1, step: 1, provider: 'p', model: 'm', contextTokens: -1 },
|
||||
{ type: 'session/model-request', sessionId: 's', turn: 1, step: 1, provider: 'p', model: 'm', contextWindow: 0 },
|
||||
{ type: 'session/projection', sessionId: 's', key: '', value: null, seq: 0 },
|
||||
{ type: 'session/projection', sessionId: 's', key: 'todos', value: null, seq: -1 },
|
||||
|
||||
@@ -1,143 +0,0 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { Context } from 'cordis'
|
||||
import { createAssistantMessage, createUserMessage } from '@deepseek-ai/dsh-llm'
|
||||
import { Session, SessionId } from '@deepseek-ai/dsh-session'
|
||||
import { affectsSessionMetrics, SessionMetricsProjector } from '../src/session-metrics.ts'
|
||||
|
||||
function assistant(
|
||||
session: Session,
|
||||
turn: number,
|
||||
step: number,
|
||||
usage: {
|
||||
inputTokens: number
|
||||
outputTokens: number
|
||||
cacheReadTokens?: number
|
||||
cacheWriteTokens?: number
|
||||
},
|
||||
): void {
|
||||
session.append('assistant/chunk', {
|
||||
turn,
|
||||
step,
|
||||
chunk: { type: 'usage', usage },
|
||||
})
|
||||
session.append('assistant/message', {
|
||||
turn,
|
||||
step,
|
||||
message: createAssistantMessage({
|
||||
content: [{ type: 'text', text: `answer-${turn}-${step}` }],
|
||||
source: { provider: 'test', model: 'alpha' },
|
||||
}),
|
||||
usage,
|
||||
}, { surfaceOp: 'append' })
|
||||
}
|
||||
|
||||
describe('SessionMetricsProjector', () => {
|
||||
it('filters text/reasoning deltas while retaining usage, headers, and surface mutations', () => {
|
||||
const session = new Session(SessionId('metrics-filter'))
|
||||
const text = session.append('assistant/chunk', {
|
||||
turn: 1,
|
||||
step: 1,
|
||||
chunk: { type: 'text-delta', index: 0, text: 'x' },
|
||||
})
|
||||
const usage = session.append('assistant/chunk', {
|
||||
turn: 1,
|
||||
step: 1,
|
||||
chunk: { type: 'usage', usage: { inputTokens: 1, outputTokens: 1 } },
|
||||
})
|
||||
const header = session.append('request/header', {
|
||||
header: { config: { provider: 'test', model: 'alpha' } },
|
||||
reason: 'initial',
|
||||
})
|
||||
const surface = session.append('user/message', createUserMessage({
|
||||
content: [{ type: 'text', text: 'question' }],
|
||||
source: { kind: 'user' },
|
||||
}), { surfaceOp: 'append' })
|
||||
const plain = session.append('step/start', { turn: 1, step: 1 })
|
||||
|
||||
expect(affectsSessionMetrics(text)).toBe(false)
|
||||
expect(affectsSessionMetrics(usage)).toBe(true)
|
||||
expect(affectsSessionMetrics(header)).toBe(true)
|
||||
expect(affectsSessionMetrics(surface)).toBe(true)
|
||||
expect(affectsSessionMetrics(plain)).toBe(false)
|
||||
})
|
||||
|
||||
it('folds usage by turn and step while synchronous pressure follows surface replacement', () => {
|
||||
const ctx = new Context()
|
||||
ctx.provide('tokenMeter', {
|
||||
measure(session: Session) {
|
||||
return { totalTokens: session.surface.nodes.length * 100 }
|
||||
},
|
||||
})
|
||||
const session = new Session(SessionId('metrics-fold'))
|
||||
const first = session.append('user/message', createUserMessage({
|
||||
content: [{ type: 'text', text: 'large old surface' }],
|
||||
source: { kind: 'user' },
|
||||
}), { surfaceOp: 'append' })
|
||||
assistant(session, 1, 1, {
|
||||
inputTokens: 11,
|
||||
outputTokens: 3,
|
||||
cacheReadTokens: 89,
|
||||
cacheWriteTokens: 8,
|
||||
})
|
||||
|
||||
const projector = new SessionMetricsProjector(ctx)
|
||||
expect(projector.snapshot(session)).toMatchObject({
|
||||
uncachedInputTokens: 11,
|
||||
outputTokens: 3,
|
||||
cacheReadTokens: 89,
|
||||
cacheWriteTokens: 8,
|
||||
contextTokens: 200,
|
||||
})
|
||||
|
||||
const assistantSeq = session.surface.nodes.at(-1)
|
||||
if (assistantSeq === undefined) throw new Error('assistant surface missing')
|
||||
session.append('user/message', createUserMessage({
|
||||
content: [{ type: 'text', text: 'compact summary' }],
|
||||
source: { kind: 'plugin', plugin: 'test' },
|
||||
}), {
|
||||
surfaceOp: { op: 'replace', start: first.seq, end: assistantSeq },
|
||||
sourceEventSeqs: [first.seq, assistantSeq],
|
||||
})
|
||||
expect(projector.snapshot(session)).toMatchObject({
|
||||
uncachedInputTokens: 11,
|
||||
outputTokens: 3,
|
||||
cacheReadTokens: 89,
|
||||
cacheWriteTokens: 8,
|
||||
contextTokens: 100,
|
||||
})
|
||||
|
||||
session.append('assistant/chunk', {
|
||||
turn: 1,
|
||||
step: 1,
|
||||
chunk: {
|
||||
type: 'usage',
|
||||
usage: { inputTokens: 12, outputTokens: 4, cacheReadTokens: 88, cacheWriteTokens: 9 },
|
||||
},
|
||||
})
|
||||
assistant(session, 1, 2, { inputTokens: 1_000, outputTokens: 500 })
|
||||
|
||||
expect(projector.snapshot(session)).toMatchObject({
|
||||
logRevision: session.events.length,
|
||||
projectionRevision: 2,
|
||||
uncachedInputTokens: 1_012,
|
||||
outputTokens: 504,
|
||||
cacheReadTokens: 88,
|
||||
cacheWriteTokens: 9,
|
||||
contextTokens: 200,
|
||||
})
|
||||
})
|
||||
|
||||
it('omits pressure when the token meter is absent or cannot measure the replay', () => {
|
||||
const session = new Session(SessionId('metrics-pressure-unknown'))
|
||||
const withoutMeter = new SessionMetricsProjector(new Context()).snapshot(session)
|
||||
expect(withoutMeter.contextTokens).toBeUndefined()
|
||||
|
||||
const ctx = new Context()
|
||||
ctx.provide('tokenMeter', {
|
||||
measure() {
|
||||
throw new Error('unmeasurable replay')
|
||||
},
|
||||
})
|
||||
expect(new SessionMetricsProjector(ctx).snapshot(session).contextTokens).toBeUndefined()
|
||||
})
|
||||
})
|
||||
Reference in New Issue
Block a user