feat: show per-step OpenRouter cost on the Trajectory tab
Extend the openRouterCost projection with a per-step priced-cost map keyed by turn:step so surfaces can render spend without re-pricing, and have the Trajectory view read it through the framework useProjection seat: each priced assistant step shows its USD cost on the request boundary chip and in the inspector summary, while unpriced steps stay blank. Also fix a pre-existing exactOptionalPropertyTypes error in extractCacheRates that the host typecheck surfaced.
This commit is contained in:
@@ -27,7 +27,7 @@ async function harness(): Promise<{ ctx: Context; session: Session }> {
|
||||
await ctx.plugin(SessionStore)
|
||||
await ctx.plugin(SessionProjectionRegistry)
|
||||
const session = ctx.sessions.create()
|
||||
ctx.sessionProjections.register(createOpenRouterCostProjection((model) => PRICING.get(model)))
|
||||
ctx.sessionProjections.register(createOpenRouterCostProjection(model => PRICING.get(model)))
|
||||
return { ctx, session }
|
||||
}
|
||||
|
||||
@@ -75,7 +75,7 @@ const projected = (ctx: Context, session: Session) => {
|
||||
describe('openRouterCost session projection', () => {
|
||||
it('serves an all-zero view on an empty log', async () => {
|
||||
const { ctx, session } = await harness()
|
||||
expect(projected(ctx, session)).toEqual({ totalUsd: 0, pricedSteps: 0, unknownModelSteps: 0, currency: 'USD' })
|
||||
expect(projected(ctx, session)).toEqual({ totalUsd: 0, pricedSteps: 0, unknownModelSteps: 0, steps: {}, currency: 'USD' })
|
||||
})
|
||||
|
||||
it('prices input/output/cache buckets at the model rates', async () => {
|
||||
@@ -98,7 +98,13 @@ describe('openRouterCost session projection', () => {
|
||||
startStep(session, 1, 1)
|
||||
const source = usageChunk(session, { inputTokens: 10, outputTokens: 4 }, 1, 1)
|
||||
finalUsage(session, { inputTokens: 10, outputTokens: 4 }, 1, 1, [source])
|
||||
expect(projected(ctx, session)).toEqual({ totalUsd: 10 * 1.4e-6 + 4 * 2.8e-6, pricedSteps: 1, unknownModelSteps: 0, currency: 'USD' })
|
||||
expect(projected(ctx, session)).toEqual({
|
||||
totalUsd: 10 * 1.4e-6 + 4 * 2.8e-6,
|
||||
pricedSteps: 1,
|
||||
unknownModelSteps: 0,
|
||||
steps: { '1:1': 10 * 1.4e-6 + 4 * 2.8e-6 },
|
||||
currency: 'USD',
|
||||
})
|
||||
})
|
||||
|
||||
it('replaces an earlier same-step chunk sample with the final usage', async () => {
|
||||
@@ -111,6 +117,7 @@ describe('openRouterCost session projection', () => {
|
||||
totalUsd: 14 * 1.4e-6 + 5 * 2.8e-6,
|
||||
pricedSteps: 1,
|
||||
unknownModelSteps: 0,
|
||||
steps: { '1:1': 14 * 1.4e-6 + 5 * 2.8e-6 },
|
||||
currency: 'USD',
|
||||
})
|
||||
})
|
||||
@@ -123,6 +130,7 @@ describe('openRouterCost session projection', () => {
|
||||
session.append('step/end', { turn: 1, step: 1 })
|
||||
expect(projected(ctx, session).totalUsd).toBeCloseTo(9 * 1.4e-6 + 1 * 2.8e-6, 12)
|
||||
expect(projected(ctx, session).pricedSteps).toBe(1)
|
||||
expect(projected(ctx, session).steps).toEqual({ '1:1': 9 * 1.4e-6 + 1 * 2.8e-6 })
|
||||
})
|
||||
|
||||
it('counts an unknown-priced OpenRouter model as an unpriced step', async () => {
|
||||
@@ -131,7 +139,7 @@ describe('openRouterCost session projection', () => {
|
||||
startStep(session, 1, 1)
|
||||
usageChunk(session, { inputTokens: 100, outputTokens: 1 }, 1, 1)
|
||||
session.append('step/end', { turn: 1, step: 1 })
|
||||
expect(projected(ctx, session)).toEqual({ totalUsd: 0, pricedSteps: 0, unknownModelSteps: 1, currency: 'USD' })
|
||||
expect(projected(ctx, session)).toEqual({ totalUsd: 0, pricedSteps: 0, unknownModelSteps: 1, steps: {}, currency: 'USD' })
|
||||
})
|
||||
|
||||
it('ignores a step on a non-openrouter provider entirely', async () => {
|
||||
@@ -140,7 +148,7 @@ describe('openRouterCost session projection', () => {
|
||||
startStep(session, 1, 1)
|
||||
usageChunk(session, { inputTokens: 100, outputTokens: 1 }, 1, 1)
|
||||
session.append('step/end', { turn: 1, step: 1 })
|
||||
expect(projected(ctx, session)).toEqual({ totalUsd: 0, pricedSteps: 0, unknownModelSteps: 0, currency: 'USD' })
|
||||
expect(projected(ctx, session)).toEqual({ totalUsd: 0, pricedSteps: 0, unknownModelSteps: 0, steps: {}, currency: 'USD' })
|
||||
})
|
||||
|
||||
it('prefers the assistant-message source over the last request/context record', async () => {
|
||||
@@ -172,7 +180,48 @@ describe('openRouterCost session projection', () => {
|
||||
startStep(session, 1, 1)
|
||||
usageChunk(session, { inputTokens: 100, outputTokens: 10 }, 1, 1)
|
||||
session.append('step/end', { turn: 1, step: 1 })
|
||||
expect(projected(ctx, session)).toEqual({ totalUsd: 0, pricedSteps: 1, unknownModelSteps: 0, currency: 'USD' })
|
||||
expect(projected(ctx, session)).toEqual({ totalUsd: 0, pricedSteps: 1, unknownModelSteps: 0, steps: { '1:1': 0 }, currency: 'USD' })
|
||||
})
|
||||
|
||||
it('maps each step to its priced cost under turn:step keys', async () => {
|
||||
const { ctx, session } = await harness()
|
||||
recordContext(session)
|
||||
startStep(session, 1, 1)
|
||||
usageChunk(session, { inputTokens: 1_000, outputTokens: 500 }, 1, 1)
|
||||
startStep(session, 1, 2)
|
||||
usageChunk(session, { inputTokens: 200, outputTokens: 100 }, 1, 2)
|
||||
session.append('step/end', { turn: 1, step: 2 })
|
||||
expect(projected(ctx, session)).toEqual({
|
||||
totalUsd: (1_000 * 1.4e-6 + 500 * 2.8e-6) + (200 * 1.4e-6 + 100 * 2.8e-6),
|
||||
pricedSteps: 2,
|
||||
unknownModelSteps: 0,
|
||||
steps: {
|
||||
'1:1': 1_000 * 1.4e-6 + 500 * 2.8e-6,
|
||||
'1:2': 200 * 1.4e-6 + 100 * 2.8e-6,
|
||||
},
|
||||
currency: 'USD',
|
||||
})
|
||||
})
|
||||
|
||||
it('drops a step entry when the final sample re-attributes it to an unknown-priced model', async () => {
|
||||
const { ctx, session } = await harness()
|
||||
recordContext(session)
|
||||
startStep(session, 1, 1)
|
||||
const source = usageChunk(session, { inputTokens: 100, outputTokens: 10 }, 1, 1)
|
||||
session.append('assistant/message', {
|
||||
turn: 1,
|
||||
step: 1,
|
||||
message: createMessage({
|
||||
role: 'assistant',
|
||||
content: [],
|
||||
source: { kind: 'model', provider: 'openrouter', model: 'brand-new/model' },
|
||||
}),
|
||||
usage: { inputTokens: 100, outputTokens: 10 },
|
||||
}, { surfaceOp: 'append', sourceEventSeqs: [source] })
|
||||
session.append('step/end', { turn: 1, step: 1 })
|
||||
// The earlier priced chunk's entry is removed with the re-attribution: the
|
||||
// step is no longer priced, so it has no entry.
|
||||
expect(projected(ctx, session)).toEqual({ totalUsd: 0, pricedSteps: 0, unknownModelSteps: 1, steps: {}, currency: 'USD' })
|
||||
})
|
||||
|
||||
it('pushes no change for unrelated events', async () => {
|
||||
@@ -199,6 +248,7 @@ describe('openRouterCost session projection', () => {
|
||||
totalUsd: 8 * 1.4e-6 + 2 * 2.8e-6,
|
||||
pricedSteps: 1,
|
||||
unknownModelSteps: 0,
|
||||
steps: { '1:1': 8 * 1.4e-6 + 2 * 2.8e-6 },
|
||||
currency: 'USD',
|
||||
})
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user