Coder/deepseek-harness
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add web multimodal image attachments

Browse Source
This commit is contained in:
Yichen Jiang
2026-07-23 15:20:47 +08:00
parent 3e3ea47296
commit cb4c11b869
116 changed files with 3177 additions and 151 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
packages/llm/llm-deepseek/src/adapter.ts
View File

@@ -116,6 +116,8 @@ export class DeepSeekAdapter extends LlmAdapter {
id: model.id,
name: model.name ?? model.id,
...model.description === undefined ? {} : { description: model.description },
inputModalities: ['text'],
outputModalities: ['text'],
})))
}

15
packages/llm/llm-deepseek/src/serialize.ts
View File

@@ -2,10 +2,12 @@
* Serialize harness messages into DeepSeek chat completions. User text is joined; assistant text
* becomes `content`, tool calls become `tool_calls`, and tool results become separate tool messages.
* Assistant reasoning is replayed as `reasoning_content` only on tool-call turns, as required by
* thinking-mode passback. Unknown declaration-merged block types are skipped rather than rejected.
* thinking-mode passback. Core image blocks are rejected explicitly because this wire route is text-only;
* unknown declaration-merged block types retain the adapter's documented extension fallback.
* @module dsh-llm-deepseek/serialize
*/
import { LlmError } from '@deepseek-ai/dsh-llm'
import type { ContentBlock, GenerateOptions, Message } from '@deepseek-ai/dsh-llm'
import type { WireMessage, WireRequest, WireTool } from './types.ts'
@@ -23,6 +25,16 @@ function flattenText(blocks: ContentBlock[]): string {
.join('')
}
/** Reject core image content before any text-flattening path can silently erase it. */
function assertTextOnly(blocks: readonly ContentBlock[]): void {
for (const block of blocks) {
if (block.type === 'image') {
throw new LlmError('The DeepSeek chat-completions adapter does not support image content.', 'UNSUPPORTED_CONTENT')
}
if (block.type === 'tool-result') assertTextOnly(block.content)
}
}
/** Serialize one assistant message (text + reasoning + tool calls). */
function serializeAssistant(message: Message): WireMessage {
const text = flattenText(message.content)
@@ -68,6 +80,7 @@ function serializeAssistant(message: Message): WireMessage {
export function serializeMessages(messages: Message[]): WireMessage[] {
const wire: WireMessage[] = []
for (const message of messages) {
assertTextOnly(message.content)
if (message.role === 'system') {
wire.push({ role: 'system', content: flattenText(message.content) })
continue

12
packages/llm/llm-deepseek/tests/adapter.spec.ts
View File

@@ -528,8 +528,8 @@ describe('plugin registration and config', () => {
await ctx.plugin(LlmDeepSeek, { apiKey: 'k', baseURL: 'http://127.0.0.1:1' })
expect(ctx.llm.listProviders()).toEqual([{ id: 'deepseek', name: 'DeepSeek' }])
await expect(ctx.llm.listModels('deepseek')).resolves.toEqual([
{ provider: 'deepseek', id: 'deepseek-v4-flash', name: 'deepseek-v4-flash' },
{ provider: 'deepseek', id: 'deepseek-v4-pro', name: 'deepseek-v4-pro' },
{ provider: 'deepseek', id: 'deepseek-v4-flash', name: 'deepseek-v4-flash', inputModalities: ['text'], outputModalities: ['text'] },
{ provider: 'deepseek', id: 'deepseek-v4-pro', name: 'deepseek-v4-pro', inputModalities: ['text'], outputModalities: ['text'] },
])
await expect(ctx.llm.resolveModelContext('deepseek', 'deepseek-v4-flash'))
.resolves.toEqual({ contextWindow: 128_000 })
@@ -540,8 +540,8 @@ describe('plugin registration and config', () => {
await ctx.plugin(LlmService)
LlmDeepSeek.apply(ctx, { apiKey: 'k', baseURL: 'http://127.0.0.1:1' })
await expect(ctx.llm.listModels('deepseek')).resolves.toEqual([
{ provider: 'deepseek', id: 'deepseek-v4-flash', name: 'deepseek-v4-flash' },
{ provider: 'deepseek', id: 'deepseek-v4-pro', name: 'deepseek-v4-pro' },
{ provider: 'deepseek', id: 'deepseek-v4-flash', name: 'deepseek-v4-flash', inputModalities: ['text'], outputModalities: ['text'] },
{ provider: 'deepseek', id: 'deepseek-v4-pro', name: 'deepseek-v4-pro', inputModalities: ['text'], outputModalities: ['text'] },
])
})
@@ -562,8 +562,8 @@ describe('plugin registration and config', () => {
],
})
await expect(ctx.llm.listModels('deepseek')).resolves.toEqual([
{ provider: 'deepseek', id: 'private-fast', name: 'private-fast' },
{ provider: 'deepseek', id: 'private-reasoner', name: 'Private Reasoner', description: 'Higher reasoning budget' },
{ provider: 'deepseek', id: 'private-fast', name: 'private-fast', inputModalities: ['text'], outputModalities: ['text'] },
{ provider: 'deepseek', id: 'private-reasoner', name: 'Private Reasoner', description: 'Higher reasoning budget', inputModalities: ['text'], outputModalities: ['text'] },
])
await expect(ctx.llm.resolveModelContext('deepseek', 'private-fast'))
.resolves.toEqual({ contextWindow: 32_000 })

14
packages/llm/llm-deepseek/tests/serialize.spec.ts
View File

@@ -1,5 +1,6 @@
import { describe, expect, it } from 'vitest'
import { CallId } from '@deepseek-ai/dsh-llm'
import { AttachmentId } from '@deepseek-ai/dsh-attachment'
import type { ContentBlock, GenerateOptions, Message } from '@deepseek-ai/dsh-llm'
import { serializeMessages, serializeRequest } from '../src/serialize.ts'
@@ -123,6 +124,19 @@ describe('serializeMessages', () => {
expect(wire).toEqual([{ role: 'user', content: 'see chart' }])
})
it('rejects image blocks instead of silently flattening them away', () => {
expect(() => serializeMessages([{
role: 'user',
content: [{
type: 'image',
attachment: {
attachmentId: AttachmentId(`sha256:${'a'.repeat(64)}`),
mediaType: 'image/png', bytes: 68, width: 1, height: 1,
},
}],
}])).toThrow(expect.objectContaining({ code: 'UNSUPPORTED_CONTENT' }))
})
it('emits an empty user message rather than dropping block-less messages', () => {
const wire = serializeMessages([{ role: 'user', content: [] }])
expect(wire).toEqual([{ role: 'user', content: '' }])

2
packages/llm/llm-pi-ai/package.json
View File

@@ -27,6 +27,7 @@
],
"license": "BSD-3-Clause",
"peerDependencies": {
"@deepseek-ai/dsh-attachment": "^0.0.1",
"@deepseek-ai/dsh-invariants": "^0.0.1",
"@deepseek-ai/dsh-llm": "^0.0.1",
"@deepseek-ai/dsh-timeout": "^0.0.1",
@@ -37,6 +38,7 @@
"schemastery": "^3.18.0"
},
"devDependencies": {
"@deepseek-ai/dsh-attachment": "workspace:^",
"@deepseek-ai/dsh-invariants": "workspace:^",
"@deepseek-ai/dsh-llm": "workspace:^",
"@deepseek-ai/dsh-llm-deepseek": "workspace:^",

25
packages/llm/llm-pi-ai/src/adapter.ts
View File

@@ -12,6 +12,7 @@ import type {
Model,
SimpleStreamOptions,
} from '@earendil-works/pi-ai'
import type { AttachmentStore } from '@deepseek-ai/dsh-attachment'
import { attributionHeaders, LlmAdapter, LlmError } from '@deepseek-ai/dsh-llm'
import type { GenerateOptions, LlmModelContext, LlmModelInfo, StreamChunk } from '@deepseek-ai/dsh-llm'
import { idleWatchdog, timeoutOf } from '@deepseek-ai/dsh-timeout'
@@ -24,6 +25,8 @@ import { toStreamChunks } from './stream.ts'
export interface PiAiAdapterOptions {
/** Validated provider profiles this adapter instance owns. */
profiles: readonly PiAiProviderProfile[]
/** Durable image resolver used only when a request contains image references. */
attachments?: AttachmentStore
}
/**
@@ -69,10 +72,12 @@ function requestHeaders(headers: Readonly<Record<string, string>> | undefined):
*/
export class PiAiAdapter extends LlmAdapter {
private readonly profiles: ReadonlyMap<string, ResolvedPiAiProviderProfile>
private readonly attachments: AttachmentStore | undefined
constructor(options: PiAiAdapterOptions) {
super()
this.profiles = new Map(resolveProfiles(options.profiles).map(profile => [profile.provider, profile]))
this.attachments = options.attachments
}
override listModels(provider: string): Promise<readonly LlmModelInfo[]> {
@@ -84,6 +89,8 @@ export class PiAiAdapter extends LlmAdapter {
provider,
id: model.id,
name: model.name,
inputModalities: [...model.input],
outputModalities: ['text'],
})))
}
@@ -112,7 +119,6 @@ export class PiAiAdapter extends LlmAdapter {
throw new LlmError(`pi-ai adapter does not own provider "${options.provider}"`, 'NO_ADAPTER')
}
const model = resolveModel(profile, options.model)
const consumer = new AbortController()
const upstream = options.signal === undefined
? consumer.signal
@@ -121,7 +127,22 @@ export class PiAiAdapter extends LlmAdapter {
using watchdog = idleWatchdog(upstream, streamIdleTimeoutMs, 'LLM_STREAM_IDLE_TIMEOUT')
try {
const events = streamSimple(model, toPiContext(options), {
const containsImage = options.messages.some((message) => {
// The discriminant is part of same-process message validity and is read before content.
void message.role
return message.content.some(block => block.type === 'image'
|| (block.type === 'tool-result' && block.content.some(piece => piece.type === 'image')))
})
if (containsImage && !model.input.includes('image')) {
throw new LlmError(`pi-ai model "${model.id}" does not support image input`, 'UNSUPPORTED_CONTENT')
}
if (containsImage && this.attachments === undefined) {
throw new LlmError('pi-ai image input requires the durable attachment service', 'UNSUPPORTED_CONTENT')
}
const context = this.attachments === undefined
? toPiContext(options)
: await toPiContext(options, this.attachments)
const events = streamSimple(model, context, {
...profileOptions(profile),
...options.temperature === undefined ? {} : { temperature: options.temperature },
...options.maxTokens === undefined ? {} : { maxTokens: options.maxTokens },

153
packages/llm/llm-pi-ai/src/context.ts
View File

@@ -4,9 +4,10 @@
* @module dsh-llm-pi-ai/context
*/
import { CallId } from '@deepseek-ai/dsh-llm'
import type { GenerateOptions, Message } from '@deepseek-ai/dsh-llm'
import type { Context as PiContext, Message as PiMessage, Tool as PiTool } from '@earendil-works/pi-ai'
import { CallId, LlmError } from '@deepseek-ai/dsh-llm'
import type { ContentBlock, GenerateOptions, Message } from '@deepseek-ai/dsh-llm'
import type { AttachmentStore } from '@deepseek-ai/dsh-attachment'
import type { Context as PiContext, ImageContent, Message as PiMessage, TextContent, Tool as PiTool } from '@earendil-works/pi-ai'
import { toPiAssistant } from './replay.ts'
/** Join the text blocks of a harness message. */
@@ -17,20 +18,122 @@ function flattenText(message: Message): string {
.join('')
}
async function userContent(
blocks: readonly ContentBlock[],
attachments: AttachmentStore,
): Promise<string | (TextContent | ImageContent)[]> {
const content: (TextContent | ImageContent)[] = []
for (const block of blocks) {
switch (block.type) {
case 'text':
if (block.text.length > 0) content.push({ type: 'text', text: block.text })
break
case 'image': {
const stored = await attachments.readImage(block.attachment)
content.push({
type: 'image',
data: Buffer.from(stored.data).toString('base64'),
mimeType: stored.ref.mediaType,
})
break
}
case 'tool-result':
break
default:
// Other merge-extensible blocks are not user-input vocabulary for pi-ai.
break
}
}
if (content.every(block => block.type === 'text')) return content.map(block => block.text).join('')
return content
}
function toolsOf(options: GenerateOptions): PiTool[] | undefined {
return options.tools?.map(tool => ({
name: tool.name,
description: tool.description,
// ToolSchema.parameters is a JSON Schema object; pi-ai's TSchema
// (TypeBox) is structurally JSON Schema, so it assigns directly.
parameters: tool.parameters,
}))
}
/** Assemble the request-level pi-ai context envelope shared by both conversion paths. */
function piContext(options: GenerateOptions, messages: PiMessage[]): PiContext {
const tools = toolsOf(options)
return {
...options.system !== undefined ? { systemPrompt: options.system } : {},
messages,
...tools !== undefined && tools.length > 0 ? { tools } : {},
}
}
function textOnlyContext(options: GenerateOptions): PiContext {
const toolNames = new Map<CallId, string>()
const messages: PiMessage[] = []
for (const message of options.messages) {
if (message.content.some(block => block.type === 'image'
|| (block.type === 'tool-result' && block.content.some(piece => piece.type === 'image')))) {
throw new LlmError('pi-ai image conversion requires the durable attachment service', 'UNSUPPORTED_CONTENT')
}
if (message.role === 'system') {
messages.push({ role: 'user', content: flattenText(message), timestamp: 0 })
continue
}
if (message.role === 'assistant') {
const assistant = toPiAssistant(message)
for (const block of assistant.content) if (block.type === 'toolCall') toolNames.set(CallId(block.id), block.name)
messages.push(assistant)
continue
}
const text = flattenText(message)
const results = message.content.filter(block => block.type === 'tool-result')
if (text.length > 0 || results.length === 0) messages.push({ role: 'user', content: text, timestamp: 0 })
for (const result of results) {
messages.push({
role: 'toolResult',
toolCallId: result.toolCallId,
toolName: toolNames.get(result.toolCallId) ?? 'unknown',
content: [{
type: 'text',
text: result.content.filter(block => block.type === 'text').map(block => block.text).join('') || '(no output)',
}],
isError: result.isError ?? false,
timestamp: 0,
})
}
}
return piContext(options, messages)
}
/**
* Convert harness history to a pi-ai Context. Tool results need the tool
* NAME (pi-ai's `toolName`), which the harness doesn't carry on the result
* block — it is recovered from the preceding assistant tool-call with the
* same id.
* Convert text-only harness history to a synchronous pi-ai Context. Tool
* result names are recovered from preceding assistant tool calls.
* @param options - the harness request; `options.system` maps to pi-ai's single `systemPrompt` slot.
* @returns the pi-ai context; `tools` is omitted entirely when the request declares none.
* @returns the pi-ai context; `tools` is omitted when the request declares none.
*/
export function toPiContext(options: GenerateOptions): PiContext {
export function toPiContext(options: GenerateOptions): PiContext
/**
* Convert harness history to a pi-ai Context while resolving durable images.
* Tool result names are recovered from preceding assistant tool calls.
* @param options - the harness request; `options.system` maps to pi-ai's single `systemPrompt` slot.
* @param attachments - durable byte resolver for image references.
* @returns the asynchronously resolved pi-ai context.
*/
export function toPiContext(options: GenerateOptions, attachments: AttachmentStore): Promise<PiContext>
export function toPiContext(options: GenerateOptions, attachments?: AttachmentStore): PiContext | Promise<PiContext> {
return attachments === undefined ? textOnlyContext(options) : toPiContextWithImages(options, attachments)
}
async function toPiContextWithImages(options: GenerateOptions, attachments: AttachmentStore): Promise<PiContext> {
const toolNames = new Map<CallId, string>()
const messages: PiMessage[] = []
for (const message of options.messages) {
if (message.role === 'system') {
if (message.content.some(block => block.type === 'image')) {
throw new LlmError('pi-ai cannot represent an image in an in-history system message', 'UNSUPPORTED_CONTENT')
}
// pi-ai has a single systemPrompt slot; in-history system messages are
// folded into user messages to preserve order (rare in practice — the
// harness sends the system prompt via options.system).
@@ -46,40 +149,26 @@ export function toPiContext(options: GenerateOptions): PiContext {
continue
}
// user role: text + tool results (each result becomes its own message).
const text = flattenText(message)
const regular = message.content.filter(block => block.type !== 'tool-result')
const content = await userContent(regular, attachments)
const results = message.content.filter(block => block.type === 'tool-result')
if (text.length > 0 || results.length === 0) {
messages.push({ role: 'user', content: text, timestamp: 0 })
if (content.length > 0 || results.length === 0) {
messages.push({ role: 'user', content, timestamp: 0 })
}
for (const result of results) {
const resultContent = await userContent(result.content, attachments)
messages.push({
role: 'toolResult',
toolCallId: result.toolCallId,
toolName: toolNames.get(result.toolCallId) ?? 'unknown',
content: [{
type: 'text',
text: result.content
.filter(block => block.type === 'text')
.map(block => block.text)
.join('') || '(no output)',
}],
content: typeof resultContent === 'string'
? [{ type: 'text', text: resultContent || '(no output)' }]
: resultContent,
isError: result.isError ?? false,
timestamp: 0,
})
}
}
const tools: PiTool[] | undefined = options.tools?.map(tool => ({
name: tool.name,
description: tool.description,
// ToolSchema.parameters is a JSON Schema object; pi-ai's TSchema
// (TypeBox) is structurally JSON Schema, so it assigns directly.
parameters: tool.parameters,
}))
return {
...options.system !== undefined ? { systemPrompt: options.system } : {},
messages,
...tools !== undefined && tools.length > 0 ? { tools } : {},
}
return piContext(options, messages)
}

6
packages/llm/llm-pi-ai/src/index.ts
View File

@@ -36,6 +36,10 @@ export const inject = ['llm']
/** Register one generic pi-ai adapter for all configured provider routes. */
export function apply(ctx: Context, config: Config): void {
const profiles = resolveProfiles(config.providers)
const adapter = new PiAiAdapter({ profiles })
const attachments = ctx.get('attachments')
const adapter = new PiAiAdapter({
profiles,
...(attachments === undefined ? {} : { attachments }),
})
ctx.llm.registerAdapter(profiles.map(entry => entry.provider), adapter)
}

2
packages/llm/llm-pi-ai/src/replay.ts
View File

@@ -134,6 +134,8 @@ function foreignAssistant(message: Message): AssistantMessage {
name: block.name,
arguments: parseArguments(block.arguments),
}); break
case 'image':
throw new LlmError('pi-ai chat history cannot represent structured assistant image output', 'UNSUPPORTED_CONTENT')
default:
// plugin-added block types are not representable in pi-ai.
break

1
packages/llm/llm-pi-ai/tests/adapter.spec.ts
View File

@@ -320,6 +320,7 @@ describe('provider profile lifecycle', () => {
const models = await ctx.llm.listModels('openai')
expect(models.find(model => model.id === 'gpt-4.1')).toEqual({
provider: 'openai', id: 'gpt-4.1', name: 'GPT-4.1',
inputModalities: ['text', 'image'], outputModalities: ['text'],
})
expect(models.every(model => model.provider === 'openai')).toBe(true)
const context = await ctx.llm.resolveModelContext('openai', 'gpt-4.1')

46
packages/llm/llm-pi-ai/tests/convert.spec.ts
View File

@@ -1,5 +1,7 @@
import { describe, expect, it } from 'vitest'
import { describe, expect, it, vi } from 'vitest'
import { CallId, CONTEXT_WINDOW_EXCEEDED_CODE, LlmError } from '@deepseek-ai/dsh-llm'
import { AttachmentId } from '@deepseek-ai/dsh-attachment'
import type { AttachmentStore } from '@deepseek-ai/dsh-attachment'
import type { ContentBlock, StreamChunk } from '@deepseek-ai/dsh-llm'
import type { AssistantMessage, AssistantMessageEvent, Usage } from '@earendil-works/pi-ai'
import { toPiContext } from '../src/context.ts'
@@ -63,6 +65,48 @@ describe('toPiContext', () => {
expect(context.tools).toBeUndefined()
})
it('resolves durable image references into native pi-ai image content', async () => {
const attachment = {
attachmentId: AttachmentId(`sha256:${'a'.repeat(64)}`),
mediaType: 'image/png' as const,
bytes: 3,
width: 1,
height: 1,
}
const readImage = vi.fn().mockResolvedValue({ ref: attachment, data: Uint8Array.of(1, 2, 3) })
const context = await toPiContext({
provider: 'openai',
model: 'gpt-4.1',
messages: [{
role: 'user',
content: [{ type: 'text', text: 'describe' }, { type: 'image', attachment }],
}],
}, { readImage } as unknown as AttachmentStore)
expect(readImage).toHaveBeenCalledWith(attachment)
expect(context.messages[0]).toEqual({
role: 'user',
content: [
{ type: 'text', text: 'describe' },
{ type: 'image', data: 'AQID', mimeType: 'image/png' },
],
timestamp: 0,
})
})
it('rejects structured image history when no durable resolver is supplied', () => {
expect(() => toPiContext({
provider: 'openai', model: 'gpt-4.1',
messages: [{ role: 'user', content: [{
type: 'image',
attachment: {
attachmentId: AttachmentId(`sha256:${'b'.repeat(64)}`),
mediaType: 'image/png', bytes: 1, width: 1, height: 1,
},
}] }],
})).toThrow(expect.objectContaining({ code: 'UNSUPPORTED_CONTENT' }))
})
it('maps assistant text/reasoning/tool-call blocks', () => {
const context = toPiContext({
provider: 'deepseek',

3
packages/llm/llm-pi-ai/tsconfig.json
View File

@@ -20,6 +20,9 @@
{
"path": "../../llm/llm"
},
{
"path": "../../attachment/attachment"
},
{
"path": "../../support/invariants"
},

2
packages/llm/llm/package.json
View File

@@ -36,11 +36,13 @@
],
"license": "BSD-3-Clause",
"peerDependencies": {
"@deepseek-ai/dsh-attachment": "^0.0.1",
"@deepseek-ai/dsh-brand": "^0.0.1",
"@deepseek-ai/dsh-invariants": "^0.0.1",
"cordis": "^4.0.0-rc.7"
},
"devDependencies": {
"@deepseek-ai/dsh-attachment": "workspace:^",
"@deepseek-ai/dsh-brand": "workspace:^",
"@deepseek-ai/dsh-invariants": "workspace:^",
"cordis": "^4.0.0-rc.7"

2
packages/llm/llm/src/index.ts
View File

@@ -235,6 +235,8 @@ export class LlmService extends Service {
id: model.id,
name: model.name,
...model.description === undefined ? {} : { description: model.description },
...model.inputModalities === undefined ? {} : { inputModalities: [...model.inputModalities] },
...model.outputModalities === undefined ? {} : { outputModalities: [...model.outputModalities] },
}
})
}

24
packages/llm/llm/src/types.ts
View File

@@ -5,6 +5,7 @@
*/
import type { Branded } from '@deepseek-ai/dsh-brand'
import type { ImageAttachmentRef } from '@deepseek-ai/dsh-attachment'
import type { CallId, ProviderRequestId } from './brand.ts'
/** Serializable provider-boundary facts; policy decides whether they are retryable. */
@@ -33,6 +34,15 @@ export interface ReasoningBlock {
text: string
}
/** A durable raster image reference, valid in user or assistant content. */
export interface ImageBlock {
type: 'image'
/** Immutable bytes and intrinsic display metadata owned by the attachment service. */
attachment: ImageAttachmentRef
/** Optional provider- and UI-facing alternative text. */
alt?: string
}
/** A tool invocation requested by the model. */
export interface ToolCallBlock {
type: 'tool-call'
@@ -58,6 +68,7 @@ export interface ToolResultBlock {
export interface ContentBlockMap {
'text': TextBlock
'reasoning': ReasoningBlock
'image': ImageBlock
'tool-call': ToolCallBlock
'tool-result': ToolResultBlock
}
@@ -143,6 +154,15 @@ export interface LlmProviderInfo {
name: string
}
/** Merge-extensible provider model modality vocabulary. */
export interface ModelModalityMap {
text: 'text'
image: 'image'
}
/** Any declared provider model modality. */
export type ModelModality = ModelModalityMap[keyof ModelModalityMap]
/** One adapter-discovered model; catalog membership is advisory, not request validation. */
export interface LlmModelInfo {
/** Provider route that owns this model entry. */
@@ -153,6 +173,10 @@ export interface LlmModelInfo {
name: string
/** Optional user-facing distinction from otherwise similar models. */
description?: string
/** Accepted request modalities; absent means unknown, while an explicit omission is negative capability. */
inputModalities?: readonly ModelModality[]
/** Structured response modalities; absent means unknown, while an explicit omission is negative capability. */
outputModalities?: readonly ModelModality[]
}
/** Provider-owned context capacity for one exact provider/model route. */

3
packages/llm/llm/tsconfig.json
View File

@@ -17,6 +17,9 @@
{
"path": "../../util/brand"
},
{
"path": "../../attachment/attachment"
},
{
"path": "../../support/invariants"
}

11
packages/llm/token-meter/src/index.ts
View File

@@ -25,6 +25,11 @@ const CHARS_PER_TOKEN = 4
/** Per-block structural overhead for JSON framing and type tags. */
const BLOCK_OVERHEAD = 4
/** Provider-neutral visual estimate: base cost plus one cost unit per 512px tile. */
const IMAGE_BASE_TOKENS = 85
const IMAGE_TILE_TOKENS = 170
const IMAGE_TILE_EDGE = 512
/** Role-field framing overhead added to every priced message. */
const ROLE_OVERHEAD = 4
@@ -357,6 +362,12 @@ export class TokenMeterService extends Service {
case 'reasoning':
tokens += Math.ceil(block.text.length / CHARS_PER_TOKEN) + BLOCK_OVERHEAD
break
case 'image': {
const tiles = Math.ceil(block.attachment.width / IMAGE_TILE_EDGE)
* Math.ceil(block.attachment.height / IMAGE_TILE_EDGE)
tokens += IMAGE_BASE_TOKENS + tiles * IMAGE_TILE_TOKENS + BLOCK_OVERHEAD
break
}
case 'tool-call':
tokens += Math.ceil(block.name.length / CHARS_PER_TOKEN)
+ Math.ceil(block.arguments.length / CHARS_PER_TOKEN)