refactor: narrow web image input v1
This commit is contained in:
@@ -6,19 +6,13 @@ import z from 'schemastery'
|
||||
import { AttachmentStore } from '@deepseek-ai/dsh-attachment'
|
||||
import type { ImageAttachmentLimits, ImageAttachmentRef, SaveImageAttachment, StoredImageAttachment } from '@deepseek-ai/dsh-attachment'
|
||||
import { resolveDshHome } from '@deepseek-ai/dsh-paths'
|
||||
import { readImageFile, saveImageFile, validateImageFile } from './store.ts'
|
||||
import { readImageFile, saveImageFile } from './store.ts'
|
||||
|
||||
export { detectImage } from './image.ts'
|
||||
export { readImageFile, saveImageFile, validateImageFile } from './store.ts'
|
||||
export { AttachmentError } from '@deepseek-ai/dsh-attachment'
|
||||
export type { ImageAttachmentRef } from '@deepseek-ai/dsh-attachment'
|
||||
export { readImageFile, saveImageFile } from './store.ts'
|
||||
|
||||
/** Default maximum encoded bytes for one image. */
|
||||
export const DEFAULT_MAX_IMAGE_BYTES = 5 * 1024 * 1024
|
||||
/** Default maximum images in one prompt. */
|
||||
export const DEFAULT_MAX_IMAGES_PER_MESSAGE = 10
|
||||
/** Default maximum aggregate image bytes in one prompt. */
|
||||
export const DEFAULT_MAX_MESSAGE_IMAGE_BYTES = 20 * 1024 * 1024
|
||||
/** Default maximum intrinsic pixels for one image. */
|
||||
export const DEFAULT_MAX_IMAGE_PIXELS = 40_000_000
|
||||
|
||||
@@ -28,10 +22,6 @@ export interface Config {
|
||||
dshHome?: string
|
||||
/** Maximum encoded bytes accepted for one image. */
|
||||
maxImageBytes?: number
|
||||
/** Maximum image count accepted in one submitted message. */
|
||||
maxImagesPerMessage?: number
|
||||
/** Maximum aggregate encoded image bytes accepted in one submitted message. */
|
||||
maxMessageImageBytes?: number
|
||||
/** Maximum intrinsic width multiplied by height accepted for one image. */
|
||||
maxImagePixels?: number
|
||||
}
|
||||
@@ -41,8 +31,6 @@ export class LocalAttachmentStore extends AttachmentStore {
|
||||
static Config: z<Config> = z.object({
|
||||
dshHome: z.string(),
|
||||
maxImageBytes: z.number().step(1).min(1).default(DEFAULT_MAX_IMAGE_BYTES),
|
||||
maxImagesPerMessage: z.number().step(1).min(1).default(DEFAULT_MAX_IMAGES_PER_MESSAGE),
|
||||
maxMessageImageBytes: z.number().step(1).min(1).default(DEFAULT_MAX_MESSAGE_IMAGE_BYTES),
|
||||
maxImagePixels: z.number().step(1).min(1).default(DEFAULT_MAX_IMAGE_PIXELS),
|
||||
})
|
||||
|
||||
@@ -55,17 +43,11 @@ export class LocalAttachmentStore extends AttachmentStore {
|
||||
this.root = resolve(join(resolveDshHome(config.dshHome), 'attachments', 'v1'))
|
||||
this.imageLimits = Object.freeze({
|
||||
maxImageBytes: config.maxImageBytes ?? DEFAULT_MAX_IMAGE_BYTES,
|
||||
maxImagesPerMessage: config.maxImagesPerMessage ?? DEFAULT_MAX_IMAGES_PER_MESSAGE,
|
||||
maxMessageImageBytes: config.maxMessageImageBytes ?? DEFAULT_MAX_MESSAGE_IMAGE_BYTES,
|
||||
maxImagePixels: config.maxImagePixels ?? DEFAULT_MAX_IMAGE_PIXELS,
|
||||
mediaTypes: Object.freeze(['image/png', 'image/jpeg', 'image/webp', 'image/gif'] as const),
|
||||
})
|
||||
}
|
||||
|
||||
validateImage(input: SaveImageAttachment): void {
|
||||
validateImageFile(input, this.imageLimits)
|
||||
}
|
||||
|
||||
async saveImage(input: SaveImageAttachment): Promise<ImageAttachmentRef> {
|
||||
return saveImageFile(this.root, input, this.imageLimits)
|
||||
}
|
||||
|
||||
@@ -52,15 +52,6 @@ function validateAdmission(metadata: Omit<ImageAttachmentRef, 'attachmentId' | '
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Run the full admission policy for one image without touching storage.
|
||||
* @param input - encoded bytes and declared metadata.
|
||||
* @param limits - resolved storage policy.
|
||||
*/
|
||||
export function validateImageFile(input: SaveImageAttachment, limits: ImageAttachmentLimits): void {
|
||||
validateAdmission(inspectMetadata(input.data, input.mediaType), limits)
|
||||
}
|
||||
|
||||
/**
|
||||
* Make a directory's entries durable (fsync on a read-only directory handle).
|
||||
* A synced file alone does not survive a crash when its directory entry never
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
import { Context } from 'cordis'
|
||||
import { existsSync } from 'node:fs'
|
||||
import { mkdtemp, rm } from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
@@ -7,8 +6,6 @@ import { describe, expect, it } from 'vitest'
|
||||
import LocalAttachmentStore, {
|
||||
DEFAULT_MAX_IMAGE_BYTES,
|
||||
DEFAULT_MAX_IMAGE_PIXELS,
|
||||
DEFAULT_MAX_IMAGES_PER_MESSAGE,
|
||||
DEFAULT_MAX_MESSAGE_IMAGE_BYTES,
|
||||
} from '../src/index.ts'
|
||||
|
||||
describe('local attachment service', () => {
|
||||
@@ -16,8 +13,6 @@ describe('local attachment service', () => {
|
||||
const service = new LocalAttachmentStore(new Context(), {})
|
||||
expect(service.imageLimits).toEqual({
|
||||
maxImageBytes: DEFAULT_MAX_IMAGE_BYTES,
|
||||
maxImagesPerMessage: DEFAULT_MAX_IMAGES_PER_MESSAGE,
|
||||
maxMessageImageBytes: DEFAULT_MAX_MESSAGE_IMAGE_BYTES,
|
||||
maxImagePixels: DEFAULT_MAX_IMAGE_PIXELS,
|
||||
mediaTypes: ['image/png', 'image/jpeg', 'image/webp', 'image/gif'],
|
||||
})
|
||||
@@ -37,22 +32,4 @@ describe('local attachment service', () => {
|
||||
await rm(dshHome, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
it('validates without persisting: a rejected image leaves no storage root behind', async () => {
|
||||
const dshHome = await mkdtemp(join(tmpdir(), 'dsh-attachment-validate-'))
|
||||
try {
|
||||
const service = new LocalAttachmentStore(new Context(), { dshHome })
|
||||
expect(() => { service.validateImage({ data: Uint8Array.of(1, 2, 3), mediaType: 'image/png' }) })
|
||||
.toThrow(/Unsupported or malformed image data/)
|
||||
const valid = Uint8Array.from(Buffer.from(
|
||||
'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=',
|
||||
'base64',
|
||||
))
|
||||
expect(() => { service.validateImage({ data: valid, mediaType: 'image/png' }) }).not.toThrow()
|
||||
// Validation is storage-free: nothing below the root may exist yet.
|
||||
expect(existsSync(service.root)).toBe(false)
|
||||
} finally {
|
||||
await rm(dshHome, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
@@ -28,8 +28,6 @@ const PNG = Uint8Array.from(Buffer.from(
|
||||
|
||||
const LIMITS: ImageAttachmentLimits = {
|
||||
maxImageBytes: 1024,
|
||||
maxImagesPerMessage: 2,
|
||||
maxMessageImageBytes: 2048,
|
||||
maxImagePixels: 16,
|
||||
mediaTypes: ['image/png', 'image/jpeg', 'image/webp', 'image/gif'],
|
||||
}
|
||||
|
||||
@@ -2,5 +2,5 @@
|
||||
# side as of the last confirmed-consistent state. Both languages carry equal authority;
|
||||
# after editing either side, bring the other along and re-record with:
|
||||
# pnpm run verify-translation-pairing --write packages/attachment/attachment/README.md
|
||||
README.md: b8cee33b21caf5851f5c571ee7385fbd1b81dc97
|
||||
README.zh.md: 53107bd0f57b73043073295a9698df927e457365
|
||||
README.md: b25ef7bcf89b3b85600ed02ab12eb0cdd1117244
|
||||
README.zh.md: f45933c2a011978b31a306a1de80d7f031838c8e
|
||||
|
||||
@@ -4,7 +4,7 @@ English | [中文](README.zh.md)
|
||||
|
||||
The durable attachment seam. `ctx.attachments` validates and atomically commits immutable image bytes, then returns a serializable `ImageAttachmentRef`; consumers never persist browser paths, object URLs, provider URLs, or base64 in session events.
|
||||
|
||||
Unsent composer images remain browser-owned temporary drafts. `saveImage` is called only at message submission or while committing structured provider output, before any model-visible session event is published. `validateImage` runs the same admission policy without persisting; batch writers validate every member first so one malformed member cannot strand earlier members as unreferenced objects (there is no garbage collection). `readImage` verifies the content-addressed object against its logged metadata.
|
||||
Unsent composer images remain browser-owned temporary drafts. `saveImage` validates and commits one image at message submission or while committing structured provider output, before any model-visible session event is published. `readImage` verifies the content-addressed object against its logged metadata.
|
||||
|
||||
## Model Experience
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
|
||||
持久附件服务边界。`ctx.attachments` 校验并以原子方式提交不可变图片字节,随后返回可序列化的 `ImageAttachmentRef`;消费方绝不会在会话事件中持久保存浏览器路径、对象 URL、提供方 URL 或 base64。
|
||||
|
||||
未发送的输入区图片仍是由浏览器持有的临时草稿。只有在提交消息或提交结构化提供方输出时,才会调用 `saveImage`,并且必须先于任何模型可见的会话事件发布。`validateImage` 运行相同的准入策略,但不执行持久化;批量写入方会先校验每个成员,避免某个格式错误的成员使较早的成员成为无引用对象(系统不提供垃圾回收)。`readImage` 根据已记录的元数据校验内容寻址对象。
|
||||
未发送的输入区图片仍是由浏览器持有的临时草稿。`saveImage` 在提交消息或提交结构化提供方输出时校验并提交一张图片,且发生在发布任何模型可见的会话事件之前。`readImage` 根据已记录的元数据校验内容寻址对象。
|
||||
|
||||
## 模型体验
|
||||
|
||||
|
||||
@@ -33,14 +33,6 @@ export abstract class AttachmentStore extends Service {
|
||||
/** Deployment-resolved image policy used by authoritative and fast-path validation. */
|
||||
abstract readonly imageLimits: ImageAttachmentLimits
|
||||
|
||||
/**
|
||||
* Validate one image against the deployment policy without persisting anything.
|
||||
* Callers persisting a multi-image batch validate every member first so a
|
||||
* malformed member cannot leave earlier members as unreferenced objects.
|
||||
* @param input - encoded bytes, declared media type, and optional display name.
|
||||
*/
|
||||
abstract validateImage(input: SaveImageAttachment): void
|
||||
|
||||
/**
|
||||
* Validate and durably commit one image before its owning session event is appended.
|
||||
* @param input - encoded bytes, declared media type, and optional display name.
|
||||
|
||||
@@ -36,8 +36,6 @@ export interface ImageAttachmentRef {
|
||||
/** Deployment-resolved limits shared by upload consumers and UI preflight. */
|
||||
export interface ImageAttachmentLimits {
|
||||
maxImageBytes: number
|
||||
maxImagesPerMessage: number
|
||||
maxMessageImageBytes: number
|
||||
maxImagePixels: number
|
||||
mediaTypes: readonly ImageMediaType[]
|
||||
}
|
||||
|
||||
@@ -1260,17 +1260,8 @@ export function createFixtureApi(options: FixtureOptions = {}): ApiProxy {
|
||||
cwd: '/tmp/fixture',
|
||||
provider: 'fixture',
|
||||
model: 'fx-vision',
|
||||
activeModel: {
|
||||
provider: 'fixture',
|
||||
id: 'fx-vision',
|
||||
name: 'Fixture Vision',
|
||||
inputModalities: ['text', 'image'],
|
||||
outputModalities: ['text', 'image'],
|
||||
},
|
||||
imageLimits: {
|
||||
maxImageBytes: 5 * 1024 * 1024,
|
||||
maxImagesPerMessage: 10,
|
||||
maxMessageImageBytes: 20 * 1024 * 1024,
|
||||
maxImagePixels: 40_000_000,
|
||||
mediaTypes: ['image/png', 'image/jpeg', 'image/webp', 'image/gif'],
|
||||
},
|
||||
|
||||
@@ -52,7 +52,7 @@ export function apply(ctx: Context, config?: ConnectionConfig): void {
|
||||
for (const entry of trustedHosts) assertTrustedAuthority(entry)
|
||||
const apiHandler = toFetchHandler(ctx.apiProxy)
|
||||
const maxRequestBodyBytes = Math.ceil(
|
||||
ctx.attachments.imageLimits.maxMessageImageBytes * 4 / 3,
|
||||
ctx.attachments.imageLimits.maxImageBytes * 4 / 3,
|
||||
) + REQUEST_ENVELOPE_HEADROOM_BYTES
|
||||
const route: WebRoute = {
|
||||
kind: 'prefix',
|
||||
|
||||
@@ -23,7 +23,7 @@ function fakeHttpServer(routes: WebRoute[]): Pick<HttpServerService, 'register'
|
||||
|
||||
/** Structural attachments fake: the plugin only reads imageLimits for the body cap. */
|
||||
function fakeAttachments(): AttachmentStore {
|
||||
return { imageLimits: { maxMessageImageBytes: 20 * 1024 * 1024 } } as AttachmentStore
|
||||
return { imageLimits: { maxImageBytes: 5 * 1024 * 1024 } } as AttachmentStore
|
||||
}
|
||||
|
||||
/** Bodyless GET carrying the given headers (enough for the trust fence + bridge). */
|
||||
|
||||
@@ -36,7 +36,7 @@ export interface AssistantProvenanceView {
|
||||
export type AssistantBlock =
|
||||
| { kind: 'text'; text: string }
|
||||
| { kind: 'reasoning'; text: string }
|
||||
| { kind: 'image'; attachment: ImageAttachmentRef; alt?: string }
|
||||
| { kind: 'image'; attachment: ImageAttachmentRef }
|
||||
| { kind: 'tool-call'; callId: string; name: string; argsRaw: string }
|
||||
| { kind: 'other'; block: unknown }
|
||||
|
||||
@@ -58,11 +58,7 @@ export function toAssistantBlock(block: ContentBlock): AssistantBlock {
|
||||
switch (block.type) {
|
||||
case 'text': return { kind: 'text', text: block.text }
|
||||
case 'reasoning': return { kind: 'reasoning', text: block.text }
|
||||
case 'image': return {
|
||||
kind: 'image',
|
||||
attachment: block.attachment,
|
||||
...block.alt === undefined ? {} : { alt: block.alt },
|
||||
}
|
||||
case 'image': return { kind: 'image', attachment: block.attachment }
|
||||
case 'tool-call': return { kind: 'tool-call', callId: String(block.id), name: block.name, argsRaw: block.arguments }
|
||||
default: return { kind: 'other', block }
|
||||
}
|
||||
|
||||
@@ -39,6 +39,13 @@ function scopedConversation(sessions: ISessions, id: SessionId): IConversation {
|
||||
return conversation
|
||||
}
|
||||
|
||||
/** Resolve package-internal attachment operations from the public service registration. */
|
||||
function concreteConversation(ctx: Context): ConversationService {
|
||||
const conversation = ctx.get('conversation') as ConversationService | undefined
|
||||
if (conversation === undefined) throw new Error('ui-conversation: conversation service unavailable')
|
||||
return conversation
|
||||
}
|
||||
|
||||
/** Chain routing: claim the composer while an approval wait is pending (pure — owner props only). */
|
||||
function selectApproval({ interactions }: ComposerChainProps): ApprovalWait | null {
|
||||
return interactions.find((i): i is ApprovalWait => i.kind === 'approval') ?? null
|
||||
@@ -157,8 +164,7 @@ export function apply(ctx: Context): void {
|
||||
children: { 'conversation.view': { kind: 'list', scope: 'session' } },
|
||||
store: chatStore,
|
||||
inject: (sessionId: SessionId, _actions: BoundActions<typeof chatStore>): ConversationSessionInjected => {
|
||||
const conversation = ctx.get('conversation')
|
||||
if (conversation === undefined) throw new Error('ui-conversation: conversation service unavailable')
|
||||
const conversation = concreteConversation(ctx)
|
||||
return {
|
||||
views: {
|
||||
list: viewTabs,
|
||||
@@ -186,8 +192,7 @@ export function apply(ctx: Context): void {
|
||||
'conversation.input.model': { kind: 'single', scope: 'session' },
|
||||
},
|
||||
inject: (sessionId: SessionId): ComposerBarInjected => {
|
||||
const conversation = ctx.get('conversation')
|
||||
if (conversation === undefined) throw new Error('ui-conversation: conversation service unavailable')
|
||||
const conversation = concreteConversation(ctx)
|
||||
const shell = inputHub.shell(sessionId)
|
||||
return {
|
||||
keyboard: shell,
|
||||
@@ -248,8 +253,7 @@ export function apply(ctx: Context): void {
|
||||
},
|
||||
store: chatStore,
|
||||
inject: (sessionId: SessionId, actions: BoundActions<typeof chatStore>): ChatViewInjected => {
|
||||
const conversation = ctx.get('conversation')
|
||||
if (conversation === undefined) throw new Error('ui-conversation: conversation service unavailable')
|
||||
const conversation = concreteConversation(ctx)
|
||||
const scoped = scopedConversation(sessions, sessionId)
|
||||
return {
|
||||
openDetails: (target) => {
|
||||
|
||||
@@ -7,9 +7,8 @@ import css from './MessageImage.module.css'
|
||||
export type ImageLoader = (attachment: ImageAttachmentRef) => Promise<string>
|
||||
|
||||
/** Compact history renderer with retryable loading and double-click original preview. */
|
||||
export function MessageImage({ attachment, alt, load }: {
|
||||
export function MessageImage({ attachment, load }: {
|
||||
attachment: ImageAttachmentRef
|
||||
alt?: string
|
||||
load: ImageLoader
|
||||
}) {
|
||||
const [src, setSrc] = useState<string | null>(null)
|
||||
@@ -34,7 +33,7 @@ export function MessageImage({ attachment, alt, load }: {
|
||||
return () => { live = false }
|
||||
}, [attachment, load])
|
||||
|
||||
const label = alt ?? attachment.name ?? '图片'
|
||||
const label = attachment.name ?? '图片'
|
||||
if (error) return <button type="button" className={css.error} onClick={request}>图片加载失败,点击重试</button>
|
||||
return (
|
||||
<>
|
||||
@@ -55,7 +54,7 @@ export function MessageImage({ attachment, alt, load }: {
|
||||
|
||||
/** Wrapping image group shared by user and assistant history. */
|
||||
export function ImageGallery({ images, load, align }: {
|
||||
images: readonly { attachment: ImageAttachmentRef; alt?: string }[]
|
||||
images: readonly { attachment: ImageAttachmentRef }[]
|
||||
load: ImageLoader
|
||||
align: 'start' | 'end'
|
||||
}) {
|
||||
|
||||
@@ -23,18 +23,17 @@ type UserImage = Extract<UserMessageNode['content'][number], { type: 'image' }>
|
||||
|
||||
function contentParts(content: readonly unknown[]): {
|
||||
text: string
|
||||
images: { attachment: UserImage['attachment']; alt?: string }[]
|
||||
images: { attachment: UserImage['attachment'] }[]
|
||||
rest: unknown[]
|
||||
} {
|
||||
const texts: string[] = []
|
||||
const images: { attachment: UserImage['attachment']; alt?: string }[] = []
|
||||
const images: { attachment: UserImage['attachment'] }[] = []
|
||||
const rest: unknown[] = []
|
||||
for (const block of content) {
|
||||
const b = block as { type?: string; text?: string; attachment?: unknown; alt?: string }
|
||||
const b = block as { type?: string; text?: string; attachment?: unknown }
|
||||
if (b.type === 'text' && typeof b.text === 'string') texts.push(b.text)
|
||||
else if (b.type === 'image' && b.attachment !== undefined) {
|
||||
const image = b as UserImage
|
||||
images.push({ attachment: image.attachment, ...image.alt === undefined ? {} : { alt: image.alt } })
|
||||
images.push({ attachment: (b as UserImage).attachment })
|
||||
}
|
||||
else rest.push(block)
|
||||
}
|
||||
|
||||
@@ -151,7 +151,12 @@ export class InputHub implements InputService {
|
||||
// see them — release the drafts here instead of resurrecting them onto
|
||||
// a dead instance where they would leak for the page lifetime.
|
||||
if (this.shells.get(session.sessionId) === shell) {
|
||||
shell?.restoreImages(imageIds)
|
||||
if (shell?.snapshot.imageIds.length === 0) {
|
||||
shell.restoreImages(imageIds)
|
||||
} else {
|
||||
const conversation = this.rootCtx.get('conversation') as ConversationAttachmentFace | undefined
|
||||
for (const id of imageIds) conversation?.releaseDraftImage(id)
|
||||
}
|
||||
if (shell?.snapshot.draft === '') shell.setDraft(text)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -29,10 +29,9 @@ export interface IConversation {
|
||||
* Send a prompt into the caller scope's session.
|
||||
* @param text - prompt text, sent verbatim as one text block.
|
||||
* @param mode - queue after the current turn, or steer into it.
|
||||
* @param images - browser-owned temporary images promoted by the host during this call.
|
||||
* @returns completion; business failures reject (and land in promptError).
|
||||
*/
|
||||
send(text: string, mode: 'queue' | 'steer', images?: readonly File[]): Promise<void>
|
||||
send(text: string, mode: 'queue' | 'steer'): Promise<void>
|
||||
/**
|
||||
* Cancel the scoped session's in-flight turn.
|
||||
* @returns completion; failures reject as in send.
|
||||
@@ -43,57 +42,11 @@ export interface IConversation {
|
||||
* @returns completion of the page pull.
|
||||
*/
|
||||
loadOlder(): Promise<void>
|
||||
/**
|
||||
* Create runtime-only draft attachments and preview URLs.
|
||||
* @param files - browser-owned image files.
|
||||
* @param current - images already present in the composer.
|
||||
* @returns ordered descriptors for the input state.
|
||||
*/
|
||||
createDraftImages(
|
||||
files: readonly File[],
|
||||
current?: readonly ComposerAttachment[],
|
||||
): readonly ComposerAttachment[]
|
||||
/**
|
||||
* Resolve ordered draft ids to runtime-owned attachments.
|
||||
* @param ids - ordered composer attachment ids.
|
||||
* @returns attachments still available in this browser runtime.
|
||||
*/
|
||||
draftImages(ids: readonly string[]): readonly ComposerAttachment[]
|
||||
/**
|
||||
* Release one draft attachment and its preview URL.
|
||||
* @param id - draft-local attachment id.
|
||||
*/
|
||||
releaseDraftImage(id: string): void
|
||||
/**
|
||||
* Resolve a session-authorized historical image to an object URL.
|
||||
* @param sessionId - session whose durable log grants the read.
|
||||
* @param attachment - durable image reference from that log.
|
||||
* @returns browser URL for inline and original-size rendering.
|
||||
*/
|
||||
resolveImage(sessionId: SessionId, attachment: ImageAttachmentRef): Promise<string>
|
||||
/**
|
||||
* Release every historical image URL owned by one rendered session.
|
||||
* @param sessionId - session whose rendered image scope is ending.
|
||||
*/
|
||||
releaseSessionImages(sessionId: SessionId): void
|
||||
}
|
||||
|
||||
/** Opaque wrapper keeps browser `File` internals outside persisted store state. */
|
||||
class BrowserDraftAttachment implements ComposerAttachment {
|
||||
readonly kind = 'image' as const
|
||||
readonly id: string
|
||||
readonly previewUrl: string
|
||||
readonly #file: File
|
||||
|
||||
constructor(file: File) {
|
||||
this.id = crypto.randomUUID()
|
||||
this.previewUrl = URL.createObjectURL(file)
|
||||
this.#file = file
|
||||
}
|
||||
|
||||
get file(): File {
|
||||
return this.#file
|
||||
}
|
||||
/** Create one browser-only draft descriptor; only its id enters input state. */
|
||||
function browserDraftAttachment(file: File): ComposerAttachment {
|
||||
return { kind: 'image', id: crypto.randomUUID(), previewUrl: URL.createObjectURL(file), file }
|
||||
}
|
||||
|
||||
interface ImageUrlEntry {
|
||||
@@ -106,7 +59,7 @@ interface ImageUrlEntry {
|
||||
export class ConversationService extends Service implements IConversation {
|
||||
/** The per-session input machine registry (InputService face, design §5.2). */
|
||||
readonly input: InputService
|
||||
private readonly draftAttachments = new Map<string, BrowserDraftAttachment>()
|
||||
private readonly draftAttachments = new Map<string, ComposerAttachment>()
|
||||
private readonly imageUrls = new Map<string, ImageUrlEntry>()
|
||||
private readonly imageGenerations = new Map<SessionId, number>()
|
||||
private readonly createdImageUrls = new Set<string>()
|
||||
@@ -135,11 +88,10 @@ export class ConversationService extends Service implements IConversation {
|
||||
* exists for caller choreography (the composer restores the draft on it).
|
||||
* @param text - prompt text, sent verbatim as one text block when non-empty.
|
||||
* @param mode - queue after the current turn, or steer into it.
|
||||
* @param images - browser-owned temporary images promoted by the host during this call.
|
||||
*/
|
||||
async send(text: string, mode: 'queue' | 'steer', images: readonly File[] = []): Promise<void> {
|
||||
async send(text: string, mode: 'queue' | 'steer'): Promise<void> {
|
||||
const session = this.scopedSession('send')
|
||||
await this.sendFiles(session, text, mode, images)
|
||||
await this.sendFiles(session, text, mode, [])
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -190,7 +142,7 @@ export class ConversationService extends Service implements IConversation {
|
||||
): readonly ComposerAttachment[] {
|
||||
this.validateImages(files, current)
|
||||
return files.map((file) => {
|
||||
const attachment = new BrowserDraftAttachment(file)
|
||||
const attachment = browserDraftAttachment(file)
|
||||
this.draftAttachments.set(attachment.id, attachment)
|
||||
this.createdImageUrls.add(attachment.previewUrl)
|
||||
return attachment
|
||||
@@ -330,19 +282,12 @@ export class ConversationService extends Service implements IConversation {
|
||||
current: readonly ComposerAttachment[],
|
||||
): void {
|
||||
if (files.length === 0 && current.length === 0) return
|
||||
// Deployment-wide limits only. Model capability is deliberately NOT
|
||||
// checked here: the handshake's activeModel is the host default, not the
|
||||
// session's current target (session.selectModel never refreshes it), so a
|
||||
// client-side modality gate refuses sessions the host would accept and
|
||||
// vice versa. The host preflight on session.prompt is the authority; its
|
||||
// rejection renders through the composer error strip.
|
||||
// Model capability is checked only by the host against the session's
|
||||
// current target; the client owns deployment limits and the one-image UI.
|
||||
const description = this.requireSessions().hostDescription()
|
||||
const limits = description?.imageLimits
|
||||
const all = [...current.map(attachment => attachment.file), ...files]
|
||||
if (limits !== undefined && all.length > limits.maxImagesPerMessage) {
|
||||
throw new Error(`每条消息最多添加 ${limits.maxImagesPerMessage} 张图片`)
|
||||
}
|
||||
let totalBytes = 0
|
||||
if (all.length > 1) throw new Error('每条消息最多添加 1 张图片')
|
||||
for (const file of all) {
|
||||
const mediaType = imageMediaType(file.type)
|
||||
if (limits !== undefined && !limits.mediaTypes.includes(mediaType)) {
|
||||
@@ -351,10 +296,6 @@ export class ConversationService extends Service implements IConversation {
|
||||
if (limits !== undefined && file.size > limits.maxImageBytes) {
|
||||
throw new Error(`图片 ${file.name || '未命名图片'} 超过单张大小限制`)
|
||||
}
|
||||
totalBytes += file.size
|
||||
}
|
||||
if (limits !== undefined && totalBytes > limits.maxMessageImageBytes) {
|
||||
throw new Error('图片总大小超过单条消息限制')
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -48,14 +48,14 @@ describe('MessageImage', () => {
|
||||
<AssistantMarkdown
|
||||
blocks={[
|
||||
{ kind: 'text', text: 'before' },
|
||||
{ kind: 'image', attachment, alt: 'middle' },
|
||||
{ kind: 'image', attachment },
|
||||
{ kind: 'text', text: 'after' },
|
||||
]}
|
||||
streaming={false}
|
||||
loadImage={() => Promise.resolve('blob:middle')}
|
||||
/>,
|
||||
)
|
||||
const image = await view.findByAltText('middle')
|
||||
const image = await view.findByAltText('history.png')
|
||||
const before = view.getByText('before')
|
||||
const after = view.getByText('after')
|
||||
expect(before.compareDocumentPosition(image) & Node.DOCUMENT_POSITION_FOLLOWING).not.toBe(0)
|
||||
|
||||
@@ -29,6 +29,25 @@ async function bench() {
|
||||
}
|
||||
|
||||
describe('ConversationService', () => {
|
||||
it('keeps the browser draft to one image', async () => {
|
||||
const b = await bench()
|
||||
const created = vi.spyOn(URL, 'createObjectURL').mockReturnValue('blob:draft-one')
|
||||
const revoked = vi.spyOn(URL, 'revokeObjectURL').mockReturnValue(undefined)
|
||||
try {
|
||||
const [first] = b.root.createDraftImages([new File([Uint8Array.of(1)], 'first.png', { type: 'image/png' })])
|
||||
if (first === undefined) throw new Error('draft attachment missing')
|
||||
expect(() => b.root.createDraftImages(
|
||||
[new File([Uint8Array.of(2)], 'second.png', { type: 'image/png' })],
|
||||
[first],
|
||||
)).toThrow('每条消息最多添加 1 张图片')
|
||||
expect(created).toHaveBeenCalledOnce()
|
||||
} finally {
|
||||
created.mockRestore()
|
||||
revoked.mockRestore()
|
||||
}
|
||||
await b.runtime.dispose()
|
||||
})
|
||||
|
||||
it('routes operations through the public Session binding', async () => {
|
||||
const b = await bench()
|
||||
await b.scoped.send('hello', 'steer')
|
||||
|
||||
@@ -644,7 +644,6 @@ function assistantSourceBlock(block: AssistantBlock): TrajectorySourceBlock {
|
||||
case 'image': return {
|
||||
type: 'image',
|
||||
content: stringifySourceValue(block.attachment),
|
||||
...(block.alt !== undefined ? { imageAlt: block.alt } : {}),
|
||||
}
|
||||
case 'other': return sourceBlock(block.block)
|
||||
}
|
||||
|
||||
@@ -60,7 +60,7 @@ describe('tsdown client artifact', () => {
|
||||
const { handoff, surface } = await loadArtifact()
|
||||
expect(handoff.id).toBe(PLUGIN_ID)
|
||||
expect(surface.apply).toBeTypeOf('function')
|
||||
expect(surface.inject).toEqual(['slots', 'conversation', 'sessions'])
|
||||
expect(surface.inject).toEqual(['slots', 'conversation', 'sessionHistory'])
|
||||
})
|
||||
|
||||
it.skipIf(code === undefined)('mounted as an object plugin, apply registers the view tab on the real ring', async () => {
|
||||
@@ -72,10 +72,10 @@ describe('tsdown client artifact', () => {
|
||||
name: 'root',
|
||||
children: { 'conversation.view': { kind: 'list', scope: 'session' } },
|
||||
}, (_p: { renderSlot?: unknown }) => null)
|
||||
// The plugin injects 'conversation' as an ordering edge and 'sessions'
|
||||
// The plugin injects 'conversation' as an ordering edge and 'sessionHistory'
|
||||
// for its per-session history callback; this bench supplies both.
|
||||
ctx.provide('conversation', {})
|
||||
ctx.provide('sessions', {})
|
||||
ctx.provide('sessionHistory', {})
|
||||
const fiber = ctx.plugin(surface as { apply: (ctx: Context) => void })
|
||||
await fiber.await()
|
||||
expect(slots.entries('conversation.view').map(e => e.options.id)).toEqual(['trajectory'])
|
||||
|
||||
@@ -160,10 +160,6 @@ export const SERVICE_API: readonly ServiceApiEntry[] = [
|
||||
key: 'attachments',
|
||||
summary: 'Immutable binary attachment service.',
|
||||
methods: [
|
||||
{
|
||||
signature: 'abstract validateImage(input: SaveImageAttachment): void',
|
||||
jsDoc: '/**\n * Validate one image against the deployment policy without persisting anything.\n * Callers persisting a multi-image batch validate every member first so a\n * malformed member cannot leave earlier members as unreferenced objects.\n * @param input - encoded bytes, declared media type, and optional display name.\n */',
|
||||
},
|
||||
{
|
||||
signature: 'abstract saveImage(input: SaveImageAttachment): Promise<ImageAttachmentRef>',
|
||||
jsDoc: '/**\n * Validate and durably commit one image before its owning session event is appended.\n * @param input - encoded bytes, declared media type, and optional display name.\n * @returns a durable content-addressed reference.\n */',
|
||||
@@ -1859,7 +1855,7 @@ export const TYPE_API: readonly TypeApiEntry[] = [
|
||||
},
|
||||
{
|
||||
name: 'ImageBlock',
|
||||
declaration: 'export interface ImageBlock {\n type: \'image\';\n attachment: ImageAttachmentRef;\n alt?: string;\n}',
|
||||
declaration: 'export interface ImageBlock {\n type: \'image\';\n attachment: ImageAttachmentRef;\n}',
|
||||
},
|
||||
{
|
||||
name: 'ImageMediaType',
|
||||
@@ -1919,7 +1915,7 @@ export const TYPE_API: readonly TypeApiEntry[] = [
|
||||
},
|
||||
{
|
||||
name: 'LlmModelInfo',
|
||||
declaration: 'export interface LlmModelInfo {\n provider: string;\n id: string;\n name: string;\n description?: string;\n inputModalities?: readonly ModelModality[];\n outputModalities?: readonly ModelModality[];\n}',
|
||||
declaration: 'export interface LlmModelInfo {\n provider: string;\n id: string;\n name: string;\n description?: string;\n inputModalities?: readonly ModelModality[];\n}',
|
||||
},
|
||||
{
|
||||
name: 'LlmModelReasoningInfo',
|
||||
|
||||
@@ -41,7 +41,6 @@
|
||||
"license": "BSD-3-Clause",
|
||||
"dependencies": {
|
||||
"@deepseek-ai/dsh-attachment": "workspace:^",
|
||||
"@deepseek-ai/dsh-attachment-local": "workspace:^",
|
||||
"@deepseek-ai/dsh-agent": "workspace:^",
|
||||
"@deepseek-ai/dsh-brand": "workspace:^",
|
||||
"@deepseek-ai/dsh-commands": "workspace:^",
|
||||
|
||||
@@ -11,8 +11,8 @@ import { installAgentLlmTarget } from '@deepseek-ai/dsh-agent'
|
||||
import type {
|
||||
Agent, AgentLlmTarget, AgentLlmTargetRef, AgentStatus, InboxPlacement,
|
||||
} from '@deepseek-ai/dsh-agent'
|
||||
import { AttachmentError } from '@deepseek-ai/dsh-attachment-local'
|
||||
import type { ImageAttachmentRef } from '@deepseek-ai/dsh-attachment-local'
|
||||
import { AttachmentError } from '@deepseek-ai/dsh-attachment'
|
||||
import type { ImageAttachmentRef } from '@deepseek-ai/dsh-attachment'
|
||||
import { createUserMessage, ReasoningEffortId } from '@deepseek-ai/dsh-llm'
|
||||
import { errorChain } from '@deepseek-ai/dsh-llm'
|
||||
import type { ContentBlock, MessageId, MessageSource } from '@deepseek-ai/dsh-llm'
|
||||
@@ -79,37 +79,23 @@ async function durablePromptContent(ctx: Context, content: readonly PromptConten
|
||||
if (content.every(part => part.type === 'text')) {
|
||||
return content.map(part => ({ type: 'text', text: part.text }))
|
||||
}
|
||||
const limits = ctx.attachments.imageLimits
|
||||
const prepared = content.map(part => part.type === 'text'
|
||||
? part
|
||||
: { part, data: decodeBase64(part.data) })
|
||||
const images = prepared.filter((part): part is Extract<typeof part, { data: Uint8Array }> => 'data' in part)
|
||||
if (images.length > limits.maxImagesPerMessage) {
|
||||
throw new AttachmentError('Prompt exceeds the configured image-count limit.', 'TOO_MANY_IMAGES')
|
||||
if (content.filter(part => part.type === 'image').length > 1) {
|
||||
throw new AttachmentError('A prompt may contain at most one image.', 'TOO_MANY_IMAGES')
|
||||
}
|
||||
const totalBytes = images.reduce((sum, image) => sum + image.data.byteLength, 0)
|
||||
if (totalBytes > limits.maxMessageImageBytes) {
|
||||
throw new AttachmentError('Prompt exceeds the configured aggregate image-byte limit.', 'IMAGES_TOO_LARGE')
|
||||
}
|
||||
// Validate the complete batch before persisting any member: the store has no
|
||||
// garbage collection, so one malformed image must not leave the batch's
|
||||
// valid members as published objects no message event will ever reference.
|
||||
for (const image of images) {
|
||||
ctx.attachments.validateImage({
|
||||
data: image.data,
|
||||
mediaType: image.part.mediaType,
|
||||
...image.part.name === undefined ? {} : { name: image.part.name },
|
||||
})
|
||||
}
|
||||
return Promise.all(prepared.map(async (item): Promise<ContentBlock> => {
|
||||
if (!('data' in item)) return { type: 'text', text: item.text }
|
||||
const durable: ContentBlock[] = []
|
||||
for (const part of content) {
|
||||
if (part.type === 'text') {
|
||||
durable.push({ type: 'text', text: part.text })
|
||||
continue
|
||||
}
|
||||
const attachment = await ctx.attachments.saveImage({
|
||||
data: item.data,
|
||||
mediaType: item.part.mediaType,
|
||||
...item.part.name === undefined ? {} : { name: item.part.name },
|
||||
data: decodeBase64(part.data),
|
||||
mediaType: part.mediaType,
|
||||
...part.name === undefined ? {} : { name: part.name },
|
||||
})
|
||||
return { type: 'image', attachment, ...item.part.alt === undefined ? {} : { alt: item.part.alt } }
|
||||
}))
|
||||
durable.push({ type: 'image', attachment })
|
||||
}
|
||||
return durable
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -1222,8 +1208,8 @@ export function createApiProxy(ctx: Context, defaults: ApiProxyDefaults): ApiPro
|
||||
const target = targetFor(agent).current
|
||||
const provider = target.provider
|
||||
const model = target.model
|
||||
const activeModel = await ctx.llm.resolveModelInfo(provider, model)
|
||||
if (activeModel.inputModalities !== undefined && !activeModel.inputModalities.includes('image')) {
|
||||
const modelInfo = await ctx.llm.resolveModelInfo(provider, model)
|
||||
if (modelInfo.inputModalities !== undefined && !modelInfo.inputModalities.includes('image')) {
|
||||
return err(request, {
|
||||
code: 'attachment-error',
|
||||
message: `Model "${model}" does not support image input.`,
|
||||
@@ -1419,24 +1405,21 @@ export function createApiProxy(ctx: Context, defaults: ApiProxyDefaults): ApiPro
|
||||
},
|
||||
|
||||
host: {
|
||||
async describe(request) {
|
||||
const activeModel = (await ctx.llm.listModels(defaults.provider))
|
||||
.find(model => model.id === defaults.model)
|
||||
describe(request) {
|
||||
// TODO(step2): version should read apps/cli's package.json; placeholder for now.
|
||||
return ok(request, {
|
||||
return Promise.resolve(ok(request, {
|
||||
version: '0.0.1',
|
||||
// Same source as session.create's fallback: the UI's default project
|
||||
// must match where an unspecified-cwd session actually lands.
|
||||
cwd: defaults.cwd,
|
||||
provider: defaults.provider,
|
||||
model: defaults.model,
|
||||
...activeModel === undefined ? {} : { activeModel },
|
||||
imageLimits: {
|
||||
...ctx.attachments.imageLimits,
|
||||
mediaTypes: [...ctx.attachments.imageLimits.mediaTypes],
|
||||
},
|
||||
attachedSessions: ctx.agents.list().length,
|
||||
})
|
||||
}))
|
||||
},
|
||||
|
||||
async pickDirectory(request, signal) {
|
||||
|
||||
@@ -3,15 +3,11 @@
|
||||
*/
|
||||
|
||||
import { z } from 'zod'
|
||||
import type { ModelModality } from '@deepseek-ai/dsh-llm'
|
||||
import type { DirectoryEntry } from './host.ts'
|
||||
import type { RequestPayload, ResponseValue } from './rpc-map.ts'
|
||||
import type { Wire } from './rpc.schema.ts'
|
||||
import { imageMediaTypeSchema } from './sessions.schema.ts'
|
||||
|
||||
/** Merge-extensible modality passthrough: declaration merging cannot extend a runtime Zod union. */
|
||||
const modalitySchema = z.string() as unknown as z.ZodType<ModelModality>
|
||||
|
||||
/** host.describe request payload (empty object literal). */
|
||||
export const hostDescribeRequestSchema = z.object({}) satisfies z.ZodType<Wire<RequestPayload<'host.describe'>>>
|
||||
|
||||
@@ -21,18 +17,8 @@ export const hostDescribeValueSchema = z.object({
|
||||
cwd: z.string(),
|
||||
provider: z.string().optional(),
|
||||
model: z.string().optional(),
|
||||
activeModel: z.object({
|
||||
provider: z.string(),
|
||||
id: z.string(),
|
||||
name: z.string(),
|
||||
description: z.string().optional(),
|
||||
inputModalities: z.array(modalitySchema).optional(),
|
||||
outputModalities: z.array(modalitySchema).optional(),
|
||||
}).optional(),
|
||||
imageLimits: z.object({
|
||||
maxImageBytes: z.number().int().positive(),
|
||||
maxImagesPerMessage: z.number().int().positive(),
|
||||
maxMessageImageBytes: z.number().int().positive(),
|
||||
maxImagePixels: z.number().int().positive(),
|
||||
mediaTypes: z.array(imageMediaTypeSchema),
|
||||
}).optional(),
|
||||
|
||||
@@ -5,7 +5,6 @@
|
||||
|
||||
import type { RpcRequest, RpcResponse } from './rpc.ts'
|
||||
import type { ImageAttachmentLimits } from '@deepseek-ai/dsh-attachment'
|
||||
import type { LlmModelInfo } from '@deepseek-ai/dsh-llm/types'
|
||||
|
||||
/** One directory row of a listing: a child entry or a breadcrumb ancestor. */
|
||||
export interface DirectoryEntry {
|
||||
@@ -49,8 +48,6 @@ export interface HostApi {
|
||||
cwd: string
|
||||
provider?: string
|
||||
model?: string
|
||||
/** Catalog entry for the active route; absent means its capabilities are unknown. */
|
||||
activeModel?: LlmModelInfo
|
||||
/** Resolved authoritative image-upload limits. */
|
||||
imageLimits?: ImageAttachmentLimits
|
||||
attachedSessions: number
|
||||
|
||||
@@ -210,7 +210,7 @@ export const imageMediaTypeSchema = z.union([
|
||||
/** Prompt wire content is intentionally narrower than merge-extensible durable core content. */
|
||||
export const promptContentPartSchema = z.discriminatedUnion('type', [
|
||||
z.object({ type: z.literal('text'), text: z.string() }),
|
||||
z.object({ type: z.literal('image'), mediaType: imageMediaTypeSchema, data: z.string(), name: z.string().optional(), alt: z.string().optional() }),
|
||||
z.object({ type: z.literal('image'), mediaType: imageMediaTypeSchema, data: z.string(), name: z.string().optional() }),
|
||||
])
|
||||
|
||||
/** session.prompt request payload. */
|
||||
|
||||
@@ -162,7 +162,7 @@ export interface SessionSummary {
|
||||
/** Browser-submitted prompt content; image bytes are promoted to durable references by the host. */
|
||||
export type PromptContentPart =
|
||||
| { type: 'text'; text: string }
|
||||
| { type: 'image'; mediaType: ImageMediaType; data: string; name?: string; alt?: string }
|
||||
| { type: 'image'; mediaType: ImageMediaType; data: string; name?: string }
|
||||
|
||||
/** Session-domain unary methods (the map keys session.* of RpcMethodMap). */
|
||||
export interface SessionsApi {
|
||||
|
||||
@@ -118,6 +118,26 @@ function expectValue<T>(response: { result: { ok: true; value: T } | { ok: false
|
||||
}
|
||||
|
||||
describe('Web session model selection', () => {
|
||||
it('rejects a second prompt image before attachment persistence', async () => {
|
||||
const { ctx, sessionId } = await harness()
|
||||
const api = createApiProxy(ctx, { provider: 'deepseek', model: 'deepseek-chat', cwd: '/tmp', workspaceRoot: '/tmp' })
|
||||
const image = { type: 'image' as const, mediaType: 'image/png' as const, data: 'AA==' }
|
||||
const response = await api.sessions.prompt(request({
|
||||
sessionId,
|
||||
mode: 'queue' as const,
|
||||
content: [image, image],
|
||||
}))
|
||||
expect(response.result).toEqual({
|
||||
ok: false,
|
||||
error: {
|
||||
code: 'attachment-error',
|
||||
message: 'A prompt may contain at most one image.',
|
||||
details: { reason: 'TOO_MANY_IMAGES' },
|
||||
},
|
||||
})
|
||||
await ctx.fiber.dispose()
|
||||
})
|
||||
|
||||
it('groups successful providers, isolates failures, and preserves an unlisted current model', async () => {
|
||||
const { ctx, sessionId } = await harness({
|
||||
provider: 'deepseek',
|
||||
|
||||
@@ -264,13 +264,6 @@ describe('unary round trip (handler ⇄ client, no network)', () => {
|
||||
hostDescription: {
|
||||
version: 'v',
|
||||
cwd: '/w',
|
||||
activeModel: {
|
||||
provider: 'future',
|
||||
id: 'audio-model',
|
||||
name: 'Audio Model',
|
||||
inputModalities: ['text', 'audio'],
|
||||
outputModalities: ['audio'],
|
||||
},
|
||||
attachedSessions: 0,
|
||||
},
|
||||
}))
|
||||
@@ -281,13 +274,6 @@ describe('unary round trip (handler ⇄ client, no network)', () => {
|
||||
value: {
|
||||
version: 'v',
|
||||
cwd: '/w',
|
||||
activeModel: {
|
||||
provider: 'future',
|
||||
id: 'audio-model',
|
||||
name: 'Audio Model',
|
||||
inputModalities: ['text', 'audio'],
|
||||
outputModalities: ['audio'],
|
||||
},
|
||||
attachedSessions: 0,
|
||||
},
|
||||
})
|
||||
|
||||
@@ -249,25 +249,10 @@ describe('host domain schemas', () => {
|
||||
cwd: '/x',
|
||||
provider: 'p',
|
||||
model: 'm',
|
||||
activeModel: {
|
||||
provider: 'p',
|
||||
id: 'm',
|
||||
name: 'Model',
|
||||
inputModalities: ['text', 'audio'],
|
||||
outputModalities: ['text', 'audio'],
|
||||
},
|
||||
attachedSessions: 2,
|
||||
})
|
||||
expect(value.attachedSessions).toBe(2)
|
||||
expect(value.activeModel?.inputModalities).toEqual(['text', 'audio'])
|
||||
expect(value.activeModel?.outputModalities).toEqual(['text', 'audio'])
|
||||
expect(hostDescribeValueSchema.parse({ version: '1', cwd: '/x', attachedSessions: 0 }).provider).toBeUndefined()
|
||||
expect(() => hostDescribeValueSchema.parse({
|
||||
version: '1',
|
||||
cwd: '/x',
|
||||
activeModel: { provider: 'p', id: 'm', name: 'Model', inputModalities: [{ type: 'audio' }] },
|
||||
attachedSessions: 0,
|
||||
})).toThrow()
|
||||
})
|
||||
|
||||
it('validates the browse listing/creation payloads', () => {
|
||||
|
||||
@@ -23,9 +23,6 @@
|
||||
{
|
||||
"path": "../../attachment/attachment"
|
||||
},
|
||||
{
|
||||
"path": "../../attachment/attachment-local"
|
||||
},
|
||||
{
|
||||
"path": "../../llm/llm"
|
||||
},
|
||||
|
||||
@@ -74,7 +74,6 @@ function modelInfo(provider: string, model: DeepSeekCatalogModel): LlmModelInfo
|
||||
name: model.name ?? model.id,
|
||||
...model.description === undefined ? {} : { description: model.description },
|
||||
inputModalities: ['text'],
|
||||
outputModalities: ['text'],
|
||||
}
|
||||
}
|
||||
|
||||
@@ -171,7 +170,7 @@ export class DeepSeekAdapter extends LlmAdapter {
|
||||
// capability — "unknown" here would let the host accept and persist
|
||||
// images the serializer must then reject.
|
||||
...configured === undefined
|
||||
? { provider, id: model, name: model, inputModalities: ['text' as const], outputModalities: ['text' as const] }
|
||||
? { provider, id: model, name: model, inputModalities: ['text' as const] }
|
||||
: modelInfo(provider, configured),
|
||||
...contextWindow === undefined ? {} : { context: { contextWindow } },
|
||||
...this.options.defaults?.thinking === 'disabled'
|
||||
|
||||
@@ -658,8 +658,8 @@ describe('plugin registration and config', () => {
|
||||
await ctx.plugin(LlmDeepSeek, { apiKey: 'k', baseURL: 'http://127.0.0.1:1' })
|
||||
expect(ctx.llm.listProviders()).toEqual([{ id: 'deepseek', name: 'DeepSeek' }])
|
||||
await expect(ctx.llm.listModels('deepseek')).resolves.toEqual([
|
||||
{ provider: 'deepseek', id: 'deepseek-v4-flash', name: 'DeepSeek-V4-Flash', inputModalities: ['text'], outputModalities: ['text'] },
|
||||
{ provider: 'deepseek', id: 'deepseek-v4-pro', name: 'DeepSeek-V4-Pro', inputModalities: ['text'], outputModalities: ['text'] },
|
||||
{ provider: 'deepseek', id: 'deepseek-v4-flash', name: 'DeepSeek-V4-Flash', inputModalities: ['text'] },
|
||||
{ provider: 'deepseek', id: 'deepseek-v4-pro', name: 'DeepSeek-V4-Pro', inputModalities: ['text'] },
|
||||
])
|
||||
await expect(ctx.llm.resolveModelInfo('deepseek', 'deepseek-v4-flash'))
|
||||
.resolves.toMatchObject({
|
||||
@@ -762,8 +762,8 @@ describe('plugin registration and config', () => {
|
||||
await ctx.plugin(LlmService)
|
||||
LlmDeepSeek.apply(ctx, { apiKey: 'k', baseURL: 'http://127.0.0.1:1' })
|
||||
await expect(ctx.llm.listModels('deepseek')).resolves.toEqual([
|
||||
{ provider: 'deepseek', id: 'deepseek-v4-flash', name: 'DeepSeek-V4-Flash', inputModalities: ['text'], outputModalities: ['text'] },
|
||||
{ provider: 'deepseek', id: 'deepseek-v4-pro', name: 'DeepSeek-V4-Pro', inputModalities: ['text'], outputModalities: ['text'] },
|
||||
{ provider: 'deepseek', id: 'deepseek-v4-flash', name: 'DeepSeek-V4-Flash', inputModalities: ['text'] },
|
||||
{ provider: 'deepseek', id: 'deepseek-v4-pro', name: 'DeepSeek-V4-Pro', inputModalities: ['text'] },
|
||||
])
|
||||
})
|
||||
|
||||
@@ -784,8 +784,8 @@ describe('plugin registration and config', () => {
|
||||
],
|
||||
})
|
||||
await expect(ctx.llm.listModels('deepseek')).resolves.toEqual([
|
||||
{ provider: 'deepseek', id: 'private-fast', name: 'private-fast', inputModalities: ['text'], outputModalities: ['text'] },
|
||||
{ provider: 'deepseek', id: 'private-reasoner', name: 'Private Reasoner', description: 'Higher reasoning budget', inputModalities: ['text'], outputModalities: ['text'] },
|
||||
{ provider: 'deepseek', id: 'private-fast', name: 'private-fast', inputModalities: ['text'] },
|
||||
{ provider: 'deepseek', id: 'private-reasoner', name: 'Private Reasoner', description: 'Higher reasoning budget', inputModalities: ['text'] },
|
||||
])
|
||||
await expect(ctx.llm.resolveModelInfo('deepseek', 'private-fast'))
|
||||
.resolves.toMatchObject({ context: { contextWindow: 32_000 } })
|
||||
|
||||
@@ -130,7 +130,6 @@ export class PiAiAdapter extends LlmAdapter {
|
||||
id: model.id,
|
||||
name: model.name,
|
||||
inputModalities: [...model.input],
|
||||
outputModalities: ['text'],
|
||||
})))
|
||||
}
|
||||
|
||||
@@ -155,7 +154,6 @@ export class PiAiAdapter extends LlmAdapter {
|
||||
id: model,
|
||||
name: resolvedModel.name,
|
||||
inputModalities: [...resolvedModel.input],
|
||||
outputModalities: ['text'],
|
||||
context: { contextWindow: resolvedModel.contextWindow },
|
||||
reasoning: {
|
||||
efforts: levels.map(level => ({
|
||||
|
||||
@@ -250,16 +250,10 @@ describe('PiAiAdapter provider routing', () => {
|
||||
class LateAttachmentStore extends AttachmentStore {
|
||||
readonly imageLimits: ImageAttachmentLimits = {
|
||||
maxImageBytes: 1,
|
||||
maxImagesPerMessage: 1,
|
||||
maxMessageImageBytes: 1,
|
||||
maxImagePixels: 1,
|
||||
mediaTypes: ['image/png'],
|
||||
}
|
||||
|
||||
validateImage(_input: SaveImageAttachment): void {
|
||||
throw new Error('not used')
|
||||
}
|
||||
|
||||
saveImage(_input: SaveImageAttachment): Promise<ImageAttachmentRef> {
|
||||
return Promise.reject(new Error('not used'))
|
||||
}
|
||||
@@ -440,7 +434,7 @@ describe('provider profile lifecycle', () => {
|
||||
const models = await ctx.llm.listModels('openai')
|
||||
expect(models.find(model => model.id === 'gpt-4.1')).toEqual({
|
||||
provider: 'openai', id: 'gpt-4.1', name: 'GPT-4.1',
|
||||
inputModalities: ['text', 'image'], outputModalities: ['text'],
|
||||
inputModalities: ['text', 'image'],
|
||||
})
|
||||
expect(models.every(model => model.provider === 'openai')).toBe(true)
|
||||
const info = await ctx.llm.resolveModelInfo('openai', 'gpt-4.1')
|
||||
|
||||
@@ -68,16 +68,10 @@ async function harness(image?: StoredImageAttachment): Promise<Context> {
|
||||
class E2eAttachmentStore extends AttachmentStore {
|
||||
readonly imageLimits: ImageAttachmentLimits = {
|
||||
maxImageBytes: fixture.data.byteLength,
|
||||
maxImagesPerMessage: 1,
|
||||
maxMessageImageBytes: fixture.data.byteLength,
|
||||
maxImagePixels: fixture.ref.width * fixture.ref.height,
|
||||
mediaTypes: [fixture.ref.mediaType],
|
||||
}
|
||||
|
||||
validateImage(_input: SaveImageAttachment): void {
|
||||
throw new Error('e2e attachment fixture is read-only')
|
||||
}
|
||||
|
||||
saveImage(_input: SaveImageAttachment): Promise<ImageAttachmentRef> {
|
||||
return Promise.reject(new Error('e2e attachment fixture is read-only'))
|
||||
}
|
||||
@@ -228,7 +222,7 @@ for (const profile of providerCases) {
|
||||
type: 'text',
|
||||
text: 'What type of machine-readable symbol is shown in the attached image? Reply with exactly: QR code',
|
||||
},
|
||||
{ type: 'image', attachment: ref, alt: 'machine-readable symbol' },
|
||||
{ type: 'image', attachment: ref },
|
||||
],
|
||||
source: { kind: 'plugin', plugin: 'test' },
|
||||
})],
|
||||
|
||||
@@ -2,5 +2,5 @@
|
||||
# side as of the last confirmed-consistent state. Both languages carry equal authority;
|
||||
# after editing either side, bring the other along and re-record with:
|
||||
# pnpm run verify-translation-pairing --write packages/llm/llm/README.md
|
||||
README.md: ecd6da304a894a687153608f5b468f867ad32e6b
|
||||
README.zh.md: c43da45bf0c573a78c194d731bff1d9765b1eed6
|
||||
README.md: f6bf6ed88cc3aa21c57b5a08249848bdfff2f0c1
|
||||
README.zh.md: e466568de5a57b36dcc9fb0bd876ec2216a6c5f3
|
||||
|
||||
@@ -23,7 +23,7 @@ An adapter registry plus a single streaming call surface, interceptable via a wa
|
||||
|
||||
Provider and model metadata is a discovery surface, not a routing whitelist. `registerAdapter()` still owns provider exclusivity and captures the adapter's retry policy for each route, while an adapter may accept model ids absent from `listModels()`; consumers must not reject a request because its model is unlisted. Returned selector metadata is detached and invalid or duplicate adapter entries fail with `INVALID_ADAPTER` or `INVALID_CATALOG`.
|
||||
|
||||
Exact-model metadata is a separate correctness query, not a catalog decoration or global LLM setting. `resolveModelInfo()` asks the adapter that owns the exact provider/model route once; an adapter can describe an unlisted dynamic model, and absent `context` or `reasoning` fields mean only that those capabilities are unavailable. Invalid identity or modality metadata fails with `INVALID_MODEL_INFO`, and invalid context or reasoning metadata with `INVALID_MODEL_CONTEXT` or `INVALID_MODEL_REASONING`.
|
||||
Exact-model metadata is a separate correctness query, not a catalog decoration or global LLM setting. `resolveModelInfo()` asks the adapter that owns the exact provider/model route once; an adapter can describe an unlisted dynamic model, and absent `context` or `reasoning` fields mean only that those capabilities are unavailable. Invalid identity metadata fails with `INVALID_MODEL_INFO`, and invalid context or reasoning metadata with `INVALID_MODEL_CONTEXT` or `INVALID_MODEL_REASONING`.
|
||||
|
||||
Reasoning identifiers are opaque adapter-owned strings rather than a core enum. An adapter publishes its ordered selectable list, including an `off` id when that model's capability API exposes one. `resolveCallConfig()` accepts only an exact advertised identifier, materializes `defaultEffort` when present, and otherwise preserves the provider default. Asynchronous model resolvers receive the caller's signal and must settle promptly after cancellation. `prepareCall()` additionally retains the exact adapter registration through header logging and terminal dispatch, so HMR cannot combine one adapter's capability result with another adapter's request; reusing its one-shot handle or changing its call-config fields fails with `INVALID_PREPARED_CALL`. An unsupported explicit or configured effort fails with `UNSUPPORTED_REASONING_EFFORT` before provider I/O.
|
||||
|
||||
|
||||
@@ -23,7 +23,7 @@
|
||||
|
||||
提供方与模型元数据是发现接口,不是路由白名单。`registerAdapter()` 仍拥有提供方排他性,并为每条路由捕获适配器的重试策略;适配器则可以接受 `listModels()` 中不存在的模型 id,消费方禁止因模型未列出而拒绝请求。返回的 selector 元数据与输入脱离,无效或重复适配器配置项会以 `INVALID_ADAPTER` 或 `INVALID_CATALOG` 失败。
|
||||
|
||||
确切模型元数据是独立的正确性查询,不是 catalog 装饰或全局 LLM 设置。`resolveModelInfo()` 会向拥有精确提供方/模型路由的适配器查询一次;适配器可以描述未列出的动态模型,缺少 `context` 或 `reasoning` 字段只表示相应能力不可用。无效的身份或模态元数据会以 `INVALID_MODEL_INFO` 失败,无效的上下文或推理元数据则以 `INVALID_MODEL_CONTEXT` 或 `INVALID_MODEL_REASONING` 失败。
|
||||
确切模型元数据是独立的正确性查询,不是 catalog 装饰或全局 LLM 设置。`resolveModelInfo()` 会向拥有精确提供方/模型路由的适配器查询一次;适配器可以描述未列出的动态模型,缺少 `context` 或 `reasoning` 字段只表示相应能力不可用。无效的身份元数据会以 `INVALID_MODEL_INFO` 失败,无效的上下文或推理元数据则以 `INVALID_MODEL_CONTEXT` 或 `INVALID_MODEL_REASONING` 失败。
|
||||
|
||||
推理标识符是由适配器持有的不透明字符串,而非核心枚举。适配器会公布有序可选列表;模型能力 API 提供 `off` id 时,列表也会包含它。`resolveCallConfig()` 只接受与已公布标识符完全一致的值,在存在 `defaultEffort` 时填入它,否则保留提供方默认值。异步模型解析器会接收调用方的 signal,并且必须在取消后迅速结束。`prepareCall()` 还会让精确适配器注册跨越请求头记录和最终分派,因此 HMR(热模块替换)不会将一个适配器的能力结果与另一个适配器的请求混用;复用其一次性句柄或更改调用配置字段会以 `INVALID_PREPARED_CALL` 失败。不支持的显式或配置推理强度会在提供方 I/O 前以 `UNSUPPORTED_REASONING_EFFORT` 失败。
|
||||
|
||||
|
||||
@@ -254,26 +254,9 @@ export class LlmService extends Service {
|
||||
return this.registration(provider).retryPolicy
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate adapter-owned modality arrays and detach them. One rule for the
|
||||
* advisory catalog and exact resolution: both validate, both copy — two
|
||||
* readings of the same adapter field with different trust or detachment
|
||||
* would be an unexplained asymmetry.
|
||||
* @param provider - provider route (diagnostic context).
|
||||
* @param code - error code matching the calling surface.
|
||||
* @param modalities - adapter-owned array, or undefined for unknown.
|
||||
* @returns a detached copy, or undefined when absent.
|
||||
*/
|
||||
private detachedModalities(
|
||||
provider: string,
|
||||
code: 'INVALID_CATALOG' | 'INVALID_MODEL_INFO',
|
||||
modalities: readonly unknown[] | undefined,
|
||||
): ModelModality[] | undefined {
|
||||
if (modalities === undefined) return undefined
|
||||
if (!Array.isArray(modalities) || modalities.some(entry => typeof entry !== 'string')) {
|
||||
throw new LlmError(`adapter returned invalid modality metadata for provider "${provider}"`, code)
|
||||
}
|
||||
return [...(modalities as readonly ModelModality[])]
|
||||
/** Detach typed adapter-owned modality metadata. */
|
||||
private detachedModalities(modalities: readonly ModelModality[] | undefined): ModelModality[] | undefined {
|
||||
return modalities === undefined ? undefined : [...modalities]
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -300,15 +283,13 @@ export class LlmService extends Service {
|
||||
throw new LlmError(`adapter returned invalid or duplicate model metadata for provider "${provider}"`, 'INVALID_CATALOG')
|
||||
}
|
||||
seen.add(model.id)
|
||||
const inputModalities = this.detachedModalities(provider, 'INVALID_CATALOG', model.inputModalities)
|
||||
const outputModalities = this.detachedModalities(provider, 'INVALID_CATALOG', model.outputModalities)
|
||||
const inputModalities = this.detachedModalities(model.inputModalities)
|
||||
return {
|
||||
provider: model.provider,
|
||||
id: model.id,
|
||||
name: model.name,
|
||||
...model.description === undefined ? {} : { description: model.description },
|
||||
...inputModalities === undefined ? {} : { inputModalities },
|
||||
...outputModalities === undefined ? {} : { outputModalities },
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -360,15 +341,13 @@ export class LlmService extends Service {
|
||||
}
|
||||
// Capability metadata rides through: an explicit modality omission is
|
||||
// negative capability downstream preflights act on (image admission).
|
||||
const inputModalities = this.detachedModalities(provider, 'INVALID_MODEL_INFO', resolved.inputModalities)
|
||||
const outputModalities = this.detachedModalities(provider, 'INVALID_MODEL_INFO', resolved.outputModalities)
|
||||
const inputModalities = this.detachedModalities(resolved.inputModalities)
|
||||
const info: LlmResolvedModelInfo = {
|
||||
provider,
|
||||
id: model,
|
||||
name: resolved.name,
|
||||
...resolved.description === undefined ? {} : { description: resolved.description },
|
||||
...inputModalities === undefined ? {} : { inputModalities },
|
||||
...outputModalities === undefined ? {} : { outputModalities },
|
||||
...context === undefined ? {} : { context: { contextWindow: context.contextWindow } },
|
||||
}
|
||||
const reasoning = resolved.reasoning
|
||||
|
||||
@@ -57,8 +57,6 @@ export interface ImageBlock {
|
||||
type: 'image'
|
||||
/** Immutable bytes and intrinsic display metadata owned by the attachment service. */
|
||||
attachment: ImageAttachmentRef
|
||||
/** Optional provider- and UI-facing alternative text, carried from the prompt wire's image part. */
|
||||
alt?: string
|
||||
}
|
||||
|
||||
/** A tool invocation requested by the model. */
|
||||
@@ -156,8 +154,6 @@ export interface LlmModelInfo {
|
||||
description?: string
|
||||
/** Accepted request modalities; absent means unknown, while an explicit omission is negative capability. */
|
||||
inputModalities?: readonly ModelModality[]
|
||||
/** Structured response modalities; absent means unknown, while an explicit omission is negative capability. */
|
||||
outputModalities?: readonly ModelModality[]
|
||||
}
|
||||
|
||||
/** Provider-owned context capacity for one exact provider/model route. */
|
||||
|
||||
@@ -857,8 +857,6 @@ describe('LlmService', () => {
|
||||
[{ provider: 'route', id: 'model', name: 1 }, 'non-string name'],
|
||||
[{ provider: 'route', id: 'model', name: '' }, 'empty name'],
|
||||
[{ provider: 'route', id: 'model', name: 'Model', description: 1 }, 'non-string description'],
|
||||
[{ provider: 'route', id: 'model', name: 'Model', inputModalities: 'text' }, 'non-array input modalities'],
|
||||
[{ provider: 'route', id: 'model', name: 'Model', outputModalities: [1] }, 'non-string output modality'],
|
||||
] as const)('rejects invalid exact model metadata (%s: %s)', async (metadata, _label) => {
|
||||
const ctx = new Context()
|
||||
await ctx.plugin(LlmService)
|
||||
@@ -880,7 +878,7 @@ describe('LlmService', () => {
|
||||
override resolveModel(): Promise<LlmResolvedModelInfo> {
|
||||
return Promise.resolve({
|
||||
provider: 'route', id: 'model', name: 'Model',
|
||||
inputModalities: ['text', 'image'], outputModalities: ['text'],
|
||||
inputModalities: ['text', 'image'],
|
||||
})
|
||||
}
|
||||
}(SCRIPT)
|
||||
@@ -890,7 +888,7 @@ describe('LlmService', () => {
|
||||
// rebuild that drops it silently reads as "modalities unknown".
|
||||
await expect(ctx.llm.resolveModelInfo('route', 'model')).resolves.toEqual({
|
||||
provider: 'route', id: 'model', name: 'Model',
|
||||
inputModalities: ['text', 'image'], outputModalities: ['text'],
|
||||
inputModalities: ['text', 'image'],
|
||||
})
|
||||
})
|
||||
|
||||
@@ -1177,8 +1175,6 @@ describe('LlmService', () => {
|
||||
[{ provider: 'route', id: 'm', name: 1 }, 'non-string name'],
|
||||
[{ provider: 'route', id: 'm', name: '' }, 'empty name'],
|
||||
[{ provider: 'route', id: 'm', name: 'M', description: 1 }, 'non-string description'],
|
||||
[{ provider: 'route', id: 'm', name: 'M', inputModalities: 'text' }, 'non-array input modalities'],
|
||||
[{ provider: 'route', id: 'm', name: 'M', outputModalities: [1] }, 'non-string output modality'],
|
||||
] as const)('rejects invalid model metadata (%s: %s)', async (metadata, _label) => {
|
||||
const ctx = new Context()
|
||||
await ctx.plugin(LlmService)
|
||||
|
||||
@@ -25,11 +25,6 @@ const CHARS_PER_TOKEN = 4
|
||||
/** Per-block structural overhead for JSON framing and type tags. */
|
||||
const BLOCK_OVERHEAD = 4
|
||||
|
||||
/** Provider-neutral visual estimate: base cost plus one cost unit per 512px tile. */
|
||||
const IMAGE_BASE_TOKENS = 85
|
||||
const IMAGE_TILE_TOKENS = 170
|
||||
const IMAGE_TILE_EDGE = 512
|
||||
|
||||
/** Role-field framing overhead added to every priced message. */
|
||||
const ROLE_OVERHEAD = 4
|
||||
|
||||
@@ -362,12 +357,6 @@ export class TokenMeterService extends Service {
|
||||
case 'reasoning':
|
||||
tokens += Math.ceil(block.text.length / CHARS_PER_TOKEN) + BLOCK_OVERHEAD
|
||||
break
|
||||
case 'image': {
|
||||
const tiles = Math.ceil(block.attachment.width / IMAGE_TILE_EDGE)
|
||||
* Math.ceil(block.attachment.height / IMAGE_TILE_EDGE)
|
||||
tokens += IMAGE_BASE_TOKENS + tiles * IMAGE_TILE_TOKENS + BLOCK_OVERHEAD
|
||||
break
|
||||
}
|
||||
case 'tool-call':
|
||||
tokens += Math.ceil(block.name.length / CHARS_PER_TOKEN)
|
||||
+ Math.ceil(block.arguments.length / CHARS_PER_TOKEN)
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
import { describe, expect, expectTypeOf, it } from 'vitest'
|
||||
import { Context } from 'cordis'
|
||||
import { AttachmentId } from '@deepseek-ai/dsh-attachment'
|
||||
import { createUserMessage, CallId, createMessage } from '@deepseek-ai/dsh-llm'
|
||||
import type { ContentBlock, Message, TokenUsage } from '@deepseek-ai/dsh-llm'
|
||||
import SessionStore, { Session, SessionId, canonicalHeader } from '@deepseek-ai/dsh-session'
|
||||
@@ -129,16 +128,6 @@ describe('TokenMeterService pricing', () => {
|
||||
const blocks: ContentBlock[] = [
|
||||
{ type: 'text', text: 'abcd' },
|
||||
{ type: 'reasoning', text: 'ab' },
|
||||
{
|
||||
type: 'image',
|
||||
attachment: {
|
||||
attachmentId: AttachmentId(`sha256:${'a'.repeat(64)}`),
|
||||
mediaType: 'image/png',
|
||||
bytes: 1,
|
||||
width: 1024,
|
||||
height: 513,
|
||||
},
|
||||
},
|
||||
{ type: 'tool-call', id: CallId('c'), name: 'read', arguments: '{"x":1}' },
|
||||
{
|
||||
type: 'tool-result',
|
||||
@@ -152,7 +141,7 @@ describe('TokenMeterService pricing', () => {
|
||||
role: 'assistant', content: blocks,
|
||||
source: { kind: 'plugin', plugin: 'test' },
|
||||
}))
|
||||
expect(estimated).toBe(813)
|
||||
expect(estimated).toBeGreaterThan(30)
|
||||
expect(service.estimateMessage(textMessage('abcd'))).toBe(9)
|
||||
})
|
||||
|
||||
|
||||
Reference in New Issue
Block a user