Coder/deepseek-harness
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: narrow web image input v1

Browse Source
This commit is contained in:
Tianyi Cui
2026-07-29 23:18:57 +08:00
parent 8348a409aa
commit 3a4c436d83
69 changed files with 280 additions and 613 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
packages/host/apiproxy/package.json
View File

@@ -41,7 +41,6 @@
"license": "BSD-3-Clause",
"dependencies": {
"@deepseek-ai/dsh-attachment": "workspace:^",
"@deepseek-ai/dsh-attachment-local": "workspace:^",
"@deepseek-ai/dsh-agent": "workspace:^",
"@deepseek-ai/dsh-brand": "workspace:^",
"@deepseek-ai/dsh-commands": "workspace:^",

59
packages/host/apiproxy/src/api-proxy.ts
View File

@@ -11,8 +11,8 @@ import { installAgentLlmTarget } from '@deepseek-ai/dsh-agent'
import type {
Agent, AgentLlmTarget, AgentLlmTargetRef, AgentStatus, InboxPlacement,
} from '@deepseek-ai/dsh-agent'
import { AttachmentError } from '@deepseek-ai/dsh-attachment-local'
import type { ImageAttachmentRef } from '@deepseek-ai/dsh-attachment-local'
import { AttachmentError } from '@deepseek-ai/dsh-attachment'
import type { ImageAttachmentRef } from '@deepseek-ai/dsh-attachment'
import { createUserMessage, ReasoningEffortId } from '@deepseek-ai/dsh-llm'
import { errorChain } from '@deepseek-ai/dsh-llm'
import type { ContentBlock, MessageId, MessageSource } from '@deepseek-ai/dsh-llm'
@@ -79,37 +79,23 @@ async function durablePromptContent(ctx: Context, content: readonly PromptConten
if (content.every(part => part.type === 'text')) {
return content.map(part => ({ type: 'text', text: part.text }))
}
const limits = ctx.attachments.imageLimits
const prepared = content.map(part => part.type === 'text'
? part
: { part, data: decodeBase64(part.data) })
const images = prepared.filter((part): part is Extract<typeof part, { data: Uint8Array }> => 'data' in part)
if (images.length > limits.maxImagesPerMessage) {
throw new AttachmentError('Prompt exceeds the configured image-count limit.', 'TOO_MANY_IMAGES')
if (content.filter(part => part.type === 'image').length > 1) {
throw new AttachmentError('A prompt may contain at most one image.', 'TOO_MANY_IMAGES')
}
const totalBytes = images.reduce((sum, image) => sum + image.data.byteLength, 0)
if (totalBytes > limits.maxMessageImageBytes) {
throw new AttachmentError('Prompt exceeds the configured aggregate image-byte limit.', 'IMAGES_TOO_LARGE')
}
// Validate the complete batch before persisting any member: the store has no
// garbage collection, so one malformed image must not leave the batch's
// valid members as published objects no message event will ever reference.
for (const image of images) {
ctx.attachments.validateImage({
data: image.data,
mediaType: image.part.mediaType,
...image.part.name === undefined ? {} : { name: image.part.name },
})
}
return Promise.all(prepared.map(async (item): Promise<ContentBlock> => {
if (!('data' in item)) return { type: 'text', text: item.text }
const durable: ContentBlock[] = []
for (const part of content) {
if (part.type === 'text') {
durable.push({ type: 'text', text: part.text })
continue
}
const attachment = await ctx.attachments.saveImage({
data: item.data,
mediaType: item.part.mediaType,
...item.part.name === undefined ? {} : { name: item.part.name },
data: decodeBase64(part.data),
mediaType: part.mediaType,
...part.name === undefined ? {} : { name: part.name },
})
return { type: 'image', attachment, ...item.part.alt === undefined ? {} : { alt: item.part.alt } }
}))
durable.push({ type: 'image', attachment })
}
return durable
}
/**
@@ -1222,8 +1208,8 @@ export function createApiProxy(ctx: Context, defaults: ApiProxyDefaults): ApiPro
const target = targetFor(agent).current
const provider = target.provider
const model = target.model
const activeModel = await ctx.llm.resolveModelInfo(provider, model)
if (activeModel.inputModalities !== undefined && !activeModel.inputModalities.includes('image')) {
const modelInfo = await ctx.llm.resolveModelInfo(provider, model)
if (modelInfo.inputModalities !== undefined && !modelInfo.inputModalities.includes('image')) {
return err(request, {
code: 'attachment-error',
message: `Model "${model}" does not support image input.`,
@@ -1419,24 +1405,21 @@ export function createApiProxy(ctx: Context, defaults: ApiProxyDefaults): ApiPro
},
host: {
async describe(request) {
const activeModel = (await ctx.llm.listModels(defaults.provider))
.find(model => model.id === defaults.model)
describe(request) {
// TODO(step2): version should read apps/cli's package.json; placeholder for now.
return ok(request, {
return Promise.resolve(ok(request, {
version: '0.0.1',
// Same source as session.create's fallback: the UI's default project
// must match where an unspecified-cwd session actually lands.
cwd: defaults.cwd,
provider: defaults.provider,
model: defaults.model,
...activeModel === undefined ? {} : { activeModel },
imageLimits: {
...ctx.attachments.imageLimits,
mediaTypes: [...ctx.attachments.imageLimits.mediaTypes],
},
attachedSessions: ctx.agents.list().length,
})
}))
},
async pickDirectory(request, signal) {

14
packages/host/apiproxy/src/api/host.schema.ts
View File

@@ -3,15 +3,11 @@
*/
import { z } from 'zod'
import type { ModelModality } from '@deepseek-ai/dsh-llm'
import type { DirectoryEntry } from './host.ts'
import type { RequestPayload, ResponseValue } from './rpc-map.ts'
import type { Wire } from './rpc.schema.ts'
import { imageMediaTypeSchema } from './sessions.schema.ts'
/** Merge-extensible modality passthrough: declaration merging cannot extend a runtime Zod union. */
const modalitySchema = z.string() as unknown as z.ZodType<ModelModality>
/** host.describe request payload (empty object literal). */
export const hostDescribeRequestSchema = z.object({}) satisfies z.ZodType<Wire<RequestPayload<'host.describe'>>>
@@ -21,18 +17,8 @@ export const hostDescribeValueSchema = z.object({
cwd: z.string(),
provider: z.string().optional(),
model: z.string().optional(),
activeModel: z.object({
provider: z.string(),
id: z.string(),
name: z.string(),
description: z.string().optional(),
inputModalities: z.array(modalitySchema).optional(),
outputModalities: z.array(modalitySchema).optional(),
}).optional(),
imageLimits: z.object({
maxImageBytes: z.number().int().positive(),
maxImagesPerMessage: z.number().int().positive(),
maxMessageImageBytes: z.number().int().positive(),
maxImagePixels: z.number().int().positive(),
mediaTypes: z.array(imageMediaTypeSchema),
}).optional(),

3
packages/host/apiproxy/src/api/host.ts
View File

@@ -5,7 +5,6 @@
import type { RpcRequest, RpcResponse } from './rpc.ts'
import type { ImageAttachmentLimits } from '@deepseek-ai/dsh-attachment'
import type { LlmModelInfo } from '@deepseek-ai/dsh-llm/types'
/** One directory row of a listing: a child entry or a breadcrumb ancestor. */
export interface DirectoryEntry {
@@ -49,8 +48,6 @@ export interface HostApi {
cwd: string
provider?: string
model?: string
/** Catalog entry for the active route; absent means its capabilities are unknown. */
activeModel?: LlmModelInfo
/** Resolved authoritative image-upload limits. */
imageLimits?: ImageAttachmentLimits
attachedSessions: number

2
packages/host/apiproxy/src/api/sessions.schema.ts
View File

@@ -210,7 +210,7 @@ export const imageMediaTypeSchema = z.union([
/** Prompt wire content is intentionally narrower than merge-extensible durable core content. */
export const promptContentPartSchema = z.discriminatedUnion('type', [
z.object({ type: z.literal('text'), text: z.string() }),
z.object({ type: z.literal('image'), mediaType: imageMediaTypeSchema, data: z.string(), name: z.string().optional(), alt: z.string().optional() }),
z.object({ type: z.literal('image'), mediaType: imageMediaTypeSchema, data: z.string(), name: z.string().optional() }),
])
/** session.prompt request payload. */

2
packages/host/apiproxy/src/api/sessions.ts
View File

@@ -162,7 +162,7 @@ export interface SessionSummary {
/** Browser-submitted prompt content; image bytes are promoted to durable references by the host. */
export type PromptContentPart =
| { type: 'text'; text: string }
| { type: 'image'; mediaType: ImageMediaType; data: string; name?: string; alt?: string }
| { type: 'image'; mediaType: ImageMediaType; data: string; name?: string }
/** Session-domain unary methods (the map keys session.* of RpcMethodMap). */
export interface SessionsApi {

20
packages/host/apiproxy/tests/api-proxy-models.spec.ts
View File

@@ -118,6 +118,26 @@ function expectValue<T>(response: { result: { ok: true; value: T } | { ok: false
}
describe('Web session model selection', () => {
it('rejects a second prompt image before attachment persistence', async () => {
const { ctx, sessionId } = await harness()
const api = createApiProxy(ctx, { provider: 'deepseek', model: 'deepseek-chat', cwd: '/tmp', workspaceRoot: '/tmp' })
const image = { type: 'image' as const, mediaType: 'image/png' as const, data: 'AA==' }
const response = await api.sessions.prompt(request({
sessionId,
mode: 'queue' as const,
content: [image, image],
}))
expect(response.result).toEqual({
ok: false,
error: {
code: 'attachment-error',
message: 'A prompt may contain at most one image.',
details: { reason: 'TOO_MANY_IMAGES' },
},
})
await ctx.fiber.dispose()
})
it('groups successful providers, isolates failures, and preserves an unlisted current model', async () => {
const { ctx, sessionId } = await harness({
provider: 'deepseek',

14
packages/host/apiproxy/tests/fetch-carrier.spec.ts
View File

@@ -264,13 +264,6 @@ describe('unary round trip (handler ⇄ client, no network)', () => {
hostDescription: {
version: 'v',
cwd: '/w',
activeModel: {
provider: 'future',
id: 'audio-model',
name: 'Audio Model',
inputModalities: ['text', 'audio'],
outputModalities: ['audio'],
},
attachedSessions: 0,
},
}))
@@ -281,13 +274,6 @@ describe('unary round trip (handler ⇄ client, no network)', () => {
value: {
version: 'v',
cwd: '/w',
activeModel: {
provider: 'future',
id: 'audio-model',
name: 'Audio Model',
inputModalities: ['text', 'audio'],
outputModalities: ['audio'],
},
attachedSessions: 0,
},
})

15
packages/host/apiproxy/tests/rpc-schemas.spec.ts
View File

@@ -249,25 +249,10 @@ describe('host domain schemas', () => {
cwd: '/x',
provider: 'p',
model: 'm',
activeModel: {
provider: 'p',
id: 'm',
name: 'Model',
inputModalities: ['text', 'audio'],
outputModalities: ['text', 'audio'],
},
attachedSessions: 2,
})
expect(value.attachedSessions).toBe(2)
expect(value.activeModel?.inputModalities).toEqual(['text', 'audio'])
expect(value.activeModel?.outputModalities).toEqual(['text', 'audio'])
expect(hostDescribeValueSchema.parse({ version: '1', cwd: '/x', attachedSessions: 0 }).provider).toBeUndefined()
expect(() => hostDescribeValueSchema.parse({
version: '1',
cwd: '/x',
activeModel: { provider: 'p', id: 'm', name: 'Model', inputModalities: [{ type: 'audio' }] },
attachedSessions: 0,
})).toThrow()
})
it('validates the browse listing/creation payloads', () => {

3
packages/host/apiproxy/tsconfig.json
View File

@@ -23,9 +23,6 @@
{
"path": "../../attachment/attachment"
},
{
"path": "../../attachment/attachment-local"
},
{
"path": "../../llm/llm"
},