Coder/deepseek-harness
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add web multimodal image attachments

Browse Source
This commit is contained in:
Yichen Jiang
2026-07-23 15:20:47 +08:00
parent 3e3ea47296
commit cb4c11b869
116 changed files with 3177 additions and 151 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
packages/client/connection/package.json
View File

@@ -29,6 +29,7 @@
},
"license": "BSD-3-Clause",
"dependencies": {
"@deepseek-ai/dsh-attachment": "workspace:^",
"@deepseek-ai/dsh-host-apiproxy": "workspace:^",
"@deepseek-ai/dsh-llm": "workspace:^",
"@deepseek-ai/dsh-session": "workspace:^",

2
packages/client/connection/src/client/api.ts
View File

@@ -6,7 +6,7 @@
// The ./api and ./client subpath exports are the browser-safe channels added for this.
export type {
ApiProxy, SessionsApi, SessionSummary, HostApi, EventsApi, MuxFrame, HostFrame,
ApiProxy, SessionsApi, SessionSummary, PromptContentPart, HostApi, EventsApi, MuxFrame, HostFrame,
ApprovalResponsePayload, QuestionResponsePayload, HistoryEntry, ToolEventView,
} from '@deepseek-ai/dsh-host-apiproxy/api'
export type { ToolCallView, ToolResultView } from '@deepseek-ai/dsh-tools/presentation'

82
packages/client/connection/src/client/fixture.ts
View File

@@ -6,6 +6,7 @@
// approval (placeholder-card material, subscribed-baseline-replay semantics: stable rpcId reuse).
import type { ContentBlock } from '@deepseek-ai/dsh-llm/types'
import type { AttachmentIdType, ImageAttachmentRef } from '@deepseek-ai/dsh-attachment'
import type { SessionEvent, SessionId } from '@deepseek-ai/dsh-session/types'
import type {
ApiProxy, ClientRequest, ClientResponse, HistoryEntry, HostFrame, MuxFrame, RpcReceipt,
@@ -28,6 +29,16 @@ function sid(id: string): SessionId {
return id as SessionId
}
const FIXTURE_IMAGE_DATA = 'iVBORw0KGgoAAAANSUhEUgAAAKAAAABaCAYAAAA/xl1SAAAAvklEQVR42u3SMQ0AAAjAMIyhELM4AAe8PD1qYFlk9cCXEAEDYkAwIAYEA2JAMCAGBANiQDAgBgQDYkAwIAYEA2JAMCAGBANiQDAgBgQDYkAwIAYEA2JAMCAGxIBCYEAMCAbEgGBADAgGxIBgQAwIBsSAYEAMCAbEgGBADAgGxIBgQAwIBsSAYEAMCAbEgGBADAgGxIAYEAyIAcGAGBAMiAHBgBgQDIgBwYAYEAyIAcGAGBAMiAHBgBgQDIgB4bYWLb6pnOb1xAAAAABJRU5ErkJggg=='
const FIXTURE_IMAGE_REF: ImageAttachmentRef = {
attachmentId: 'fixture:image' as AttachmentIdType,
mediaType: 'image/png',
bytes: 68,
width: 160,
height: 90,
name: 'fixture-image.png',
}
/** fx-alpha history script: 60 turns (~130+ messages -> 3 pages at PAGE_MESSAGES=50),
* mixing reasoning blocks / tool call+result / steering / context. */
function buildAlphaLog(): SessionEvent[] {
@@ -88,6 +99,12 @@ function buildAlphaLog(): SessionEvent[] {
toolTurn(60, 'fx-bash', '{"command":"ls -la","cwd":"/tmp/fixture"}', 'total 2\ndrwxr-xr-x fixture\n-rw-r--r-- demo.txt')
toolTurn(61, 'fx-write', '{"path":"notes/demo.txt","content":"hello fixture\\n"}', 'wrote notes/demo.txt')
toolTurn(62, 'fx-note', '{"note":"三型卡验收样本"}', '已记录')
push({ type: 'turn/start', data: { turn: 63, trigger: { kind: 'message', source: { kind: 'user' } } } })
push({ type: 'user/message', surfaceOp: 'append', data: { content: [{ type: 'image', attachment: FIXTURE_IMAGE_REF }, ...text('历史用户图片')], source: { kind: 'user' } } })
push({ type: 'step/start', data: { turn: 63, step: 0 } })
push({ type: 'assistant/message', surfaceOp: 'append', data: { turn: 63, step: 0, content: [...text('结构化模型图片：'), { type: 'image', attachment: FIXTURE_IMAGE_REF }], provenance: { provider: 'fixture', model: 'fx-vision' } } })
push({ type: 'step/end', data: { turn: 63, step: 0 } })
push({ type: 'turn/end', data: { turn: 63, reason: { kind: 'completed' } } })
return events as unknown as SessionEvent[]
}
@@ -186,6 +203,18 @@ function pageOf(
return { events, hasMore: start > 0 }
}
/** Fixture mirror of host session-scoped attachment authorization. */
function logReferencesAttachment(log: readonly SessionEvent[], attachmentId: string): boolean {
const visit = (value: unknown): boolean => {
if (Array.isArray(value)) return value.some(visit)
if (typeof value !== 'object' || value === null) return false
const record = value as Record<string, unknown>
if (record.attachmentId === attachmentId) return true
return Object.values(record).some(visit)
}
return log.some(event => visit(event.data))
}
interface StreamConn<F> {
push(envelope: RpcRequest<F>): void
}
@@ -243,7 +272,11 @@ export function createFixtureApi(): ApiProxy {
{ sessionId: sid('fx-gamma'), updatedAt: Date.now() - 120_000, running: false, cwd: '/tmp/fixture' },
]
const logs = new Map<SessionId, SessionEvent[]>([[sid('fx-alpha'), buildAlphaLog()]])
const nextTurn = new Map<SessionId, number>([[sid('fx-alpha'), 60]])
const nextTurn = new Map<SessionId, number>([[sid('fx-alpha'), 64]])
const attachments = new Map<string, { attachment: ImageAttachmentRef; data: string }>([[
String(FIXTURE_IMAGE_REF.attachmentId),
{ attachment: FIXTURE_IMAGE_REF, data: FIXTURE_IMAGE_DATA },
]])
let nextSession = 1
let nextRpc = 1
const mint = (): ReturnType<typeof RpcId> => RpcId(`fx-rpc-${nextRpc++}`)
@@ -394,21 +427,44 @@ export function createFixtureApi(): ApiProxy {
}
summary.updatedAt = Date.now()
const userText = content.map(b => (b.type === 'text' ? b.text : '')).join('')
const durable: ContentBlock[] = content.map((block) => {
if (block.type === 'text') return block
const attachment: ImageAttachmentRef = {
attachmentId: `fixture:${crypto.randomUUID()}` as AttachmentIdType,
mediaType: block.mediaType,
bytes: Math.max(1, Math.floor(block.data.length * 3 / 4) - (block.data.endsWith('==') ? 2 : block.data.endsWith('=') ? 1 : 0)),
width: 160,
height: 90,
...block.name === undefined ? {} : { name: block.name },
}
attachments.set(String(attachment.attachmentId), { attachment, data: block.data })
return { type: 'image', attachment }
})
if (mode === 'steer' && replays.has(id)) {
// Steering: insert a steering message into the current turn; the replay continues.
/* v8 ignore next -- the ?? arm needs a missing counter, but a live replay implies a prior prompt already set it. */
const turn = (nextTurn.get(id) ?? 1) - 1
append(id, { type: 'steering/message', surfaceOp: 'append', data: { turn, content, source: { kind: 'user' } } })
append(id, { type: 'steering/message', surfaceOp: 'append', data: { turn, content: durable, source: { kind: 'user' } } })
return ok(request, { accepted: true as const })
}
const turn = nextTurn.get(id) ?? 0
nextTurn.set(id, turn + 1)
setRunning(id, true)
append(id, { type: 'turn/start', data: { turn, trigger: { kind: 'message', source: { kind: 'user' } } } })
append(id, { type: 'user/message', surfaceOp: 'append', data: { content, source: { kind: 'user' } } })
append(id, { type: 'user/message', surfaceOp: 'append', data: { content: durable, source: { kind: 'user' } } })
startReply(id, turn, `回声：${userText}。这是 fixture 的流式回复，用于验证打字机增长与定稿切换。`)
return ok(request, { accepted: true as const })
},
attachment: (request) => {
const stored = attachments.get(String(request.payload.attachmentId))
if (stored === undefined) {
return err(request, { code: 'attachment-error', message: 'fixture attachment missing', details: { reason: 'ATTACHMENT_NOT_FOUND' } })
}
if (!logReferencesAttachment(logs.get(request.payload.sessionId) ?? [], String(request.payload.attachmentId))) {
return err(request, { code: 'attachment-error', message: 'fixture attachment is not referenced by this session', details: { reason: 'ATTACHMENT_NOT_REFERENCED' } })
}
return ok(request, stored)
},
cancel: (request) => {
const replay = replays.get(request.payload.sessionId)
if (replay !== undefined) {
@@ -421,7 +477,24 @@ export function createFixtureApi(): ApiProxy {
},
},
host: {
describe: request => ok(request, { version: '0.0.0-fixture', cwd: '/tmp/fixture', attachedSessions: 1 }),
describe: request => ok(request, {
version: '0.0.0-fixture',
cwd: '/tmp/fixture',
provider: 'fixture',
model: 'fx-vision',
activeModel: {
provider: 'fixture', id: 'fx-vision', name: 'Fixture Vision',
inputModalities: ['text', 'image'], outputModalities: ['text', 'image'],
},
imageLimits: {
maxImageBytes: 5 * 1024 * 1024,
maxImagesPerMessage: 10,
maxMessageImageBytes: 20 * 1024 * 1024,
maxImagePixels: 40_000_000,
mediaTypes: ['image/png', 'image/jpeg', 'image/webp', 'image/gif'],
},
attachedSessions: 1,
}),
},
events: {
async *mux(_request, signal) {
@@ -512,6 +585,7 @@ export class FixtureApiClient extends AbstractApiClient {
case 'session.create': return this.api.sessions.create(request)
case 'session.history': return this.api.sessions.history(request)
case 'session.prompt': return this.api.sessions.prompt(request)
case 'session.attachment': return this.api.sessions.attachment(request)
case 'session.cancel': return this.api.sessions.cancel(request)
case 'host.describe': return this.api.host.describe(request)
}

2
packages/client/connection/src/client/index.ts
View File

@@ -14,7 +14,7 @@ import { WebApiClient } from './web-api-client.ts'
// ---- Contract re-exports (browser-safe apiproxy channels + core types) ----
export type {
ApiProxy, SessionsApi, SessionSummary, HostApi, EventsApi, MuxFrame, HostFrame,
ApiProxy, SessionsApi, SessionSummary, PromptContentPart, HostApi, EventsApi, MuxFrame, HostFrame,
ApprovalResponsePayload, QuestionResponsePayload, HistoryEntry, ToolEventView,
ToolCallView, ToolResultView,
RpcRequest, RpcResponse, RpcResult, RpcError, RpcErrorCode,

3
packages/client/connection/tests/fake-api.ts
View File

@@ -48,6 +48,8 @@ export class FakeApiClient implements IApiClient {
() => Promise.resolve(ok({ events: [], hasMore: false }))
onPrompt: (payload: unknown) => Promise<RpcResponse<{ accepted: true }>> = () => Promise.resolve(ok({ accepted: true as const }))
onAttachment: (payload: unknown) => Promise<RpcResponse<{ attachment: { attachmentId: never; mediaType: 'image/png'; bytes: number; width: number; height: number }; data: string }>> =
() => Promise.resolve(ok({ attachment: { attachmentId: 'a' as never, mediaType: 'image/png', bytes: 1, width: 1, height: 1 }, data: 'AA==' }))
onCancel: (payload: unknown) => Promise<RpcResponse<{ accepted: true }>> = () => Promise.resolve(ok({ accepted: true as const }))
onDescribe: (payload: unknown) => Promise<RpcResponse<{ version: string; cwd: string; attachedSessions: number }>> =
() => Promise.resolve(ok({ version: '0-fake', cwd: '/f', attachedSessions: 0 }))
@@ -64,6 +66,7 @@ export class FakeApiClient implements IApiClient {
history: (payload: { sessionId: SessionId; beforeSeq?: number; maxMessages?: number }) =>
this.record('session.history', payload, this.onHistory(payload)),
prompt: (payload: unknown) => this.record('session.prompt', payload, this.onPrompt(payload)),
attachment: (payload: unknown) => this.record('session.attachment', payload, this.onAttachment(payload)),
cancel: (payload: unknown) => this.record('session.cancel', payload, this.onCancel(payload)),
}

29
packages/client/connection/tests/fixture.spec.ts
View File

@@ -167,7 +167,7 @@ describe('createFixtureApi', () => {
expect(second[1]?.rpcId).toBe(first[1]?.rpcId) // stable rpcId across replays (host replay semantics)
})
it('steer with no replay in flight falls through to a fresh queued turn; non-text blocks stringify empty', async () => {
it('steer with no replay in flight promotes image bytes to a session-scoped reference', async () => {
const api = createFixtureApi()
const abort = new AbortController()
const framesPromise = collect<MuxFrame>(api.events.mux(req({}), abort.signal), abort,
@@ -175,14 +175,36 @@ describe('createFixtureApi', () => {
await new Promise(resolve => setTimeout(resolve, 10))
const created = await api.sessions.create(req({}))
if (!created.result.ok) throw new Error('create failed')
// steer while idle + a non-text content block (covers the '' arm of the text join).
// steer while idle + an image: the fixture mirrors the host's durable send boundary.
await api.sessions.prompt(req({
sessionId: created.result.value.sessionId, mode: 'steer' as const,
content: [{ type: 'text' as const, text: '短' }, { type: 'image', data: 'x' } as never],
content: [{ type: 'text' as const, text: '短' }, {
type: 'image' as const,
mediaType: 'image/png' as const,
data: 'iVBORw0KGgoAAAANSUhEUgAAAKAAAABaCAYAAAA/xl1SAAAAvklEQVR42u3SMQ0AAAjAMIyhELM4AAe8PD1qYFlk9cCXEAEDYkAwIAYEA2JAMCAGBANiQDAgBgQDYkAwIAYEA2JAMCAGBANiQDAgBgQDYkAwIAYEA2JAMCAGxIBCYEAMCAbEgGBADAgGxIBgQAwIBsSAYEAMCAbEgGBADAgGxIBgQAwIBsSAYEAMCAbEgGBADAgGxIAYEAyIAcGAGBAMiAHBgBgQDIgBwYAYEAyIAcGAGBAMiAHBgBgQDIgB4bYWLb6pnOb1xAAAAABJRU5ErkJggg==',
name: 'pixel.png',
}],
}))
const frames = await framesPromise
const types = frames.filter((f): f is Extract<MuxFrame, { type: 'session/event' }> => f.type === 'session/event').map(f => f.event.type)
expect(types[0]).toBe('turn/start') // idle steer degraded to a queued turn, not a steering insert
const user = frames.find((f): f is Extract<MuxFrame, { type: 'session/event' }> =>
f.type === 'session/event' && f.event.type === 'user/message')
const image = ((user?.event.data as { content?: { type: string; attachment?: { attachmentId: never } }[] } | undefined)?.content)
?.find(block => block.type === 'image')
expect(image?.attachment).toBeDefined()
if (image?.attachment === undefined) throw new Error('fixture image missing')
const loaded = await api.sessions.attachment(req({
sessionId: created.result.value.sessionId,
attachmentId: image.attachment.attachmentId,
}))
expect(loaded.result).toMatchObject({ ok: true, value: { attachment: { name: 'pixel.png' } } })
const denied = await api.sessions.attachment(req({
sessionId: sid('fx-beta'), attachmentId: image.attachment.attachmentId,
}))
expect(denied.result).toMatchObject({
ok: false, error: { details: { reason: 'ATTACHMENT_NOT_REFERENCED' } },
})
})
it('gamma interval flip emits host/session-status and a running log-less session subscribes at lastSeq -1', async () => {
@@ -306,6 +328,7 @@ describe('FixtureApiClient (protocol-level fake carrier)', () => {
const id = created.result.value.sessionId
expect((await client.sessions.history({ sessionId: id })).result.ok).toBe(true)
expect((await client.sessions.prompt({ sessionId: id, mode: 'queue', content: [{ type: 'text', text: '嗨' }] })).result.ok).toBe(true)
expect((await client.sessions.attachment({ sessionId: sid('fx-alpha'), attachmentId: 'fixture:image' as never })).result.ok).toBe(true)
expect((await client.sessions.cancel({ sessionId: id })).result.ok).toBe(true)
expect((await client.host.describe({})).result.ok).toBe(true)
})

3
packages/client/connection/tsconfig.json
View File

@@ -8,6 +8,9 @@
"src"
],
"references": [
{
"path": "../../attachment/attachment"
},
{
"path": "../../llm/llm"
},