Coder/deepseek-harness
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: narrow web image input v1

Browse Source
This commit is contained in:
Tianyi Cui
2026-07-29 23:18:57 +08:00
parent 8348a409aa
commit 3a4c436d83
69 changed files with 280 additions and 613 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
packages/client/connection/src/client/fixture.ts
View File

@@ -1260,17 +1260,8 @@ export function createFixtureApi(options: FixtureOptions = {}): ApiProxy {
cwd: '/tmp/fixture',
provider: 'fixture',
model: 'fx-vision',
activeModel: {
provider: 'fixture',
id: 'fx-vision',
name: 'Fixture Vision',
inputModalities: ['text', 'image'],
outputModalities: ['text', 'image'],
},
imageLimits: {
maxImageBytes: 5 * 1024 * 1024,
maxImagesPerMessage: 10,
maxMessageImageBytes: 20 * 1024 * 1024,
maxImagePixels: 40_000_000,
mediaTypes: ['image/png', 'image/jpeg', 'image/webp', 'image/gif'],
},

2
packages/client/connection/src/index.ts
View File

@@ -52,7 +52,7 @@ export function apply(ctx: Context, config?: ConnectionConfig): void {
for (const entry of trustedHosts) assertTrustedAuthority(entry)
const apiHandler = toFetchHandler(ctx.apiProxy)
const maxRequestBodyBytes = Math.ceil(
ctx.attachments.imageLimits.maxMessageImageBytes * 4 / 3,
ctx.attachments.imageLimits.maxImageBytes * 4 / 3,
) + REQUEST_ENVELOPE_HEADROOM_BYTES
const route: WebRoute = {
kind: 'prefix',

2
packages/client/connection/tests/node-half.spec.ts
View File

@@ -23,7 +23,7 @@ function fakeHttpServer(routes: WebRoute[]): Pick<HttpServerService, 'register'
/** Structural attachments fake: the plugin only reads imageLimits for the body cap. */
function fakeAttachments(): AttachmentStore {
return { imageLimits: { maxMessageImageBytes: 20 * 1024 * 1024 } } as AttachmentStore
return { imageLimits: { maxImageBytes: 5 * 1024 * 1024 } } as AttachmentStore
}
/** Bodyless GET carrying the given headers (enough for the trust fence + bridge). */

8
packages/client/runtime/src/client/sessions/conversation.ts
View File

@@ -36,7 +36,7 @@ export interface AssistantProvenanceView {
export type AssistantBlock =
| { kind: 'text'; text: string }
| { kind: 'reasoning'; text: string }
| { kind: 'image'; attachment: ImageAttachmentRef; alt?: string }
| { kind: 'image'; attachment: ImageAttachmentRef }
| { kind: 'tool-call'; callId: string; name: string; argsRaw: string }
| { kind: 'other'; block: unknown }
@@ -58,11 +58,7 @@ export function toAssistantBlock(block: ContentBlock): AssistantBlock {
switch (block.type) {
case 'text': return { kind: 'text', text: block.text }
case 'reasoning': return { kind: 'reasoning', text: block.text }
case 'image': return {
kind: 'image',
attachment: block.attachment,
...block.alt === undefined ? {} : { alt: block.alt },
}
case 'image': return { kind: 'image', attachment: block.attachment }
case 'tool-call': return { kind: 'tool-call', callId: String(block.id), name: block.name, argsRaw: block.arguments }
default: return { kind: 'other', block }
}

16
packages/client/ui-conversation/src/client/apply.ts
View File

@@ -39,6 +39,13 @@ function scopedConversation(sessions: ISessions, id: SessionId): IConversation {
return conversation
}
/** Resolve package-internal attachment operations from the public service registration. */
function concreteConversation(ctx: Context): ConversationService {
const conversation = ctx.get('conversation') as ConversationService | undefined
if (conversation === undefined) throw new Error('ui-conversation: conversation service unavailable')
return conversation
}
/** Chain routing: claim the composer while an approval wait is pending (pure — owner props only). */
function selectApproval({ interactions }: ComposerChainProps): ApprovalWait | null {
return interactions.find((i): i is ApprovalWait => i.kind === 'approval') ?? null
@@ -157,8 +164,7 @@ export function apply(ctx: Context): void {
children: { 'conversation.view': { kind: 'list', scope: 'session' } },
store: chatStore,
inject: (sessionId: SessionId, _actions: BoundActions<typeof chatStore>): ConversationSessionInjected => {
const conversation = ctx.get('conversation')
if (conversation === undefined) throw new Error('ui-conversation: conversation service unavailable')
const conversation = concreteConversation(ctx)
return {
views: {
list: viewTabs,
@@ -186,8 +192,7 @@ export function apply(ctx: Context): void {
'conversation.input.model': { kind: 'single', scope: 'session' },
},
inject: (sessionId: SessionId): ComposerBarInjected => {
const conversation = ctx.get('conversation')
if (conversation === undefined) throw new Error('ui-conversation: conversation service unavailable')
const conversation = concreteConversation(ctx)
const shell = inputHub.shell(sessionId)
return {
keyboard: shell,
@@ -248,8 +253,7 @@ export function apply(ctx: Context): void {
},
store: chatStore,
inject: (sessionId: SessionId, actions: BoundActions<typeof chatStore>): ChatViewInjected => {
const conversation = ctx.get('conversation')
if (conversation === undefined) throw new Error('ui-conversation: conversation service unavailable')
const conversation = concreteConversation(ctx)
const scoped = scopedConversation(sessions, sessionId)
return {
openDetails: (target) => {

7
packages/client/ui-conversation/src/client/chat/MessageImage.tsx
View File

@@ -7,9 +7,8 @@ import css from './MessageImage.module.css'
export type ImageLoader = (attachment: ImageAttachmentRef) => Promise<string>
/** Compact history renderer with retryable loading and double-click original preview. */
export function MessageImage({ attachment, alt, load }: {
export function MessageImage({ attachment, load }: {
attachment: ImageAttachmentRef
alt?: string
load: ImageLoader
}) {
const [src, setSrc] = useState<string | null>(null)
@@ -34,7 +33,7 @@ export function MessageImage({ attachment, alt, load }: {
return () => { live = false }
}, [attachment, load])
const label = alt ?? attachment.name ?? '图片'
const label = attachment.name ?? '图片'
if (error) return <button type="button" className={css.error} onClick={request}>图片加载失败，点击重试</button>
return (
<>
@@ -55,7 +54,7 @@ export function MessageImage({ attachment, alt, load }: {
/** Wrapping image group shared by user and assistant history. */
export function ImageGallery({ images, load, align }: {
images: readonly { attachment: ImageAttachmentRef; alt?: string }[]
images: readonly { attachment: ImageAttachmentRef }[]
load: ImageLoader
align: 'start' | 'end'
}) {

9
packages/client/ui-conversation/src/client/chat/MessageItem.tsx
View File

@@ -23,18 +23,17 @@ type UserImage = Extract<UserMessageNode['content'][number], { type: 'image' }>
function contentParts(content: readonly unknown[]): {
text: string
images: { attachment: UserImage['attachment']; alt?: string }[]
images: { attachment: UserImage['attachment'] }[]
rest: unknown[]
} {
const texts: string[] = []
const images: { attachment: UserImage['attachment']; alt?: string }[] = []
const images: { attachment: UserImage['attachment'] }[] = []
const rest: unknown[] = []
for (const block of content) {
const b = block as { type?: string; text?: string; attachment?: unknown; alt?: string }
const b = block as { type?: string; text?: string; attachment?: unknown }
if (b.type === 'text' && typeof b.text === 'string') texts.push(b.text)
else if (b.type === 'image' && b.attachment !== undefined) {
const image = b as UserImage
images.push({ attachment: image.attachment, ...image.alt === undefined ? {} : { alt: image.alt } })
images.push({ attachment: (b as UserImage).attachment })
}
else rest.push(block)
}

7
packages/client/ui-conversation/src/client/input/hub.ts
View File

@@ -151,7 +151,12 @@ export class InputHub implements InputService {
// see them — release the drafts here instead of resurrecting them onto
// a dead instance where they would leak for the page lifetime.
if (this.shells.get(session.sessionId) === shell) {
shell?.restoreImages(imageIds)
if (shell?.snapshot.imageIds.length === 0) {
shell.restoreImages(imageIds)
} else {
const conversation = this.rootCtx.get('conversation') as ConversationAttachmentFace | undefined
for (const id of imageIds) conversation?.releaseDraftImage(id)
}
if (shell?.snapshot.draft === '') shell.setDraft(text)
return
}

81
packages/client/ui-conversation/src/client/service.ts
View File

@@ -29,10 +29,9 @@ export interface IConversation {
* Send a prompt into the caller scope's session.
* @param text - prompt text, sent verbatim as one text block.
* @param mode - queue after the current turn, or steer into it.
* @param images - browser-owned temporary images promoted by the host during this call.
* @returns completion; business failures reject (and land in promptError).
*/
send(text: string, mode: 'queue' | 'steer', images?: readonly File[]): Promise<void>
send(text: string, mode: 'queue' | 'steer'): Promise<void>
/**
* Cancel the scoped session's in-flight turn.
* @returns completion; failures reject as in send.
@@ -43,57 +42,11 @@ export interface IConversation {
* @returns completion of the page pull.
*/
loadOlder(): Promise<void>
/**
* Create runtime-only draft attachments and preview URLs.
* @param files - browser-owned image files.
* @param current - images already present in the composer.
* @returns ordered descriptors for the input state.
*/
createDraftImages(
files: readonly File[],
current?: readonly ComposerAttachment[],
): readonly ComposerAttachment[]
/**
* Resolve ordered draft ids to runtime-owned attachments.
* @param ids - ordered composer attachment ids.
* @returns attachments still available in this browser runtime.
*/
draftImages(ids: readonly string[]): readonly ComposerAttachment[]
/**
* Release one draft attachment and its preview URL.
* @param id - draft-local attachment id.
*/
releaseDraftImage(id: string): void
/**
* Resolve a session-authorized historical image to an object URL.
* @param sessionId - session whose durable log grants the read.
* @param attachment - durable image reference from that log.
* @returns browser URL for inline and original-size rendering.
*/
resolveImage(sessionId: SessionId, attachment: ImageAttachmentRef): Promise<string>
/**
* Release every historical image URL owned by one rendered session.
* @param sessionId - session whose rendered image scope is ending.
*/
releaseSessionImages(sessionId: SessionId): void
}
/** Opaque wrapper keeps browser `File` internals outside persisted store state. */
class BrowserDraftAttachment implements ComposerAttachment {
readonly kind = 'image' as const
readonly id: string
readonly previewUrl: string
readonly #file: File
constructor(file: File) {
this.id = crypto.randomUUID()
this.previewUrl = URL.createObjectURL(file)
this.#file = file
}
get file(): File {
return this.#file
}
/** Create one browser-only draft descriptor; only its id enters input state. */
function browserDraftAttachment(file: File): ComposerAttachment {
return { kind: 'image', id: crypto.randomUUID(), previewUrl: URL.createObjectURL(file), file }
}
interface ImageUrlEntry {
@@ -106,7 +59,7 @@ interface ImageUrlEntry {
export class ConversationService extends Service implements IConversation {
/** The per-session input machine registry (InputService face, design §5.2). */
readonly input: InputService
private readonly draftAttachments = new Map<string, BrowserDraftAttachment>()
private readonly draftAttachments = new Map<string, ComposerAttachment>()
private readonly imageUrls = new Map<string, ImageUrlEntry>()
private readonly imageGenerations = new Map<SessionId, number>()
private readonly createdImageUrls = new Set<string>()
@@ -135,11 +88,10 @@ export class ConversationService extends Service implements IConversation {
* exists for caller choreography (the composer restores the draft on it).
* @param text - prompt text, sent verbatim as one text block when non-empty.
* @param mode - queue after the current turn, or steer into it.
* @param images - browser-owned temporary images promoted by the host during this call.
*/
async send(text: string, mode: 'queue' | 'steer', images: readonly File[] = []): Promise<void> {
async send(text: string, mode: 'queue' | 'steer'): Promise<void> {
const session = this.scopedSession('send')
await this.sendFiles(session, text, mode, images)
await this.sendFiles(session, text, mode, [])
}
/**
@@ -190,7 +142,7 @@ export class ConversationService extends Service implements IConversation {
): readonly ComposerAttachment[] {
this.validateImages(files, current)
return files.map((file) => {
const attachment = new BrowserDraftAttachment(file)
const attachment = browserDraftAttachment(file)
this.draftAttachments.set(attachment.id, attachment)
this.createdImageUrls.add(attachment.previewUrl)
return attachment
@@ -330,19 +282,12 @@ export class ConversationService extends Service implements IConversation {
current: readonly ComposerAttachment[],
): void {
if (files.length === 0 && current.length === 0) return
// Deployment-wide limits only. Model capability is deliberately NOT
// checked here: the handshake's activeModel is the host default, not the
// session's current target (session.selectModel never refreshes it), so a
// client-side modality gate refuses sessions the host would accept and
// vice versa. The host preflight on session.prompt is the authority; its
// rejection renders through the composer error strip.
// Model capability is checked only by the host against the session's
// current target; the client owns deployment limits and the one-image UI.
const description = this.requireSessions().hostDescription()
const limits = description?.imageLimits
const all = [...current.map(attachment => attachment.file), ...files]
if (limits !== undefined && all.length > limits.maxImagesPerMessage) {
throw new Error(`每条消息最多添加 ${limits.maxImagesPerMessage} 张图片`)
}
let totalBytes = 0
if (all.length > 1) throw new Error('每条消息最多添加 1 张图片')
for (const file of all) {
const mediaType = imageMediaType(file.type)
if (limits !== undefined && !limits.mediaTypes.includes(mediaType)) {
@@ -351,10 +296,6 @@ export class ConversationService extends Service implements IConversation {
if (limits !== undefined && file.size > limits.maxImageBytes) {
throw new Error(`图片 ${file.name || '未命名图片'} 超过单张大小限制`)
}
totalBytes += file.size
}
if (limits !== undefined && totalBytes > limits.maxMessageImageBytes) {
throw new Error('图片总大小超过单条消息限制')
}
}

4
packages/client/ui-conversation/tests/message-image.spec.tsx
View File

@@ -48,14 +48,14 @@ describe('MessageImage', () => {
<AssistantMarkdown
blocks={[
{ kind: 'text', text: 'before' },
{ kind: 'image', attachment, alt: 'middle' },
{ kind: 'image', attachment },
{ kind: 'text', text: 'after' },
]}
streaming={false}
loadImage={() => Promise.resolve('blob:middle')}
/>,
)
const image = await view.findByAltText('middle')
const image = await view.findByAltText('history.png')
const before = view.getByText('before')
const after = view.getByText('after')
expect(before.compareDocumentPosition(image) & Node.DOCUMENT_POSITION_FOLLOWING).not.toBe(0)

19
packages/client/ui-conversation/tests/service-orchestration.spec.ts
View File

@@ -29,6 +29,25 @@ async function bench() {
}
describe('ConversationService', () => {
it('keeps the browser draft to one image', async () => {
const b = await bench()
const created = vi.spyOn(URL, 'createObjectURL').mockReturnValue('blob:draft-one')
const revoked = vi.spyOn(URL, 'revokeObjectURL').mockReturnValue(undefined)
try {
const [first] = b.root.createDraftImages([new File([Uint8Array.of(1)], 'first.png', { type: 'image/png' })])
if (first === undefined) throw new Error('draft attachment missing')
expect(() => b.root.createDraftImages(
[new File([Uint8Array.of(2)], 'second.png', { type: 'image/png' })],
[first],
)).toThrow('每条消息最多添加 1 张图片')
expect(created).toHaveBeenCalledOnce()
} finally {
created.mockRestore()
revoked.mockRestore()
}
await b.runtime.dispose()
})
it('routes operations through the public Session binding', async () => {
const b = await bench()
await b.scoped.send('hello', 'steer')

1
packages/client/ui-trajectory/src/client/layout.ts
View File

@@ -644,7 +644,6 @@ function assistantSourceBlock(block: AssistantBlock): TrajectorySourceBlock {
case 'image': return {
type: 'image',
content: stringifySourceValue(block.attachment),
...(block.alt !== undefined ? { imageAlt: block.alt } : {}),
}
case 'other': return sourceBlock(block.block)
}

6
packages/client/ui-trajectory/tests/client-bundle.spec.ts
View File

@@ -60,7 +60,7 @@ describe('tsdown client artifact', () => {
const { handoff, surface } = await loadArtifact()
expect(handoff.id).toBe(PLUGIN_ID)
expect(surface.apply).toBeTypeOf('function')
expect(surface.inject).toEqual(['slots', 'conversation', 'sessions'])
expect(surface.inject).toEqual(['slots', 'conversation', 'sessionHistory'])
})
it.skipIf(code === undefined)('mounted as an object plugin, apply registers the view tab on the real ring', async () => {
@@ -72,10 +72,10 @@ describe('tsdown client artifact', () => {
name: 'root',
children: { 'conversation.view': { kind: 'list', scope: 'session' } },
}, (_p: { renderSlot?: unknown }) => null)
// The plugin injects 'conversation' as an ordering edge and 'sessions'
// The plugin injects 'conversation' as an ordering edge and 'sessionHistory'
// for its per-session history callback; this bench supplies both.
ctx.provide('conversation', {})
ctx.provide('sessions', {})
ctx.provide('sessionHistory', {})
const fiber = ctx.plugin(surface as { apply: (ctx: Context) => void })
await fiber.await()
expect(slots.entries('conversation.view').map(e => e.options.id)).toEqual(['trajectory'])