fix: harden Web image admission

This commit is contained in:
Tianyi Cui
2026-07-30 01:58:36 +08:00
parent d6c82001b3
commit 515d48875e
52 changed files with 999 additions and 444 deletions

View File

@@ -2,5 +2,5 @@
# side as of the last confirmed-consistent state. Both languages carry equal authority;
# after editing either side, bring the other along and re-record with:
# pnpm run verify-translation-pairing --write packages/attachment/attachment-local/README.md
README.md: 38331df827d64c8370d0208f6898971e46010afa
README.zh.md: bc6a471465c7bffc136e8ec2233e200a292f2d1a
README.md: 310120bd1c3573da1e7c60334d5c2712195f3186
README.zh.md: 9fd3a857eca1c25c90b665735f67d2c27c92334a

View File

@@ -2,7 +2,7 @@
English | [中文](README.zh.md)
The private local implementation of [`@deepseek-ai/dsh-attachment`](../attachment). Objects land at `<DSH_HOME>/attachments/v1/objects/<sha256-prefix>/<sha256>` and are addressed by an opaque `sha256:` id. Writes use a private staging directory, owner-only files, a synced temporary file, an atomic exclusive hard-link publish, and a directory sync on the publication directories (POSIX; Windows relies on filesystem metadata journaling) so the reported reference survives a crash; reads re-check the digest, media signature, dimensions, and logged metadata. Byte and pixel limits are write-time admission policy, so a later policy reduction does not make already-admitted history unreadable.
The private local implementation of [`@deepseek-ai/dsh-attachment`](../attachment). Objects land at `<DSH_HOME>/attachments/v1/objects/<sha256-prefix>/<sha256>` and are addressed by an opaque `sha256:` id. Writes use a private staging directory, owner-only files, a synced temporary file, an atomic exclusive hard-link publish, and a directory sync on the publication directories (POSIX; Windows relies on filesystem metadata journaling) so the reported reference survives a crash. Write admission and reads fully decode the raster before accepting its format and dimensions; reads also re-check the digest and logged metadata. Byte and pixel limits are write-time admission policy, so a later policy reduction does not make already-admitted history unreadable.
`DSH_HOME` resolves through the shared path policy: explicit config, `$DSH_HOME`, then `~/.dsh`. Session logs contain only the reference and verified metadata, never this host path.

View File

@@ -2,7 +2,7 @@
[English](README.md) | 中文
这是 [`@deepseek-ai/dsh-attachment`](../attachment) 的私有本地实现。对象存放在 `<DSH_HOME>/attachments/v1/objects/<sha256-prefix>/<sha256>`,并通过不透明的 `sha256:` 标识符寻址。写入过程使用私有暂存目录、仅所有者可访问的文件、经过同步的临时文件、原子且排他的硬链接发布,并对发布目录执行同步(适用于 POSIXWindows 依赖文件系统元数据日志),确保已报告的引用能够在崩溃后继续存在;读取过程会重新校验摘要、媒体签名、尺寸和已记录的元数据。字节和像素限制属于写入时的准入策略,因此后续收紧限制不会导致已经接纳的历史记录变得不可读。
这是 [`@deepseek-ai/dsh-attachment`](../attachment) 的私有本地实现。对象存放在 `<DSH_HOME>/attachments/v1/objects/<sha256-prefix>/<sha256>`,并通过不透明的 `sha256:` 标识符寻址。写入过程使用私有暂存目录、仅所有者可访问的文件、经过同步的临时文件、原子且排他的硬链接发布,并对发布目录执行同步(适用于 POSIXWindows 依赖文件系统元数据日志),确保已报告的引用能够在崩溃后继续存在。写入准入与读取都会完整解码光栅图片,之后才接受其格式和尺寸;读取会重新校验摘要和已记录的元数据。字节和像素限制属于写入时的准入策略,因此后续收紧限制不会导致已经接纳的历史记录变得不可读。
`DSH_HOME` 按共享路径策略解析:显式配置、`$DSH_HOME`,最后是 `~/.dsh`。会话日志只包含引用和经过校验的元数据,绝不包含这个宿主路径。

View File

@@ -20,7 +20,10 @@
"@deepseek-ai/dsh-paths": "^0.0.1",
"cordis": "^4.0.0-rc.6"
},
"dependencies": { "schemastery": "^3.18.0" },
"dependencies": {
"schemastery": "^3.18.0",
"sharp": "^0.35.3"
},
"devDependencies": {
"@deepseek-ai/dsh-attachment": "workspace:^",
"@deepseek-ai/dsh-invariants": "workspace:^",

View File

@@ -1,102 +1,45 @@
/** Minimal raster header validation used before bytes enter durable storage. */
/** Raster decoding used before bytes enter durable storage. */
import sharp from 'sharp'
import { AttachmentError } from '@deepseek-ai/dsh-attachment'
import type { ImageMediaType } from '@deepseek-ai/dsh-attachment'
/** Decoded metadata from a supported image header. */
/** Decoded metadata from a supported image. */
export interface DetectedImage {
mediaType: ImageMediaType
width: number
height: number
}
function ascii(data: Uint8Array, start: number, value: string): boolean {
/* v8 ignore next -- Every call site establishes the fixed header span before comparing it. */
if (data.length < start + value.length) return false
for (let i = 0; i < value.length; i++) if (data[start + i] !== value.charCodeAt(i)) return false
return true
}
function u16be(data: Uint8Array, offset: number): number {
return new DataView(data.buffer, data.byteOffset, data.byteLength).getUint16(offset)
}
function u16le(data: Uint8Array, offset: number): number {
return new DataView(data.buffer, data.byteOffset, data.byteLength).getUint16(offset, true)
}
function u24le(data: Uint8Array, offset: number): number {
const view = new DataView(data.buffer, data.byteOffset, data.byteLength)
return view.getUint8(offset) | (view.getUint8(offset + 1) << 8) | (view.getUint8(offset + 2) << 16)
}
function u32be(data: Uint8Array, offset: number): number {
return new DataView(data.buffer, data.byteOffset, data.byteLength).getUint32(offset)
}
function u32le(data: Uint8Array, offset: number): number {
return new DataView(data.buffer, data.byteOffset, data.byteLength).getUint32(offset, true)
}
function dimensions(width: number, height: number, mediaType: ImageMediaType): DetectedImage {
if (width < 1 || height < 1) throw new AttachmentError('Image dimensions must be positive.', 'INVALID_IMAGE')
return { mediaType, width, height }
}
function jpeg(data: Uint8Array): DetectedImage | null {
if (data[0] !== 0xff || data[1] !== 0xd8) return null
const sof = new Set([0xc0, 0xc1, 0xc2, 0xc3, 0xc5, 0xc6, 0xc7, 0xc9, 0xca, 0xcb, 0xcd, 0xce, 0xcf])
let offset = 2
while (offset + 3 < data.length) {
while (data[offset] === 0xff) offset++
const marker = data[offset]
if (marker === undefined || marker === 0xd9 || marker === 0xda) break
if (marker === 0x01 || (marker >= 0xd0 && marker <= 0xd7)) {
offset++
continue
}
const length = u16be(data, offset + 1)
if (length < 2 || offset + 1 + length > data.length) throw new AttachmentError('JPEG data is truncated.', 'INVALID_IMAGE')
if (sof.has(marker)) {
if (length < 7) throw new AttachmentError('JPEG dimensions are truncated.', 'INVALID_IMAGE')
return dimensions(u16be(data, offset + 6), u16be(data, offset + 4), 'image/jpeg')
}
offset += length + 1
}
throw new AttachmentError('JPEG dimensions are missing.', 'INVALID_IMAGE')
const MEDIA_TYPES: Readonly<Record<string, ImageMediaType>> = {
png: 'image/png',
jpeg: 'image/jpeg',
webp: 'image/webp',
gif: 'image/gif',
}
/**
* Detect a supported raster type and intrinsic dimensions from encoded bytes.
* Decode a supported raster and return its intrinsic metadata.
* @param data - complete encoded image bytes.
* @param maxPixels - optional write-time decoded-pixel limit; reads omit it.
* @returns verified format and dimensions.
*/
export function detectImage(data: Uint8Array): DetectedImage {
if (data.length >= 24
&& data[0] === 0x89 && ascii(data, 1, 'PNG\r\n\u001a\n') && ascii(data, 12, 'IHDR')) {
return dimensions(u32be(data, 16), u32be(data, 20), 'image/png')
}
if (data.length >= 10 && (ascii(data, 0, 'GIF87a') || ascii(data, 0, 'GIF89a'))) {
return dimensions(u16le(data, 6), u16le(data, 8), 'image/gif')
}
const detectedJpeg = jpeg(data)
if (detectedJpeg !== null) return detectedJpeg
if (data.length >= 30 && ascii(data, 0, 'RIFF') && ascii(data, 8, 'WEBP')) {
const declaredLength = u32le(data, 4) + 8
if (declaredLength > data.length) throw new AttachmentError('WebP data is truncated.', 'INVALID_IMAGE')
if (ascii(data, 12, 'VP8X')) return dimensions(u24le(data, 24) + 1, u24le(data, 27) + 1, 'image/webp')
if (ascii(data, 12, 'VP8L') && data[20] === 0x2f) {
const view = new DataView(data.buffer, data.byteOffset, data.byteLength)
const b0 = view.getUint8(21)
const b1 = view.getUint8(22)
const b2 = view.getUint8(23)
const b3 = view.getUint8(24)
return dimensions(1 + b0 + ((b1 & 0x3f) << 8), 1 + (b1 >> 6) + (b2 << 2) + ((b3 & 0x0f) << 10), 'image/webp')
export async function detectImage(data: Uint8Array, maxPixels?: number): Promise<DetectedImage> {
try {
const image = sharp(data, { failOn: 'error', limitInputPixels: false })
const metadata = await image.metadata()
const mediaType = MEDIA_TYPES[metadata.format as string]
if (mediaType === undefined) {
throw new AttachmentError('Unsupported or malformed image data.', 'INVALID_IMAGE')
}
if (ascii(data, 12, 'VP8 ') && data[23] === 0x9d && data[24] === 0x01 && data[25] === 0x2a) {
return dimensions(u16le(data, 26) & 0x3fff, u16le(data, 28) & 0x3fff, 'image/webp')
const { width, height } = metadata
if (maxPixels !== undefined && width * height > maxPixels) {
throw new AttachmentError('Image exceeds the configured decoded-pixel limit.', 'IMAGE_TOO_MANY_PIXELS')
}
throw new AttachmentError('WebP dimensions are missing.', 'INVALID_IMAGE')
await image.raw().toBuffer()
return { mediaType, width, height }
} catch (error) {
if (error instanceof AttachmentError) throw error
throw new AttachmentError('Unsupported or malformed image data.', 'INVALID_IMAGE', { cause: error })
}
throw new AttachmentError('Unsupported or malformed image data.', 'INVALID_IMAGE')
}

View File

@@ -60,8 +60,8 @@ export class LocalAttachmentStore extends AttachmentStore {
})
}
validateImage(input: SaveImageAttachment): void {
validateImageFile(input, this.imageLimits)
async validateImage(input: SaveImageAttachment): Promise<void> {
await validateImageFile(input, this.imageLimits)
}
async saveImage(input: SaveImageAttachment): Promise<ImageAttachmentRef> {

View File

@@ -38,27 +38,28 @@ function ensureReference(ref: ImageAttachmentRef): string {
return match[1]
}
function inspectMetadata(data: Uint8Array, declaredMediaType: ImageAttachmentRef['mediaType']): Omit<ImageAttachmentRef, 'attachmentId' | 'name'> {
async function inspectMetadata(
data: Uint8Array,
declaredMediaType: ImageAttachmentRef['mediaType'],
maxPixels?: number,
): Promise<Omit<ImageAttachmentRef, 'attachmentId' | 'name'>> {
if (data.byteLength === 0) throw new AttachmentError('Image is empty.', 'INVALID_IMAGE')
const detected = detectImage(data)
const detected = await detectImage(data, maxPixels)
if (detected.mediaType !== declaredMediaType) throw new AttachmentError('Declared image type does not match its bytes.', 'IMAGE_TYPE_MISMATCH')
return { ...detected, bytes: data.byteLength }
}
function validateAdmission(metadata: Omit<ImageAttachmentRef, 'attachmentId' | 'name'>, limits: ImageAttachmentLimits): void {
if (metadata.bytes > limits.maxImageBytes) throw new AttachmentError('Image exceeds the configured byte limit.', 'IMAGE_TOO_LARGE')
if (metadata.width * metadata.height > limits.maxImagePixels) {
throw new AttachmentError('Image exceeds the configured decoded-pixel limit.', 'IMAGE_TOO_MANY_PIXELS')
}
}
/**
* Run the full admission policy for one image without touching storage.
* @param input - encoded bytes and declared metadata.
* @param limits - resolved storage policy.
* @returns completion after the encoded raster has been fully decoded.
*/
export function validateImageFile(input: SaveImageAttachment, limits: ImageAttachmentLimits): void {
validateAdmission(inspectMetadata(input.data, input.mediaType), limits)
export async function validateImageFile(input: SaveImageAttachment, limits: ImageAttachmentLimits): Promise<void> {
if (input.data.byteLength > limits.maxImageBytes) {
throw new AttachmentError('Image exceeds the configured byte limit.', 'IMAGE_TOO_LARGE')
}
await inspectMetadata(input.data, input.mediaType, limits.maxImagePixels)
}
/**
@@ -112,8 +113,8 @@ async function ensureDurableDirectory(path: string, boundary: string): Promise<v
* @returns durable content-addressed reference.
*/
export async function saveImageFile(root: string, input: SaveImageAttachment, limits: ImageAttachmentLimits): Promise<ImageAttachmentRef> {
const metadata = inspectMetadata(input.data, input.mediaType)
validateAdmission(metadata, limits)
if (input.data.byteLength > limits.maxImageBytes) throw new AttachmentError('Image exceeds the configured byte limit.', 'IMAGE_TOO_LARGE')
const metadata = await inspectMetadata(input.data, input.mediaType, limits.maxImagePixels)
const sha256 = digest(input.data)
const bucket = join(root, 'objects', sha256.slice(0, 2))
const staging = join(root, 'tmp')
@@ -187,7 +188,7 @@ export async function readImageFile(root: string, ref: ImageAttachmentRef): Prom
throw new AttachmentError('Unable to read image attachment.', 'ATTACHMENT_READ_FAILED', { cause: error })
}
if (digest(data) !== sha256) throw new AttachmentError('Stored attachment failed integrity verification.', 'ATTACHMENT_CORRUPT')
const metadata = inspectMetadata(data, ref.mediaType)
const metadata = await inspectMetadata(data, ref.mediaType)
if (metadata.bytes !== ref.bytes || metadata.width !== ref.width || metadata.height !== ref.height) {
throw new AttachmentError('Stored attachment metadata does not match its reference.', 'ATTACHMENT_CORRUPT')
}

View File

@@ -1,93 +1,42 @@
import sharp from 'sharp'
import { describe, expect, it } from 'vitest'
import { detectImage } from '../src/image.ts'
function bytes(text: string): number[] {
return [...Buffer.from(text, 'ascii')]
async function raster(format: 'png' | 'jpeg' | 'webp' | 'gif'): Promise<Uint8Array> {
const image = sharp({
create: { width: 3, height: 2, channels: 4, background: { r: 1, g: 2, b: 3, alpha: 1 } },
})
return new Uint8Array(await image.toFormat(format).toBuffer())
}
function webp(chunk: string, mutate: (data: Uint8Array) => void): Uint8Array {
const data = new Uint8Array(30)
data.set(bytes('RIFF'), 0)
data.set([22, 0, 0, 0], 4)
data.set(bytes('WEBP'), 8)
data.set(bytes(chunk), 12)
mutate(data)
return data
}
describe('raster header detection', () => {
it('detects PNG dimensions', () => {
const data = Uint8Array.from(Buffer.from(
'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=',
'base64',
))
expect(detectImage(data)).toEqual({ mediaType: 'image/png', width: 1, height: 1 })
describe('raster decoding', () => {
it('decodes every supported format and its intrinsic dimensions', async () => {
for (const [format, mediaType] of [
['png', 'image/png'],
['jpeg', 'image/jpeg'],
['webp', 'image/webp'],
['gif', 'image/gif'],
] as const) {
await expect(detectImage(await raster(format)))
.resolves.toEqual({ mediaType, width: 3, height: 2 })
}
})
it('detects both GIF revisions and rejects zero dimensions', () => {
expect(detectImage(Uint8Array.from([...bytes('GIF87a'), 3, 0, 2, 0])))
.toEqual({ mediaType: 'image/gif', width: 3, height: 2 })
expect(detectImage(Uint8Array.from([...bytes('GIF89a'), 4, 0, 5, 0])))
.toEqual({ mediaType: 'image/gif', width: 4, height: 5 })
expect(() => detectImage(Uint8Array.from([...bytes('GIF89a'), 0, 0, 1, 0])))
.toThrow(/positive/)
expect(() => detectImage(Uint8Array.from([...bytes('GIF89a'), 1, 0, 0, 0])))
.toThrow(/positive/)
it('rejects excess decoded pixels before decoding', async () => {
await expect(detectImage(await raster('png'), 5))
.rejects.toMatchObject({ code: 'IMAGE_TOO_MANY_PIXELS' })
})
it('walks JPEG marker forms and reports malformed dimensions', () => {
const sof = [0xff, 0xc0, 0, 7, 8, 0, 2, 0, 3]
expect(detectImage(Uint8Array.from([0xff, 0xd8, ...sof])))
.toEqual({ mediaType: 'image/jpeg', width: 3, height: 2 })
expect(detectImage(Uint8Array.from([
0xff, 0xd8,
0xe0, 0, 2,
0x01,
0xff, ...sof,
]))).toEqual({ mediaType: 'image/jpeg', width: 3, height: 2 })
expect(() => detectImage(Uint8Array.from([0xff, 0xd8, 0xd9, 0, 0, 0])))
.toThrow(/missing/)
expect(() => detectImage(Uint8Array.from([0xff, 0xd8, 0xff, 0xff, 0xff, 0xff])))
.toThrow(/missing/)
expect(() => detectImage(Uint8Array.from([0xff, 0xd8, 0xe0, 0, 1, 0])))
.toThrow(/truncated/)
expect(() => detectImage(Uint8Array.from([0xff, 0xd8, 0xe0, 0, 9, 0])))
.toThrow(/truncated/)
expect(() => detectImage(Uint8Array.from([0xff, 0xd8, 0xc0, 0, 6, 0, 0, 0, 0])))
.toThrow(/dimensions are truncated/)
})
it('detects each WebP header and rejects truncated or unknown chunks', () => {
expect(detectImage(webp('VP8X', (data) => {
data.set([2, 0, 0], 24)
data.set([3, 0, 0], 27)
}))).toEqual({ mediaType: 'image/webp', width: 3, height: 4 })
expect(detectImage(webp('VP8L', (data) => {
data[20] = 0x2f
data.set([2, 0, 1, 0], 21)
}))).toEqual({ mediaType: 'image/webp', width: 3, height: 5 })
expect(detectImage(webp('VP8 ', (data) => {
data.set([0x9d, 0x01, 0x2a], 23)
data.set([6, 0, 7, 0], 26)
}))).toEqual({ mediaType: 'image/webp', width: 6, height: 7 })
const truncated = webp('VP8X', () => {})
truncated[4] = 23
expect(() => detectImage(truncated)).toThrow(/truncated/)
expect(() => detectImage(webp('NOPE', () => {}))).toThrow(/dimensions are missing/)
expect(() => detectImage(webp('VP8L', () => {}))).toThrow(/dimensions are missing/)
expect(() => detectImage(webp('VP8 ', () => {}))).toThrow(/dimensions are missing/)
})
it('rejects unrecognized bytes and near-miss signatures', () => {
expect(() => detectImage(new Uint8Array(0))).toThrow(/Unsupported/)
expect(() => detectImage(Uint8Array.from([...bytes('GIFxxa'), 1, 0, 1, 0])))
.toThrow(/Unsupported/)
const nearWebp = webp('VP8X', () => {})
nearWebp[8] = 0
expect(() => detectImage(nearWebp)).toThrow(/Unsupported/)
it('rejects malformed bytes and truncated payloads with readable headers', async () => {
await expect(detectImage(Uint8Array.of(1, 2, 3)))
.rejects.toMatchObject({ code: 'INVALID_IMAGE' })
const unsupported = await sharp({
create: { width: 1, height: 1, channels: 4, background: { r: 0, g: 0, b: 0, alpha: 1 } },
}).tiff().toBuffer()
await expect(detectImage(unsupported)).rejects.toMatchObject({ code: 'INVALID_IMAGE' })
const complete = await raster('png')
const truncated = complete.subarray(0, 62)
await expect(sharp(truncated).metadata()).resolves.toMatchObject({ width: 3, height: 2 })
await expect(detectImage(truncated)).rejects.toMatchObject({ code: 'INVALID_IMAGE' })
})
})

View File

@@ -42,13 +42,13 @@ describe('local attachment service', () => {
const dshHome = await mkdtemp(join(tmpdir(), 'dsh-attachment-validate-'))
try {
const service = new LocalAttachmentStore(new Context(), { dshHome })
expect(() => { service.validateImage({ data: Uint8Array.of(1, 2, 3), mediaType: 'image/png' }) })
.toThrow(/Unsupported or malformed image data/)
await expect(service.validateImage({ data: Uint8Array.of(1, 2, 3), mediaType: 'image/png' }))
.rejects.toThrow(/Unsupported or malformed image data/)
const valid = Uint8Array.from(Buffer.from(
'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=',
'base64',
))
expect(() => { service.validateImage({ data: valid, mediaType: 'image/png' }) }).not.toThrow()
await expect(service.validateImage({ data: valid, mediaType: 'image/png' })).resolves.toBeUndefined()
expect(existsSync(service.root)).toBe(false)
} finally {
await rm(dshHome, { recursive: true, force: true })

View File

@@ -5,6 +5,7 @@ import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { mkdtemp, rm } from 'node:fs/promises'
import { afterEach, describe, expect, it, vi } from 'vitest'
import sharp from 'sharp'
import type { ImageAttachmentLimits } from '@deepseek-ai/dsh-attachment'
import { readImageFile, saveImageFile } from '../src/store.ts'
@@ -122,8 +123,9 @@ describe('local attachment store', () => {
data: PNG, mediaType: 'image/png',
}, { ...LIMITS, maxImageBytes: 1 })).rejects.toMatchObject({ code: 'IMAGE_TOO_LARGE' })
const wide = PNG.slice()
wide.set([0, 0, 0, 5, 0, 0, 0, 5], 16)
const wide = new Uint8Array(await sharp({
create: { width: 5, height: 5, channels: 4, background: { r: 0, g: 0, b: 0, alpha: 1 } },
}).png().toBuffer())
await expect(saveImageFile(storageRoot, {
data: wide, mediaType: 'image/png',
}, LIMITS)).rejects.toMatchObject({ code: 'IMAGE_TOO_MANY_PIXELS' })

View File

@@ -37,8 +37,9 @@ export abstract class AttachmentStore extends Service {
* Validate one image without persisting it.
* Batch callers validate every member before saving any member.
* @param input - encoded bytes, declared media type, and optional display name.
* @returns completion after the encoded raster has been fully decoded.
*/
abstract validateImage(input: SaveImageAttachment): void
abstract validateImage(input: SaveImageAttachment): Promise<void>
/**
* Validate and durably commit one image before its owning session event is appended.

View File

@@ -33,7 +33,7 @@ export interface ImageAttachmentRef {
name?: string
}
/** Deployment-resolved limits shared by upload consumers and UI preflight. */
/** Deployment-resolved limits used by upload admission and request buffering. */
export interface ImageAttachmentLimits {
maxImageBytes: number
maxImagesPerMessage: number
@@ -45,7 +45,7 @@ export interface ImageAttachmentLimits {
/** Request to validate and durably commit one image. */
export interface SaveImageAttachment {
data: Uint8Array
/** Caller-declared media type, checked against magic bytes. */
/** Caller-declared media type, checked against fully decoded bytes. */
mediaType: ImageMediaType
/** Optional browser/provider display name; it is never interpreted as a path. */
name?: string