fix(web): harden multimodal draft and storage lifecycle
This commit is contained in:
@@ -2,5 +2,5 @@
|
||||
# side as of the last confirmed-consistent state. Both languages carry equal authority;
|
||||
# after editing either side, bring the other along and re-record with:
|
||||
# pnpm run verify-translation-pairing --write packages/attachment/attachment-local/README.md
|
||||
README.md: 310120bd1c3573da1e7c60334d5c2712195f3186
|
||||
README.zh.md: 9fd3a857eca1c25c90b665735f67d2c27c92334a
|
||||
README.md: 80001b29b392fe1c8b663f46d47f1ec0726e6d0f
|
||||
README.zh.md: c3b95ace06b9f5ada156f20f33a1740a235400aa
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
English | [中文](README.zh.md)
|
||||
|
||||
The private local implementation of [`@deepseek-ai/dsh-attachment`](../attachment). Objects land at `<DSH_HOME>/attachments/v1/objects/<sha256-prefix>/<sha256>` and are addressed by an opaque `sha256:` id. Writes use a private staging directory, owner-only files, a synced temporary file, an atomic exclusive hard-link publish, and a directory sync on the publication directories (POSIX; Windows relies on filesystem metadata journaling) so the reported reference survives a crash. Write admission and reads fully decode the raster before accepting its format and dimensions; reads also re-check the digest and logged metadata. Byte and pixel limits are write-time admission policy, so a later policy reduction does not make already-admitted history unreadable.
|
||||
The private local implementation of [`@deepseek-ai/dsh-attachment`](../attachment). Objects land at `<DSH_HOME>/attachments/v1/objects/<sha256-prefix>/<sha256>` and are addressed by an opaque `sha256:` id. Each process proves a home durable once by syncing every ancestor entry to the filesystem root, so a directory another process created but has not yet synced is never mistaken for a safe boundary. Writes then use a private staging directory, owner-only files, a synced temporary file, an atomic exclusive hard-link publish, and directory syncs on the publication path (POSIX; Windows relies on filesystem metadata journaling) so the reported reference survives a crash. Write admission and reads fully decode the raster before accepting its format and dimensions; reads also re-check the digest and logged metadata. Byte and pixel limits are write-time admission policy, so a later policy reduction does not make already-admitted history unreadable.
|
||||
|
||||
`DSH_HOME` resolves through the shared path policy: explicit config, `$DSH_HOME`, then `~/.dsh`. Session logs contain only the reference and verified metadata, never this host path.
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
[English](README.md) | 中文
|
||||
|
||||
这是 [`@deepseek-ai/dsh-attachment`](../attachment) 的私有本地实现。对象存放在 `<DSH_HOME>/attachments/v1/objects/<sha256-prefix>/<sha256>`,并通过不透明的 `sha256:` 标识符寻址。写入过程使用私有暂存目录、仅所有者可访问的文件、经过同步的临时文件、原子且排他的硬链接发布,并对发布目录执行同步(适用于 POSIX;Windows 依赖文件系统元数据日志),确保已报告的引用能够在崩溃后继续存在。写入准入与读取都会完整解码光栅图片,之后才接受其格式和尺寸;读取还会重新校验摘要和已记录的元数据。字节和像素限制属于写入时的准入策略,因此后续收紧限制不会导致已经接纳的历史记录变得不可读。
|
||||
这是 [`@deepseek-ai/dsh-attachment`](../attachment) 的私有本地实现。对象存放在 `<DSH_HOME>/attachments/v1/objects/<sha256-prefix>/<sha256>`,并通过不透明的 `sha256:` 标识符寻址。每个进程都会通过将每个祖先目录项逐级同步到文件系统根目录,为某个 home 一次性证明其持久性,因此绝不会把另一个进程已经创建但尚未同步的目录误认为安全边界。随后,写入过程使用私有暂存目录、仅所有者可访问的文件、经过同步的临时文件、原子且排他的硬链接发布,并对发布路径执行目录同步(适用于 POSIX;Windows 依赖文件系统元数据日志),确保已报告的引用能够在崩溃后继续存在。写入准入与读取都会完整解码光栅图片,之后才接受其格式和尺寸;读取还会重新校验摘要和已记录的元数据。字节和像素限制属于写入时的准入策略,因此后续收紧限制不会导致已经接纳的历史记录变得不可读。
|
||||
|
||||
`DSH_HOME` 按共享路径策略解析:显式配置、`$DSH_HOME`,最后是 `~/.dsh`。会话日志只包含引用和经过校验的元数据,绝不包含这个宿主路径。
|
||||
|
||||
|
||||
@@ -2,8 +2,8 @@
|
||||
|
||||
import { createHash, randomUUID } from 'node:crypto'
|
||||
import { constants } from 'node:fs'
|
||||
import { chmod, link, mkdir, open, readFile, stat, unlink } from 'node:fs/promises'
|
||||
import { dirname, join, resolve } from 'node:path'
|
||||
import { chmod, link, mkdir, open, readFile, unlink } from 'node:fs/promises'
|
||||
import { dirname, join, parse, resolve } from 'node:path'
|
||||
import {
|
||||
AttachmentError,
|
||||
AttachmentId,
|
||||
@@ -17,6 +17,7 @@ import type {
|
||||
import { detectImage, probeImage } from './image.ts'
|
||||
|
||||
const ID_PATTERN = /^sha256:([a-f0-9]{64})$/
|
||||
const durableHomes = new Set<string>()
|
||||
|
||||
function digest(data: Uint8Array): string {
|
||||
return createHash('sha256').update(data).digest('hex')
|
||||
@@ -83,31 +84,6 @@ async function syncDirectory(path: string): Promise<void> {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Walk up from a preferred boundary to the deepest ancestor that already
|
||||
* exists. A first save may create DSH_HOME itself (recursive mkdir), and a
|
||||
* directory this process creates is not durable until its parent entry syncs
|
||||
* — so only a pre-existing directory may be vouched as the durable stop.
|
||||
* @param path - preferred absolute boundary.
|
||||
* @returns `path` when it exists, else its closest existing ancestor.
|
||||
*/
|
||||
async function existingBoundary(path: string): Promise<string> {
|
||||
let level = resolve(path)
|
||||
while (true) {
|
||||
try {
|
||||
await stat(level)
|
||||
return level
|
||||
} catch {
|
||||
// Swallows only the stat probe's failure: a missing (or unreadable)
|
||||
// level simply moves the boundary up; mkdir later surfaces real errors.
|
||||
}
|
||||
const parent = dirname(level)
|
||||
/* v8 ignore next -- filesystem-root guard: the root directory always exists, so stat returns first. */
|
||||
if (parent === level) return level
|
||||
level = parent
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create one private directory tree and persist every ancestor entry up to a
|
||||
* caller-vouched durable boundary. The walk deliberately ignores what mkdir
|
||||
@@ -134,6 +110,20 @@ async function ensureDurableDirectory(path: string, boundary: string): Promise<v
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Establish this process's proof that one DSH_HOME entry and every ancestor
|
||||
* below the filesystem root are durable. Mere existence is insufficient: a
|
||||
* concurrent process may have created the directory but not synced its parent.
|
||||
*/
|
||||
async function ensureDurableHome(path: string): Promise<string> {
|
||||
const home = resolve(path)
|
||||
if (!durableHomes.has(home)) {
|
||||
await ensureDurableDirectory(home, parse(home).root)
|
||||
durableHomes.add(home)
|
||||
}
|
||||
return home
|
||||
}
|
||||
|
||||
/**
|
||||
* Save and verify immutable image bytes below a versioned attachment root.
|
||||
* @param root - absolute `DSH_HOME/attachments/v1` root.
|
||||
@@ -147,12 +137,10 @@ export async function saveImageFile(root: string, input: SaveImageAttachment, li
|
||||
const sha256 = digest(input.data)
|
||||
const bucket = join(root, 'objects', sha256.slice(0, 2))
|
||||
const staging = join(root, 'tmp')
|
||||
// The preferred boundary is the root's grandparent (DSH_HOME for the
|
||||
// documented `DSH_HOME/attachments/v1` layout): `attachments`/`v1` may be
|
||||
// first-created by a concurrent save, so their entries sync on every path.
|
||||
// When DSH_HOME itself does not exist yet, the boundary retreats to its
|
||||
// closest existing ancestor so the first save syncs the new home entry too.
|
||||
const boundary = await existingBoundary(dirname(dirname(resolve(root))))
|
||||
// Establish DSH_HOME itself against the filesystem root once per process.
|
||||
// Every process performs that proof independently, so observing a directory
|
||||
// another process created can never be mistaken for durable publication.
|
||||
const boundary = await ensureDurableHome(dirname(dirname(resolve(root))))
|
||||
await ensureDurableDirectory(bucket, boundary)
|
||||
await ensureDurableDirectory(staging, boundary)
|
||||
const temporary = join(staging, randomUUID())
|
||||
|
||||
@@ -2,7 +2,7 @@ import { createHash } from 'node:crypto'
|
||||
import { constants } from 'node:fs'
|
||||
import { chmod, mkdir, readFile, stat, writeFile } from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { dirname, join, parse, resolve } from 'node:path'
|
||||
import { mkdtemp, rm } from 'node:fs/promises'
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import sharp from 'sharp'
|
||||
@@ -43,6 +43,17 @@ async function root(): Promise<string> {
|
||||
return join(value, 'attachments', 'v1')
|
||||
}
|
||||
|
||||
function parentChainToRoot(path: string): string[] {
|
||||
const parents: string[] = []
|
||||
let level = resolve(path)
|
||||
const root = parse(level).root
|
||||
while (level !== root) {
|
||||
level = dirname(level)
|
||||
parents.push(level)
|
||||
}
|
||||
return parents
|
||||
}
|
||||
|
||||
afterEach(async () => {
|
||||
await Promise.all(roots.splice(0).map(path => rm(path, { recursive: true, force: true })))
|
||||
})
|
||||
@@ -58,10 +69,11 @@ describe('local attachment store', () => {
|
||||
|
||||
await saveImageFile(storageRoot, { data: PNG, mediaType: 'image/png' }, LIMITS)
|
||||
|
||||
// Every level between each created directory and the vouched boundary
|
||||
// syncs unconditionally — "already existed" is not "already durable"
|
||||
// when a concurrent first save may have created but not yet synced it.
|
||||
// Each process first proves DSH_HOME durable all the way to the filesystem
|
||||
// root; existence alone cannot vouch for a concurrent creator's fsync.
|
||||
// Later directory creation can then stop at that process-proven boundary.
|
||||
expect(fsControl.syncedDirectories).toEqual([
|
||||
...parentChainToRoot(base),
|
||||
// bucket chain: every parent entry between the bucket and the boundary.
|
||||
objects,
|
||||
storageRoot,
|
||||
@@ -77,7 +89,7 @@ describe('local attachment store', () => {
|
||||
])
|
||||
})
|
||||
|
||||
it('retreats the durable boundary to the closest existing ancestor when the home directory does not exist yet', async () => {
|
||||
it('creates and persists a missing nested home directory against the filesystem root', async () => {
|
||||
const storageRoot = join(await root(), 'home', 'attachments', 'v1')
|
||||
|
||||
const ref = await saveImageFile(storageRoot, { data: PNG, mediaType: 'image/png' }, LIMITS)
|
||||
|
||||
Reference in New Issue
Block a user