From 0dc66d5b77b012230595f2022c45b416608686fe Mon Sep 17 00:00:00 2001 From: Hypatia May Date: Mon, 27 Jul 2026 13:05:59 +0800 Subject: [PATCH] fix(web): support large search corpora (round 4) --- .../2026-07-27-web-session-search.i18n.yaml | 4 +- .../feature/2026-07-27-web-session-search.md | 2 +- .../2026-07-27-web-session-search.zh.md | 2 +- packages/host/apiproxy/README.i18n.yaml | 4 +- packages/host/apiproxy/README.md | 2 +- packages/host/apiproxy/README.zh.md | 2 +- packages/host/apiproxy/src/api-proxy.ts | 69 ++++++++++++------- .../apiproxy/tests/api-proxy-search.spec.ts | 50 ++++++++++++-- 8 files changed, 96 insertions(+), 39 deletions(-) diff --git a/.agents/notes/implemented/feature/2026-07-27-web-session-search.i18n.yaml b/.agents/notes/implemented/feature/2026-07-27-web-session-search.i18n.yaml index 1dfb037886..721f0f9483 100644 --- a/.agents/notes/implemented/feature/2026-07-27-web-session-search.i18n.yaml +++ b/.agents/notes/implemented/feature/2026-07-27-web-session-search.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write .agents/notes/implemented/feature/2026-07-27-web-session-search.md -2026-07-27-web-session-search.md: 8791d02249cc310e768712b3967dcfead3a950e0 -2026-07-27-web-session-search.zh.md: 737f91fbe2c00ac5aa75fb6c30b8b22f80b0854f +2026-07-27-web-session-search.md: e3219f865aa3f13cdb7e806570ef6134dd5bd448 +2026-07-27-web-session-search.zh.md: 27d0efee5d218c2e737fb7378d3fd71c6b13e4ce diff --git a/.agents/notes/implemented/feature/2026-07-27-web-session-search.md b/.agents/notes/implemented/feature/2026-07-27-web-session-search.md index 8791d02249..e3219f865a 100644 --- a/.agents/notes/implemented/feature/2026-07-27-web-session-search.md +++ b/.agents/notes/implemented/feature/2026-07-27-web-session-search.md @@ -12,7 +12,7 @@ The Web sidebar exposes session titles and Workspace membership but cannot retri The shared Web/headless composition mounts [`@deepseek-ai/dsh-session-query-sqlite`](../../../../packages/session-query/session-query-sqlite/README.md) with an in-memory database. Each service instance owns one connection-private index, preserving the SQLite backend's single-owner contract across parallel CLI or Web invocations without leaving process-scoped derived files behind. The database starts empty; the first content query of each invocation lazily reconciles live and persisted sessions. It remains a disposable derived index, separate from canonical JSONL persistence. -The host gateway exposes `session.search` through the existing typed RPC stack. It derives the authorization set from the same visible summaries as `session.list`, passes those ids to `ctx.sessionQuery.searchSessions`, and restricts indexed matches to current-surface `user/message`, `assistant/message`, and `steering/message` events. The response is one page of at most 20 session ids and snippets; `hasMore` tells the UI to ask for a narrower query rather than exposing pagination. The carrier signal cancels superseded work, including the persistence listing and bounded batches of cold-session metadata stats that build the visibility set. A missing query service or an indexing/query failure remains a business error and does not mutate the canonical session store. +The host gateway exposes `session.search` through the existing typed RPC stack. It derives the authorization set from the same visible summaries as `session.list`, asks `ctx.sessionQuery.searchSessions` for globally ranked current-surface `user/message`, `assistant/message`, and `steering/message` matches, and consumes provider pages until it has 20 authorized sessions plus one lookahead or exhausts the stream. Every hit's session id, best-match session id, surface, and event type are revalidated before its snippet leaves the Host. Keeping the potentially large authorization set out of SQLite bindings avoids the portable variable ceiling while preserving global ranking. The response remains one bounded page; `hasMore` tells the UI to ask for a narrower query rather than exposing pagination. The carrier signal cancels superseded work, including persistence listing, bounded batches of cold-session metadata stats, and each provider page. A missing query service or an indexing/query failure remains a business error and does not mutate the canonical session store. [`WorkspaceBrowser`](../../../../packages/client/ui-workspace/README.md) keeps metadata and content search deliberately separate. A non-blank query immediately computes case-insensitive title and Workspace substring matches from the Session list, starts a 250 ms debounced content request, aborts the preceding request when the query changes, and ignores stale completions. It merges local matches first in recency order with backend-ranked content-only matches, deduplicates by session id, and renders a flat list regardless of the normal grouping mode. Each row shows the title, Workspace, and an available one-line snippet. Selecting a row opens the Session only and preserves the query; it does not navigate to an exact event. diff --git a/.agents/notes/implemented/feature/2026-07-27-web-session-search.zh.md b/.agents/notes/implemented/feature/2026-07-27-web-session-search.zh.md index 737f91fbe2..27d0efee5d 100644 --- a/.agents/notes/implemented/feature/2026-07-27-web-session-search.zh.md +++ b/.agents/notes/implemented/feature/2026-07-27-web-session-search.zh.md @@ -12,7 +12,7 @@ Web 侧边栏会展示会话标题及其 Workspace 归属，但无法根据只 Web 与 headless 共用的组合会使用内存数据库挂载 [`@deepseek-ai/dsh-session-query-sqlite`](../../../../packages/session-query/session-query-sqlite/README.md)。每个服务实例都独占一个连接私有索引，因此并行 CLI 或 Web 调用可维持 SQLite 后端的单一所有者契约，又不会留下进程级派生文件。数据库从空状态启动；每次调用的首次内容查询会惰性对齐实时会话与持久化会话。它仍是与规范 JSONL 持久化相互独立的可丢弃派生索引。 -宿主网关通过现有的类型化 RPC 栈公开 `session.search`。它根据 `session.list` 使用的同一组可见摘要推导授权集合，将这些 id 传给 `ctx.sessionQuery.searchSessions`，并将索引匹配限制为当前 surface 中的 `user/message`、`assistant/message` 和 `steering/message` 事件。响应只包含一页，最多 20 个会话 id 及其摘要片段；`hasMore` 会指示 UI 提示用户缩小查询范围，而不是公开分页能力。载体信号会取消已被取代的工作，包括持久化列表枚举和构建可见集合时分批受限执行的冷会话元数据 stat。查询服务缺失或索引、查询失败仍作为业务错误处理，不会修改规范会话存储。 +宿主网关通过现有的类型化 RPC 栈公开 `session.search`。它根据 `session.list` 使用的同一组可见摘要推导授权集合，向 `ctx.sessionQuery.searchSessions` 请求全局排序后的当前 surface `user/message`、`assistant/message` 和 `steering/message` 匹配项，并持续消费提供方分页，直到获得 20 个已授权会话及一个前瞻项，或结果流耗尽。每个命中的会话 id、最佳匹配会话 id、surface 和事件类型都会经过重新校验，其 snippet 才能离开宿主。将可能很大的授权集合排除在 SQLite 绑定之外，可避开可移植变量上限，同时保持全局排序。响应仍只有一个有界页面；`hasMore` 会指示 UI 提示用户缩小查询范围，而不是公开分页能力。载体信号会取消已被取代的工作，包括持久化列表枚举、分批受限执行的冷会话元数据 stat，以及每一页提供方搜索。查询服务缺失或索引、查询失败仍作为业务错误处理，不会修改规范会话存储。 [`WorkspaceBrowser`](../../../../packages/client/ui-workspace/README.md) 有意将元数据搜索与内容搜索保持独立。非空白查询会立即从会话列表中计算不区分大小写的标题和 Workspace 子串匹配，在 250 ms 防抖后发起内容请求，在查询变化时中止前一请求，并忽略陈旧的完成结果。它先按新近程度排列本地匹配，再合并由后端排序且仅匹配内容的结果，按会话 id 去重；无论常规分组模式如何，最终都渲染为扁平列表。每一行显示标题、Workspace，并在存在时显示一行摘要片段。选择某一行只会打开对应会话，并保留查询条件；不会跳转至确切事件。 diff --git a/packages/host/apiproxy/README.i18n.yaml b/packages/host/apiproxy/README.i18n.yaml index 7d323dbeea..5079e05965 100644 --- a/packages/host/apiproxy/README.i18n.yaml +++ b/packages/host/apiproxy/README.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write packages/host/apiproxy/README.md -README.md: deb1073c2ff9e7a533e595ee3f5537e649660e5b -README.zh.md: e3c521d5f6a09414d087e3fb142852e6d3eb0cb7 +README.md: b9f0fcd8506afda733774868d78fe6e851b4fe2a +README.zh.md: c57990b029b8d1e143396bb13718dbed165e2b47 diff --git a/packages/host/apiproxy/README.md b/packages/host/apiproxy/README.md index deb1073c2f..b9f0fcd850 100644 --- a/packages/host/apiproxy/README.md +++ b/packages/host/apiproxy/README.md @@ -14,7 +14,7 @@ The mux stream projects the latest log-backed title as a validated `session/titl Workspace and Session lists are separate reconnect baselines. `workspace.create` creates a unique name or adopts an existing directory, `session.create` accepts an optional preallocated Session id, and `host/workspace-changed` plus `host/session-added` carry committed increments in either arrival order. `SessionSummary.blank` and the `host/session-added` frame carry the derived zero-events bit: clients hide blank sessions and reuse them per workspace, flip blank on the first `host/session-status(running:true)`, and treat `session.list` as the reconnect authority; cold summaries are never blank because lazy persistence keeps never-appended sessions out of `list()`. -`session.search` is a bounded content-search projection over the sessions visible through `session.list`. The gateway passes only those session ids and current-surface user, assistant, and steering messages to the optional `ctx.sessionQuery` service, returns at most 20 session/snippet pairs plus a refine-query bit, and forwards the carrier request signal for cancellation. A deployment without the service, or a failed index/query operation, returns an `internal` business error so clients can retain metadata-only matches. +`session.search` is a bounded content-search projection over the sessions visible through `session.list`. The gateway asks the optional `ctx.sessionQuery` service for globally ranked current-surface user, assistant, and steering matches, pages that stream until it has at most 20 visible session/snippet pairs plus one lookahead, and revalidates every hit against the list-derived authorization set before returning it. Keeping the authorization set in Host memory avoids SQLite's variable ceiling for large valid corpora without weakening visibility or ranking. The carrier request signal cancels persistence listing, cold-summary collection, and every search page. A deployment without the service, or a failed index/query operation, returns an `internal` business error so clients can retain metadata-only matches. The `command.*` and `skill.*` domains expose the host command registry and skill catalog to clients. Every method addresses one session's agent by `sessionId` (a served session always has an Agent; `command.*` resumes cold sessions through the same path as `session.*`, while `skill.list` resolves the project root from the session header without touching the Agent registry). `command.execute` runs a slash-command line host-side and returns a detached result; the carrier's request signal cancels the running handler. `host/commands-changed` is the catalog invalidation frame: clients refetch `command.list` instead of diffing. diff --git a/packages/host/apiproxy/README.zh.md b/packages/host/apiproxy/README.zh.md index e3c521d5f6..c57990b029 100644 --- a/packages/host/apiproxy/README.zh.md +++ b/packages/host/apiproxy/README.zh.md @@ -14,7 +14,7 @@ mux 流会在每个已附加会话的订阅基线之后，以及对应的实时 Workspace 列表与 Session 列表是相互独立的重连基线。`workspace.create` 会创建唯一名称或接纳现有目录，`session.create` 接受可选的预分配 Session id，`host/workspace-changed` 与 `host/session-added` 则以任意到达顺序携带已提交的增量。`SessionSummary.blank` 与 `host/session-added` 帧携带派生的零事件位：客户端隐藏空白会话并按 workspace 复用它们，在首个 `host/session-status(running:true)` 时翻转 blank，并以 `session.list` 作为重连权威；冷会话摘要永远不是空白——惰性持久化让从未追加过事件的会话根本不出现在 `list()` 中。 -`session.search` 是以 `session.list` 所列会话为范围的有界内容搜索投影。网关仅将这些会话 id 以及当前表层中的 user、assistant 和 steering（中途引导）消息传给可选的 `ctx.sessionQuery` 服务，返回至多 20 个会话／snippet 对和一个提示细化查询的标志位，并转发载体请求信号以支持取消。部署若未挂载该服务，或索引／查询操作失败，都会返回 `internal` 业务错误，以便客户端保留仅基于元数据的匹配项。 +`session.search` 是以 `session.list` 所列会话为范围的有界内容搜索投影。网关向可选的 `ctx.sessionQuery` 服务请求全局排序后的当前 surface user、assistant 和 steering（中途引导）匹配项，对该结果流分页，直到获得至多 20 个可见会话／snippet 对及一个前瞻项，并在返回前依据从列表推导的授权集合重新校验每个命中。将授权集合保留在宿主内存中，可在不削弱可见性或排序的前提下避开有效大型语料库的 SQLite 变量上限。载体请求信号可取消持久化列表枚举、冷会话摘要收集和每一页搜索。部署若未挂载该服务，或索引／查询操作失败，都会返回 `internal` 业务错误，以便客户端保留仅基于元数据的匹配项。 `command.*` 与 `skill.*` 领域向客户端暴露宿主命令注册表和技能目录。每个方法都通过 `sessionId` 寻址一个会话的 Agent（被服务的会话必有 Agent；`command.*` 经由与 `session.*` 相同的路径恢复冷会话，而 `skill.list` 从会话头解析项目根目录，不触碰 Agent 注册表）。`command.execute` 在宿主侧运行一条斜杠命令行并返回脱耦结果；载体的请求信号可取消正在运行的处理器。`host/commands-changed` 是目录失效帧：客户端重新拉取 `command.list` 而不是做差分。 diff --git a/packages/host/apiproxy/src/api-proxy.ts b/packages/host/apiproxy/src/api-proxy.ts index 367479e698..c9329feea8 100644 --- a/packages/host/apiproxy/src/api-proxy.ts +++ b/packages/host/apiproxy/src/api-proxy.ts @@ -11,7 +11,7 @@ import type { Agent, AgentMessage, AgentMessageId, AgentStatus } from '@deepseek import type { ContentBlock, MessageSource } from '@deepseek-ai/dsh-llm' import type { JsonValue, Session, SessionEvent, SessionHeader, SessionId } from '@deepseek-ai/dsh-session' import type { SessionPersistence } from '@deepseek-ai/dsh-session-persistence' -import { SessionQueryError } from '@deepseek-ai/dsh-session-query' +import { SessionQueryError, type SessionSearchCursor } from '@deepseek-ai/dsh-session-query' import { foldSessionTitle } from '@deepseek-ai/dsh-session-title' import type { Workspace, WorkspaceRecord } from '@deepseek-ai/dsh-workspace' import { @@ -641,30 +641,51 @@ export function createApiProxy(ctx: Context, defaults: ApiProxyDefaults): ApiPro if (isAborted(signal)) return cancelled() if (visible.length === 0) return ok(request, { items: [], hasMore: false }) const visibleIds = new Set(visible.map(item => item.sessionId)) - const page = await sessionQuery.searchSessions({ - query: request.payload.query, - sessionFilters: [{ kind: 'id', values: [...visibleIds] }], - eventFilters: [ - { kind: 'type', values: ['user/message', 'assistant/message', 'steering/message'] }, - { kind: 'surface', values: ['current'] }, - ], - limit: SESSION_SEARCH_LIMIT, - }, { signal }) - if (isAborted(signal)) return cancelled() - // The filters are the authorization boundary. Re-check the complete - // provider provenance before emitting its snippet so a backend - // regression cannot pair an allowed header with excluded content. - const authorized = page.items.filter(hit => - visibleIds.has(hit.header.id) - && hit.bestMatch.sessionId === hit.header.id - && hit.bestMatch.surface === 'current' - && MESSAGE_TYPES.has(hit.bestMatch.type)) + const authorized: SessionSearchItem[] = [] + const acceptedIds = new Set() + const seenCursors = new Set() + let cursor: SessionSearchCursor | undefined + while (authorized.length <= SESSION_SEARCH_LIMIT) { + if (isAborted(signal)) return cancelled() + const page = await sessionQuery.searchSessions({ + query: request.payload.query, + eventFilters: [ + { kind: 'type', values: ['user/message', 'assistant/message', 'steering/message'] }, + { kind: 'surface', values: ['current'] }, + ], + limit: SESSION_SEARCH_LIMIT, + ...cursor === undefined ? {} : { cursor }, + }, { signal }) + if (isAborted(signal)) return cancelled() + // Host visibility is the authorization boundary. Consume the + // provider's globally ranked stream rather than binding every + // visible id into one SQLite statement, then re-check complete + // provenance before emitting any snippet. + for (const hit of page.items) { + if ( + !visibleIds.has(hit.header.id) + || hit.bestMatch.sessionId !== hit.header.id + || hit.bestMatch.surface !== 'current' + || !MESSAGE_TYPES.has(hit.bestMatch.type) + || acceptedIds.has(hit.header.id) + ) continue + acceptedIds.add(hit.header.id) + authorized.push({ + sessionId: hit.header.id, + snippet: hit.bestMatch.snippet, + }) + if (authorized.length > SESSION_SEARCH_LIMIT) break + } + if (authorized.length > SESSION_SEARCH_LIMIT || page.nextCursor === undefined) break + if (seenCursors.has(page.nextCursor)) { + throw new Error('session search provider repeated a continuation cursor') + } + seenCursors.add(page.nextCursor) + cursor = page.nextCursor + } return ok(request, { - items: authorized.slice(0, SESSION_SEARCH_LIMIT).map(hit => ({ - sessionId: hit.header.id, - snippet: hit.bestMatch.snippet, - })), - hasMore: page.nextCursor !== undefined || authorized.length > SESSION_SEARCH_LIMIT, + items: authorized.slice(0, SESSION_SEARCH_LIMIT), + hasMore: authorized.length > SESSION_SEARCH_LIMIT, }) } catch (error: unknown) { if ( diff --git a/packages/host/apiproxy/tests/api-proxy-search.spec.ts b/packages/host/apiproxy/tests/api-proxy-search.spec.ts index 99f40a5949..5138398fe3 100644 --- a/packages/host/apiproxy/tests/api-proxy-search.spec.ts +++ b/packages/host/apiproxy/tests/api-proxy-search.spec.ts @@ -107,7 +107,6 @@ describe('session.search', () => { }, }, ], - nextCursor: 'more' as never, })) ctx.provide('sessionQuery', { searchSessions } as never) const api = createApiProxy(ctx, defaults) @@ -119,7 +118,7 @@ describe('session.search', () => { ok: true, value: { items: [{ sessionId: 'cold', snippet: 'the matching answer' }], - hasMore: true, + hasMore: false, }, }) expect(searchSessions).toHaveBeenCalledOnce() @@ -129,7 +128,6 @@ describe('session.search', () => { ] expect(query).toEqual({ query: 'matching answer', - sessionFilters: [{ kind: 'id', values: ['live', 'cold'] }], eventFilters: [ { kind: 'type', @@ -179,7 +177,6 @@ describe('session.search', () => { withBestMatch(2, { type: 'tool/result' }), withBestMatch(3, { type: 'steering/message', snippet: 'allowed snippet' }), ], - nextCursor: 'more', }), } as never) @@ -192,19 +189,25 @@ describe('session.search', () => { ok: true, value: { items: [{ sessionId: 'visible', snippet: 'allowed snippet' }], - hasMore: true, + hasMore: false, }, }) }) - it('enforces the 20-item Host boundary even if a provider overproduces', async () => { + it('pages the globally ranked stream until the 20-item Host boundary is known', async () => { const ctx = await baseContext() const items = Array.from({ length: 21 }, (_, index) => hit(`visible-${index}`, index)) for (const item of items) { ctx.sessions.create(item.header.id, { meta: item.header }) } + const searchSessions = vi.fn() + .mockResolvedValueOnce({ + items: [hit('hidden-ranked-first'), ...items.slice(0, 19)], + nextCursor: 'page-2', + }) + .mockResolvedValueOnce({ items: items.slice(19) }) ctx.provide('sessionQuery', { - searchSessions: () => Promise.resolve({ items }), + searchSessions, } as never) const response = await createApiProxy(ctx, defaults).sessions.search( request('match'), @@ -218,6 +221,39 @@ describe('session.search', () => { if (!response.result.ok) throw new Error('unreachable') expect(response.result.value.items).toHaveLength(20) expect(response.result.value.items.at(-1)?.sessionId).toBe('visible-19') + expect(searchSessions).toHaveBeenCalledTimes(2) + expect(searchSessions.mock.calls[1]?.[0]).toMatchObject({ cursor: 'page-2' }) + }) + + it('keeps visibility sets above SQLite variable limits out of provider bindings', async () => { + const ctx = await baseContext() + const cold = Array.from( + { length: 32_751 }, + (_, index) => header(`cold-${index}`, `/cold-${index}`), + ) + ctx.provide('sessionPersistence', { + list: () => Promise.resolve(cold), + locate: () => undefined, + } as never) + const searchSessions = vi.fn(() => Promise.resolve({ + items: [hit('cold-32750')], + })) + ctx.provide('sessionQuery', { searchSessions } as never) + + const response = await createApiProxy(ctx, defaults).sessions.search( + request('large corpus'), + new AbortController().signal, + ) + + expect(response.result).toEqual({ + ok: true, + value: { + items: [{ sessionId: 'cold-32750', snippet: 'match 0' }], + hasMore: false, + }, + }) + expect(searchSessions).toHaveBeenCalledOnce() + expect(searchSessions.mock.calls[0]?.[0]).not.toHaveProperty('sessionFilters') }) it('propagates cancellation through visible-session collection and stops cold-summary work', async () => {