feat: implement bounded LLM request recovery
This commit is contained in:
@@ -30,7 +30,7 @@ The `initialize` handshake reports a fixed server identity (`agentInfo: { name:
|
||||
| `session/load` | `ctx.agents.resume(...)` | reserves the id, verifies the persisted cwd, resumes, and replays user, assistant, and tool events |
|
||||
| `session/prompt` | `agent.send()` | supports ACP `text` and `resource_link` blocks; rejects image/audio/embedded resource and empty prompts; one in-flight prompt PER session (independent); settles on the OWNING turn's end (a turn that ends in `error` rejects the RPC) |
|
||||
| `session/cancel` | `agent.cancel()` | the queue-aware cancel: aborts a running step, clears queued + steering work, and drops a turn about to start, then settles the prompt `cancelled` — for ONLY that session (a cancel never touches another session's stream or prompt) |
|
||||
| `session/update` | `session/event` | streams user replay, assistant text/reasoning, and tool render intents |
|
||||
| `session/update` | `session/event` | streams user replay, assistant text/reasoning, retry/failure attempt markers, and tool render intents |
|
||||
| `elicitation/create` | `ctx.userInteraction.ask()` | maps `ask_user_question` questions to ACP form elicitations; option descriptions are shown in enum titles, `multi_select` uses ACP array enums, optionless requests use a required `custom` field, and a non-empty custom answer overrides any selected choice |
|
||||
| `session/request_permission` | `approval/request` listener | answers one-shot allow/reject requests for bridge-owned calls; foreign or call-less requests delegate and fail closed if unanswered — see "Permission prompts" |
|
||||
| `session/set_config_option` | agent-scoped request target / `ctx.permission.set()` | per-session provider+model and permission-preset switching over [session config options](https://agentclientprotocol.com/protocol/session-config-options) — see "Session config options" |
|
||||
@@ -47,6 +47,8 @@ When `ctx.permission` is composed, the bridge also advertises a `permission` sel
|
||||
|
||||
The shared [`ctx.tasks` runtime](../../tasks/tasks/) fences access to predictable task ids by the owning session; ACP sessions therefore cannot read or stop one another's background work.
|
||||
|
||||
ACP updates are append-only, so `llm/retry` emits a visible separator that marks preceding partial model output discarded before the next attempt streams. A terminal model-request failure emits the same discarded-output warning; replay derives both markers from the durable events.
|
||||
|
||||
## Per-session cwd
|
||||
|
||||
`session/new` records the request's absolute cwd in the session header. Before constructing an agent, `session/load` uses persisted metadata to require an absolute request cwd that matches the stored one. Bash defaults to that workspace; an explicit relative workdir resolves against it, and multiple sessions may use different workspaces. `additionalDirectories` remains unsupported.
|
||||
|
||||
@@ -30,6 +30,7 @@
|
||||
"@deepseek-ai/dsh-agent": "^0.0.1",
|
||||
"@deepseek-ai/dsh-bash": "^0.0.1",
|
||||
"@deepseek-ai/dsh-llm": "^0.0.1",
|
||||
"@deepseek-ai/dsh-llm-retry": "^0.0.1",
|
||||
"@deepseek-ai/dsh-permission": "^0.0.1",
|
||||
"@deepseek-ai/dsh-sandbox": "^0.0.1",
|
||||
"@deepseek-ai/dsh-session": "^0.0.1",
|
||||
@@ -50,6 +51,7 @@
|
||||
"@deepseek-ai/dsh-fs-policy": "workspace:^",
|
||||
"@deepseek-ai/dsh-invariants": "workspace:^",
|
||||
"@deepseek-ai/dsh-llm": "workspace:^",
|
||||
"@deepseek-ai/dsh-llm-retry": "workspace:^",
|
||||
"@deepseek-ai/dsh-permission": "workspace:^",
|
||||
"@deepseek-ai/dsh-sandbox": "workspace:^",
|
||||
"@deepseek-ai/dsh-session": "workspace:^",
|
||||
|
||||
@@ -44,6 +44,7 @@ import {
|
||||
} from '@agentclientprotocol/sdk'
|
||||
import type { ContentBlock, LlmCallConfig, LlmModelInfo, LlmProviderInfo } from '@deepseek-ai/dsh-llm'
|
||||
import { assertNever, CallId } from '@deepseek-ai/dsh-llm'
|
||||
import type {} from '@deepseek-ai/dsh-llm-retry'
|
||||
import type { Agent } from '@deepseek-ai/dsh-agent'
|
||||
import { SessionId } from '@deepseek-ai/dsh-session'
|
||||
// Side-effect type import: resolves `ctx.get('permission')` to the service.
|
||||
@@ -481,7 +482,7 @@ export function apply(ctx: Context, config: AcpConfig): void {
|
||||
reason: TurnEndReason,
|
||||
): void => {
|
||||
if (reason.kind === 'error') {
|
||||
inflight.reject(internalError(`turn failed: ${reason.message}`))
|
||||
inflight.reject(internalError(`turn failed: ${'failure' in reason ? reason.failure.message : reason.message}`))
|
||||
} else {
|
||||
inflight.resolve(turnEndToStopReason(reason))
|
||||
}
|
||||
@@ -1035,6 +1036,7 @@ function validateMcpServers(params: { mcpServers?: unknown[] }): void {
|
||||
* identical update stream from the same event log.
|
||||
*
|
||||
* - `assistant/chunk` text-delta/reasoning-delta → message/thought chunks
|
||||
* - `llm/retry` and terminal model failure → visible discarded-attempt markers
|
||||
* - `user/message` → `user_message_chunk` during load replay only — so a
|
||||
* loaded transcript reconstructs the USER side of each turn without echoing
|
||||
* a live `session/prompt` back to the client
|
||||
@@ -1081,6 +1083,13 @@ export function streamSessionEventUpdate(
|
||||
}
|
||||
return
|
||||
}
|
||||
case 'llm/retry': {
|
||||
const text = '\n\n[Previous model attempt discarded; retrying '
|
||||
+ `${event.data.retry}/${event.data.maxRetries} in ${event.data.delayMs}ms: `
|
||||
+ `${event.data.failure.message}]\n\n`
|
||||
notify({ sessionId, update: { sessionUpdate: 'agent_message_chunk', content: { type: 'text', text } } })
|
||||
return
|
||||
}
|
||||
case 'user/message': {
|
||||
if (!includeUserMessages) return
|
||||
// Replay the user's prompt so a loaded session shows both sides of each
|
||||
@@ -1108,7 +1117,16 @@ export function streamSessionEventUpdate(
|
||||
notify({ sessionId, update: { sessionUpdate: 'plan', ...todosToPlan(event.data.todos) } })
|
||||
return
|
||||
}
|
||||
// turn/step boundaries, context/message, steering,
|
||||
case 'turn/end': {
|
||||
if (event.data.reason.kind !== 'error') return
|
||||
const message = 'failure' in event.data.reason
|
||||
? event.data.reason.failure.message
|
||||
: event.data.reason.message
|
||||
const text = `\n\n[Model attempt failed; any partial output above is discarded: ${message}]\n\n`
|
||||
notify({ sessionId, update: { sessionUpdate: 'agent_message_chunk', content: { type: 'text', text } } })
|
||||
return
|
||||
}
|
||||
// non-error turn/step boundaries, context/message, steering,
|
||||
// assistant/message — no direct ACP client update.
|
||||
default:
|
||||
return
|
||||
|
||||
@@ -100,7 +100,7 @@ export function errorResponse(message: string): StreamChunk[] {
|
||||
return [
|
||||
{ type: 'block-start', index: 0, blockType: 'text' },
|
||||
{ type: 'text-delta', index: 0, text: 'partial' },
|
||||
{ type: 'finish', reason: { kind: 'error', message, code: 'PROVIDER_ERROR' } },
|
||||
{ type: 'finish', reason: { kind: 'error', failure: { message, code: 'PROVIDER_ERROR' } } },
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
@@ -65,6 +65,33 @@ describe('streamSessionEventUpdate', () => {
|
||||
.toEqual([])
|
||||
})
|
||||
|
||||
it('marks retry and terminal failure boundaries in the append-only update stream', () => {
|
||||
expect(updatesFor(evt('llm/retry', {
|
||||
turn: 1,
|
||||
step: 1,
|
||||
retry: 1,
|
||||
maxRetries: 2,
|
||||
delayMs: 500,
|
||||
failure: { message: 'backend busy', code: 'SERVER' },
|
||||
}))).toEqual([{
|
||||
sessionUpdate: 'agent_message_chunk',
|
||||
content: {
|
||||
type: 'text',
|
||||
text: '\n\n[Previous model attempt discarded; retrying 1/2 in 500ms: backend busy]\n\n',
|
||||
},
|
||||
}])
|
||||
expect(updatesFor(evt('turn/end', {
|
||||
turn: 1,
|
||||
reason: { kind: 'error', step: 2, failure: { message: 'still busy', code: 'SERVER' } },
|
||||
}))).toEqual([{
|
||||
sessionUpdate: 'agent_message_chunk',
|
||||
content: {
|
||||
type: 'text',
|
||||
text: '\n\n[Model attempt failed; any partial output above is discarded: still busy]\n\n',
|
||||
},
|
||||
}])
|
||||
})
|
||||
|
||||
it('maps tool/call to an in_progress tool_call with kind other and parsed rawInput (generic fallback, no presenter)', () => {
|
||||
const updates = updatesFor(evt('tool/call', { turn: 1, step: 1, callId: CallId('c1'), name: 'bash', arguments: '{"command":"ls"}' }))
|
||||
expect(updates).toEqual([{
|
||||
|
||||
@@ -49,6 +49,15 @@ describe('acp bridge — turn outcomes', () => {
|
||||
.rejects.toThrow(/turn failed: provider boom/)
|
||||
})
|
||||
|
||||
it('rejects an ordinary plugin turn failure through the same ACP boundary', async () => {
|
||||
harness = await makeBridgeHarness({ storageDir, script: [textResponse('must not run')] })
|
||||
harness.ctx.on('agent/pre-step', () => { throw new Error('plugin pre-step failed') })
|
||||
const sessionId = await newSession(harness)
|
||||
|
||||
await expect(harness.client.prompt({ sessionId, prompt: [{ type: 'text', text: 'go' }] }))
|
||||
.rejects.toThrow(/turn failed: plugin pre-step failed/)
|
||||
})
|
||||
|
||||
it('streams a tool call as tool_call then tool_call_update', async () => {
|
||||
harness = await makeBridgeHarness({
|
||||
storageDir,
|
||||
|
||||
@@ -20,6 +20,9 @@
|
||||
{
|
||||
"path": "../../llm/llm"
|
||||
},
|
||||
{
|
||||
"path": "../../llm/llm-retry"
|
||||
},
|
||||
{
|
||||
"path": "../../core/session"
|
||||
},
|
||||
|
||||
@@ -11,7 +11,7 @@ This package owns the terminal channel only. It injects `agents` and `userIntera
|
||||
| `welcome` | `ready.` | Banner printed before the first prompt |
|
||||
| `sessionId` | `main` | Exact agent/session identity driven by stdin and observed for EOF shutdown |
|
||||
|
||||
The plugin seeds display labels from the live agent registry, then tracks `agent/created` and `agent/disposed` so HMR and externally managed agents render consistently. While an initial exact identity is pending, it buffers nonblank input until `agent/session-start` and observes live `agent-loop/config-start-failed`; a matching failure drops queued lines, reports the loss, and lets piped EOF finish instead of hanging. The composing app must mount this front door before its config-created agent. Disposal closes readline and unregisters every listener/provider through Cordis effects.
|
||||
The plugin seeds display labels from the live agent registry, then tracks `agent/created` and `agent/disposed` so HMR and externally managed agents render consistently. While an initial exact identity is pending, it buffers nonblank input until `agent/session-start` and observes live `agent-loop/config-start-failed`; a matching failure drops queued lines, reports the loss, and lets piped EOF finish instead of hanging. When a composed retry policy closes a failed step, the append-only transcript inserts an explicit discarded-attempt marker before later chunks; terminal request failure marks any preceding partial output discarded. The composing app must mount this front door before its config-created agent. Disposal closes readline and unregisters every listener/provider through Cordis effects.
|
||||
|
||||
```yaml
|
||||
- id: stdio
|
||||
|
||||
@@ -25,6 +25,7 @@
|
||||
"@deepseek-ai/dsh-agent": "^0.0.1",
|
||||
"@deepseek-ai/dsh-agent-loop": "^0.0.1",
|
||||
"@deepseek-ai/dsh-llm": "^0.0.1",
|
||||
"@deepseek-ai/dsh-llm-retry": "^0.0.1",
|
||||
"@deepseek-ai/dsh-session": "^0.0.1",
|
||||
"@deepseek-ai/dsh-user-interaction": "^0.0.1",
|
||||
"cordis": "^4.0.0-rc.7"
|
||||
@@ -42,6 +43,7 @@
|
||||
"@deepseek-ai/dsh-agent": "workspace:^",
|
||||
"@deepseek-ai/dsh-agent-loop": "workspace:^",
|
||||
"@deepseek-ai/dsh-llm": "workspace:^",
|
||||
"@deepseek-ai/dsh-llm-retry": "workspace:^",
|
||||
"@deepseek-ai/dsh-session": "workspace:^",
|
||||
"@deepseek-ai/dsh-user-interaction": "workspace:^",
|
||||
"cordis": "^4.0.0-rc.7"
|
||||
|
||||
@@ -16,6 +16,7 @@ import type { Context } from 'cordis'
|
||||
import z from 'schemastery'
|
||||
import type { Agent } from '@deepseek-ai/dsh-agent'
|
||||
import type {} from '@deepseek-ai/dsh-agent-loop'
|
||||
import type {} from '@deepseek-ai/dsh-llm-retry'
|
||||
import { SessionId } from '@deepseek-ai/dsh-session'
|
||||
import {
|
||||
UserInteractionError,
|
||||
@@ -119,6 +120,10 @@ export function createStdioChat(ctx: Context, config: Config, runtime: StdioRunt
|
||||
// append order keeps `inReasoning` transitions deterministic across chunk and
|
||||
// boundary events.
|
||||
let inReasoning = false
|
||||
const resetReasoning = (): void => {
|
||||
if (inReasoning) output.write('\x1B[0m')
|
||||
inReasoning = false
|
||||
}
|
||||
ctx.on('session/event', (session, event) => {
|
||||
if (event.type === 'assistant/chunk') {
|
||||
const { chunk } = event.data
|
||||
@@ -135,22 +140,31 @@ export function createStdioChat(ctx: Context, config: Config, runtime: StdioRunt
|
||||
} else if (event.type === 'turn/start') {
|
||||
const label = target?.session === session ? 'main' : session.id
|
||||
output.write(`\n[${label} turn ${event.data.turn}] `)
|
||||
} else if (event.type === 'llm/retry') {
|
||||
resetReasoning()
|
||||
output.write(
|
||||
`\n [previous model attempt discarded; retry ${event.data.retry}/${event.data.maxRetries}`
|
||||
+ ` in ${event.data.delayMs}ms: ${event.data.failure.message}]\n `,
|
||||
)
|
||||
} else if (event.type === 'turn/end') {
|
||||
if (inReasoning) output.write('\x1B[0m')
|
||||
inReasoning = false
|
||||
resetReasoning()
|
||||
if (event.data.reason.kind === 'error') {
|
||||
const message = 'failure' in event.data.reason
|
||||
? event.data.reason.failure.message
|
||||
: event.data.reason.message
|
||||
output.write(`\n [model attempt failed; any partial output above is discarded: ${message}]`)
|
||||
}
|
||||
output.write('\n> ')
|
||||
} else if (event.type === 'tool/call') {
|
||||
const { name: toolName, arguments: args } = event.data
|
||||
if (inReasoning) output.write('\x1B[0m')
|
||||
inReasoning = false
|
||||
resetReasoning()
|
||||
output.write(`\n [tool call] ${toolName}(${args})`)
|
||||
} else if (event.type === 'tool/result') {
|
||||
const { content } = event.data
|
||||
const text = content.filter(block => block.type === 'text').map(block => block.text).join('')
|
||||
output.write(`\n [tool result] ${text}\n `)
|
||||
} else if (event.type === 'todo/write') {
|
||||
if (inReasoning) output.write('\x1B[0m')
|
||||
inReasoning = false
|
||||
resetReasoning()
|
||||
const glyph = (status: string): string =>
|
||||
status === 'completed' ? '[x]' : status === 'in_progress' ? '[~]' : '[ ]'
|
||||
const lines = event.data.todos.map(todo => ` ${glyph(todo.status)} ${todo.content}`).join('\n')
|
||||
|
||||
@@ -290,6 +290,51 @@ describe('createStdioChat rendering', () => {
|
||||
expect(out.text()).toContain('\x1B[2mmid\x1B[0m')
|
||||
})
|
||||
|
||||
it('marks failed partial output at retry and terminal failure boundaries', async () => {
|
||||
const { ctx, out } = await setup()
|
||||
const session = makeSession('main')
|
||||
ctx.emit('session/event', session, chunkEvent({ type: 'reasoning-delta', index: 0, text: 'partial' }))
|
||||
ctx.emit('session/event', session, {
|
||||
type: 'llm/retry',
|
||||
seq: 1,
|
||||
time: 0,
|
||||
data: {
|
||||
turn: 1,
|
||||
step: 1,
|
||||
retry: 1,
|
||||
maxRetries: 2,
|
||||
delayMs: 500,
|
||||
failure: { message: 'backend busy', code: 'SERVER' },
|
||||
},
|
||||
})
|
||||
ctx.emit('session/event', session, {
|
||||
type: 'turn/end',
|
||||
seq: 3,
|
||||
time: 0,
|
||||
data: { turn: 2, reason: { kind: 'error', step: 1, message: 'loop defect' } },
|
||||
})
|
||||
ctx.emit('session/event', session, chunkEvent({ type: 'text-delta', index: 0, text: 'also partial' }))
|
||||
ctx.emit('session/event', session, {
|
||||
type: 'turn/end',
|
||||
seq: 2,
|
||||
time: 0,
|
||||
data: {
|
||||
turn: 1,
|
||||
reason: { kind: 'error', step: 2, failure: { message: 'still busy', code: 'SERVER' } },
|
||||
},
|
||||
})
|
||||
|
||||
expect(out.text()).toContain(
|
||||
'\x1B[2mpartial\x1B[0m\n [previous model attempt discarded; retry 1/2 in 500ms: backend busy]',
|
||||
)
|
||||
expect(out.text()).toContain(
|
||||
'also partial\n [model attempt failed; any partial output above is discarded: still busy]\n> ',
|
||||
)
|
||||
expect(out.text()).toContain(
|
||||
'[model attempt failed; any partial output above is discarded: loop defect]\n> ',
|
||||
)
|
||||
})
|
||||
|
||||
it('drops the target object on agent/disposed', async () => {
|
||||
const { ctx, out } = await setup()
|
||||
const agent = makeAgent('main')
|
||||
|
||||
@@ -26,6 +26,9 @@
|
||||
{
|
||||
"path": "../../llm/llm"
|
||||
},
|
||||
{
|
||||
"path": "../../llm/llm-retry"
|
||||
},
|
||||
{
|
||||
"path": "../user-interaction"
|
||||
}
|
||||
|
||||
@@ -6,7 +6,7 @@ The implemented [TUI feature Agent Note](../../../.agents/notes/implemented/feat
|
||||
|
||||
This package owns interactive terminal presentation and input only. It injects `agents`, `tools`, and `userInteraction`, then drives an agent created or resumed by app or developer code. Agent lifecycle, persistence, and the model-facing [`ask_user_question`](../tool-ask-user/README.md) tool remain separate composition entries.
|
||||
|
||||
The TUI rebuilds resumed history from the active session surface, renders Markdown responses and reasoning, applies each tool's `presentCall` / `presentResult` intent to terminal, diff, or generic cards, keeps the latest `todo/write` plan above the editor, and presents `ctx.userInteraction` questions as keyboard-driven overlays. Surface replacement events rebuild the transcript so compacted history does not reappear.
|
||||
The TUI rebuilds resumed history from the active session surface, renders Markdown responses and reasoning, applies each tool's `presentCall` / `presentResult` intent to terminal, diff, or generic cards, keeps the latest `todo/write` plan above the editor, and presents `ctx.userInteraction` questions as keyboard-driven overlays. A durable `llm/retry` event retracts the failed step's live chunks and renders the scheduled retry count, delay, and failure in the transcript; success, exhaustion, and cancellation then settle through ordinary session events. The footer totals each logged model step's usage once, including failed attempts, while treating committed-message usage as a fallback for logs without a usage chunk. Surface replacement events rebuild the transcript so compacted history does not reappear.
|
||||
|
||||
Before model output, session events, tool presenters, questions, configuration, or diagnostics reach pi-tui's ANSI-aware renderers or the terminal title, the TUI renders C0 and C1 controls other than line feeds as visible `\xNN` text. Those sources cannot add terminal control sequences; the TUI and pi-tui retain ownership of terminal rendering and styling.
|
||||
|
||||
|
||||
@@ -25,6 +25,7 @@
|
||||
"@deepseek-ai/dsh-agent": "^0.0.1",
|
||||
"@deepseek-ai/dsh-agent-loop": "^0.0.1",
|
||||
"@deepseek-ai/dsh-llm": "^0.0.1",
|
||||
"@deepseek-ai/dsh-llm-retry": "^0.0.1",
|
||||
"@deepseek-ai/dsh-session": "^0.0.1",
|
||||
"@deepseek-ai/dsh-tools": "^0.0.1",
|
||||
"@deepseek-ai/dsh-user-interaction": "^0.0.1",
|
||||
@@ -39,6 +40,7 @@
|
||||
"@deepseek-ai/dsh-agent": "workspace:^",
|
||||
"@deepseek-ai/dsh-agent-loop": "workspace:^",
|
||||
"@deepseek-ai/dsh-llm": "workspace:^",
|
||||
"@deepseek-ai/dsh-llm-retry": "workspace:^",
|
||||
"@deepseek-ai/dsh-session": "workspace:^",
|
||||
"@deepseek-ai/dsh-system-prompt": "workspace:^",
|
||||
"@deepseek-ai/dsh-tool-cordis": "workspace:^",
|
||||
|
||||
@@ -35,7 +35,8 @@ import type { Context } from 'cordis'
|
||||
import z from 'schemastery'
|
||||
import type { Agent, AgentStatus } from '@deepseek-ai/dsh-agent'
|
||||
import type {} from '@deepseek-ai/dsh-agent-loop'
|
||||
import type { ContentBlock, StreamChunk } from '@deepseek-ai/dsh-llm'
|
||||
import type { ContentBlock, StreamChunk, TokenUsage } from '@deepseek-ai/dsh-llm'
|
||||
import type {} from '@deepseek-ai/dsh-llm-retry'
|
||||
import { SessionId, type Session, type SessionEvent, type TodoItem } from '@deepseek-ai/dsh-session'
|
||||
import type {
|
||||
FileDiff,
|
||||
@@ -612,15 +613,38 @@ function formatCwd(cwd: string | undefined): string {
|
||||
return displayText(cwd)
|
||||
}
|
||||
|
||||
function sessionTokens(session: Session): { input: number; output: number } {
|
||||
let input = 0
|
||||
let output = 0
|
||||
for (const event of session.events) {
|
||||
if (event.type !== 'assistant/message' || event.data.usage === undefined) continue
|
||||
input += event.data.usage.inputTokens
|
||||
output += event.data.usage.outputTokens
|
||||
interface SessionTokenTotals {
|
||||
input: number
|
||||
output: number
|
||||
readonly byStep: Map<string, TokenUsage>
|
||||
}
|
||||
|
||||
function recordTokenUsage(totals: SessionTokenTotals, turn: number, step: number, usage: TokenUsage): void {
|
||||
const key = `${turn}:${step}`
|
||||
const previous = totals.byStep.get(key)
|
||||
if (previous !== undefined) {
|
||||
totals.input -= previous.inputTokens
|
||||
totals.output -= previous.outputTokens
|
||||
}
|
||||
return { input, output }
|
||||
totals.byStep.set(key, usage)
|
||||
totals.input += usage.inputTokens
|
||||
totals.output += usage.outputTokens
|
||||
}
|
||||
|
||||
function recordEventUsage(totals: SessionTokenTotals, event: SessionEvent): void {
|
||||
if (event.type === 'assistant/chunk' && event.data.chunk.type === 'usage') {
|
||||
recordTokenUsage(totals, event.data.turn, event.data.step, event.data.chunk.usage)
|
||||
} else if (event.type === 'assistant/message' && event.data.usage !== undefined) {
|
||||
recordTokenUsage(totals, event.data.turn, event.data.step, event.data.usage)
|
||||
}
|
||||
}
|
||||
|
||||
function sessionTokens(session: Session): SessionTokenTotals {
|
||||
const totals: SessionTokenTotals = { input: 0, output: 0, byStep: new Map() }
|
||||
for (const event of session.events) {
|
||||
recordEventUsage(totals, event)
|
||||
}
|
||||
return totals
|
||||
}
|
||||
|
||||
class FooterComponent implements Component {
|
||||
@@ -899,6 +923,14 @@ export function createTuiChat(
|
||||
return card
|
||||
}
|
||||
|
||||
const clearStreaming = (): void => {
|
||||
if (streaming === undefined) return
|
||||
const index = chat.children.indexOf(streaming)
|
||||
/* v8 ignore next -- streaming is assigned only after the same component is added, and every removal clears it. */
|
||||
if (index >= 0) chat.children.splice(index, 1)
|
||||
streaming = undefined
|
||||
}
|
||||
|
||||
const renderEvent = (event: SessionEvent, options: { addHistory: boolean; renderChunks: boolean }): void => {
|
||||
switch (event.type) {
|
||||
case 'user/message': {
|
||||
@@ -941,15 +973,19 @@ export function createTuiChat(
|
||||
}
|
||||
break
|
||||
case 'assistant/message': {
|
||||
if (streaming !== undefined) {
|
||||
const index = chat.children.indexOf(streaming)
|
||||
if (index >= 0) chat.children.splice(index, 1)
|
||||
streaming = undefined
|
||||
}
|
||||
clearStreaming()
|
||||
const component = new AssistantMessageComponent(event.data.content, showReasoning, palette, mdTheme)
|
||||
if (component.children.length > 0) chat.addChild(component)
|
||||
break
|
||||
}
|
||||
case 'llm/retry': {
|
||||
clearStreaming()
|
||||
appendNotice(
|
||||
`Retrying model request (${event.data.retry}/${event.data.maxRetries}) in ${event.data.delayMs}ms: ${event.data.failure.message}`,
|
||||
'warning',
|
||||
)
|
||||
break
|
||||
}
|
||||
case 'tool/call':
|
||||
chat.addChild(new Spacer(1))
|
||||
chat.addChild(parsedTool(event))
|
||||
@@ -970,9 +1006,13 @@ export function createTuiChat(
|
||||
todo.update(event.data.todos)
|
||||
break
|
||||
case 'turn/end':
|
||||
clearStreaming()
|
||||
if (event.data.reason.kind === 'error') {
|
||||
const key = `${event.data.turn}:${event.data.reason.step}`
|
||||
if (!liveErrors.delete(key)) appendNotice(event.data.reason.message, 'error')
|
||||
const message = 'failure' in event.data.reason
|
||||
? event.data.reason.failure.message
|
||||
: event.data.reason.message
|
||||
if (!liveErrors.delete(key)) appendNotice(message, 'error')
|
||||
} else if (event.data.reason.kind === 'aborted') {
|
||||
appendNotice(event.data.reason.reason ?? 'Turn cancelled.', 'warning')
|
||||
} else if (event.data.reason.kind === 'max-tokens') {
|
||||
@@ -1245,10 +1285,7 @@ export function createTuiChat(
|
||||
|
||||
const disposeSessionEvents = ctx.on('session/event', (session, event) => {
|
||||
if (session !== agent.session) return
|
||||
if (event.type === 'assistant/message' && event.data.usage !== undefined) {
|
||||
tokens.input += event.data.usage.inputTokens
|
||||
tokens.output += event.data.usage.outputTokens
|
||||
}
|
||||
recordEventUsage(tokens, event)
|
||||
if ('surfaceOp' in event && typeof event.surfaceOp === 'object') {
|
||||
rebuildTranscript(false)
|
||||
return
|
||||
|
||||
@@ -120,10 +120,11 @@ export function appendAssistant(
|
||||
session: Session,
|
||||
content: ContentBlock[],
|
||||
usage?: { inputTokens: number; outputTokens: number },
|
||||
position: { turn: number; step: number } = { turn: 1, step: 0 },
|
||||
): void {
|
||||
session.append('assistant/message', {
|
||||
turn: 1,
|
||||
step: 0,
|
||||
turn: position.turn,
|
||||
step: position.step,
|
||||
provenance: { provider: 'mock', model: 'deepseek-v4-flash' },
|
||||
content,
|
||||
...usage === undefined ? {} : { usage },
|
||||
|
||||
48
packages/ui/tui/tests/snapshots/retry-cancelled.expected.txt
Normal file
48
packages/ui/tui/tests/snapshots/retry-cancelled.expected.txt
Normal file
@@ -0,0 +1,48 @@
|
||||
terminal 96x36 buffer=normal length=36 base=0 viewport=0
|
||||
lifecycle started=1 stopped=0 progress=inactive
|
||||
title "DSH snapshot"
|
||||
cursor hidden column=1 viewportRow=15 bufferRow=15
|
||||
buffer
|
||||
0| "╭──────────────────────────────────────────────────────────────────────────────────────────────╮"
|
||||
style 0-95 fg=bright-blue
|
||||
1| "│ DEEPSEEK HARNESS │"
|
||||
style 0-0 fg=bright-blue
|
||||
style 2-9 fg=bright-blue bold
|
||||
style 11-17 bold
|
||||
style 95-95 fg=bright-blue
|
||||
2| "│ Snapshot agent ready. │"
|
||||
style 0-0 fg=bright-blue
|
||||
style 2-22 fg=bright-black
|
||||
style 95-95 fg=bright-blue
|
||||
3| "│ deepseek-v4-flash • main-session │"
|
||||
style 0-0 fg=bright-blue
|
||||
style 2-35 dim
|
||||
style 95-95 fg=bright-blue
|
||||
4| "╰──────────────────────────────────────────────────────────────────────────────────────────────╯"
|
||||
style 0-95 fg=bright-blue
|
||||
5| <blank>
|
||||
6| "▌ "
|
||||
style 0-0 fg=bright-blue
|
||||
7| "▌ You "
|
||||
style 0-0 fg=bright-blue
|
||||
style 2-4 fg=bright-blue bold
|
||||
8| "▌ Start then cancel. "
|
||||
style 0-0 fg=bright-blue
|
||||
9| "▌ "
|
||||
style 0-0 fg=bright-blue
|
||||
10| <blank>
|
||||
11| " Retrying model request (1/2) in 1000ms: temporary transport failure "
|
||||
style 1-67 fg=yellow
|
||||
12| <blank>
|
||||
13| " cancelled during retry delay "
|
||||
style 1-28 fg=yellow
|
||||
14| "────────────────────────────────────────────────────────────────────────────────────────────────"
|
||||
style 0-95 dim
|
||||
15| " "
|
||||
style 1-1 inverse
|
||||
16| "────────────────────────────────────────────────────────────────────────────────────────────────"
|
||||
style 0-95 dim
|
||||
17| "/workspace/project ↑0 ↓0 idle reasoning:on tools:compact"
|
||||
style 0-24 dim
|
||||
style 63-95 dim
|
||||
18-35| <blank>
|
||||
45
packages/ui/tui/tests/snapshots/retry-exhausted.expected.txt
Normal file
45
packages/ui/tui/tests/snapshots/retry-exhausted.expected.txt
Normal file
@@ -0,0 +1,45 @@
|
||||
terminal 96x36 buffer=normal length=36 base=0 viewport=0
|
||||
lifecycle started=1 stopped=0 progress=inactive
|
||||
title "DSH snapshot"
|
||||
cursor hidden column=1 viewportRow=13 bufferRow=13
|
||||
buffer
|
||||
0| "╭──────────────────────────────────────────────────────────────────────────────────────────────╮"
|
||||
style 0-95 fg=bright-blue
|
||||
1| "│ DEEPSEEK HARNESS │"
|
||||
style 0-0 fg=bright-blue
|
||||
style 2-9 fg=bright-blue bold
|
||||
style 11-17 bold
|
||||
style 95-95 fg=bright-blue
|
||||
2| "│ Snapshot agent ready. │"
|
||||
style 0-0 fg=bright-blue
|
||||
style 2-22 fg=bright-black
|
||||
style 95-95 fg=bright-blue
|
||||
3| "│ deepseek-v4-flash • main-session │"
|
||||
style 0-0 fg=bright-blue
|
||||
style 2-35 dim
|
||||
style 95-95 fg=bright-blue
|
||||
4| "╰──────────────────────────────────────────────────────────────────────────────────────────────╯"
|
||||
style 0-95 fg=bright-blue
|
||||
5| <blank>
|
||||
6| "▌ "
|
||||
style 0-0 fg=bright-blue
|
||||
7| "▌ You "
|
||||
style 0-0 fg=bright-blue
|
||||
style 2-4 fg=bright-blue bold
|
||||
8| "▌ Let the bounded policy exhaust. "
|
||||
style 0-0 fg=bright-blue
|
||||
9| "▌ "
|
||||
style 0-0 fg=bright-blue
|
||||
10| <blank>
|
||||
11| " provider still unavailable "
|
||||
style 1-26 fg=red
|
||||
12| "────────────────────────────────────────────────────────────────────────────────────────────────"
|
||||
style 0-95 dim
|
||||
13| " "
|
||||
style 1-1 inverse
|
||||
14| "────────────────────────────────────────────────────────────────────────────────────────────────"
|
||||
style 0-95 dim
|
||||
15| "/workspace/project ↑0 ↓0 idle reasoning:on tools:compact"
|
||||
style 0-24 dim
|
||||
style 63-95 dim
|
||||
16-35| <blank>
|
||||
49
packages/ui/tui/tests/snapshots/retry-recovered.expected.txt
Normal file
49
packages/ui/tui/tests/snapshots/retry-recovered.expected.txt
Normal file
@@ -0,0 +1,49 @@
|
||||
terminal 96x36 buffer=normal length=36 base=0 viewport=0
|
||||
lifecycle started=1 stopped=0 progress=inactive
|
||||
title "DSH snapshot"
|
||||
cursor hidden column=1 viewportRow=16 bufferRow=16
|
||||
buffer
|
||||
0| "╭──────────────────────────────────────────────────────────────────────────────────────────────╮"
|
||||
style 0-95 fg=bright-blue
|
||||
1| "│ DEEPSEEK HARNESS │"
|
||||
style 0-0 fg=bright-blue
|
||||
style 2-9 fg=bright-blue bold
|
||||
style 11-17 bold
|
||||
style 95-95 fg=bright-blue
|
||||
2| "│ Snapshot agent ready. │"
|
||||
style 0-0 fg=bright-blue
|
||||
style 2-22 fg=bright-black
|
||||
style 95-95 fg=bright-blue
|
||||
3| "│ deepseek-v4-flash • main-session │"
|
||||
style 0-0 fg=bright-blue
|
||||
style 2-35 dim
|
||||
style 95-95 fg=bright-blue
|
||||
4| "╰──────────────────────────────────────────────────────────────────────────────────────────────╯"
|
||||
style 0-95 fg=bright-blue
|
||||
5| <blank>
|
||||
6| "▌ "
|
||||
style 0-0 fg=bright-blue
|
||||
7| "▌ You "
|
||||
style 0-0 fg=bright-blue
|
||||
style 2-4 fg=bright-blue bold
|
||||
8| "▌ Recover this request. "
|
||||
style 0-0 fg=bright-blue
|
||||
9| "▌ "
|
||||
style 0-0 fg=bright-blue
|
||||
10| <blank>
|
||||
11| " Retrying model request (1/2) in 500ms: provider rate limit "
|
||||
style 1-58 fg=yellow
|
||||
12| <blank>
|
||||
13| " Assistant "
|
||||
style 1-9 fg=bright-magenta bold
|
||||
14| " Recovered on the next bounded attempt. "
|
||||
15| "────────────────────────────────────────────────────────────────────────────────────────────────"
|
||||
style 0-95 dim
|
||||
16| " "
|
||||
style 1-1 inverse
|
||||
17| "────────────────────────────────────────────────────────────────────────────────────────────────"
|
||||
style 0-95 dim
|
||||
18| "/workspace/project ↑0 ↓0 idle reasoning:on tools:compact"
|
||||
style 0-24 dim
|
||||
style 63-95 dim
|
||||
19-35| <blank>
|
||||
45
packages/ui/tui/tests/snapshots/retry-scheduled.expected.txt
Normal file
45
packages/ui/tui/tests/snapshots/retry-scheduled.expected.txt
Normal file
@@ -0,0 +1,45 @@
|
||||
terminal 96x36 buffer=normal length=36 base=0 viewport=0
|
||||
lifecycle started=1 stopped=0 progress=inactive
|
||||
title "DSH snapshot"
|
||||
cursor hidden column=1 viewportRow=13 bufferRow=13
|
||||
buffer
|
||||
0| "╭──────────────────────────────────────────────────────────────────────────────────────────────╮"
|
||||
style 0-95 fg=bright-blue
|
||||
1| "│ DEEPSEEK HARNESS │"
|
||||
style 0-0 fg=bright-blue
|
||||
style 2-9 fg=bright-blue bold
|
||||
style 11-17 bold
|
||||
style 95-95 fg=bright-blue
|
||||
2| "│ Snapshot agent ready. │"
|
||||
style 0-0 fg=bright-blue
|
||||
style 2-22 fg=bright-black
|
||||
style 95-95 fg=bright-blue
|
||||
3| "│ deepseek-v4-flash • main-session │"
|
||||
style 0-0 fg=bright-blue
|
||||
style 2-35 dim
|
||||
style 95-95 fg=bright-blue
|
||||
4| "╰──────────────────────────────────────────────────────────────────────────────────────────────╯"
|
||||
style 0-95 fg=bright-blue
|
||||
5| <blank>
|
||||
6| "▌ "
|
||||
style 0-0 fg=bright-blue
|
||||
7| "▌ You "
|
||||
style 0-0 fg=bright-blue
|
||||
style 2-4 fg=bright-blue bold
|
||||
8| "▌ Recover this request. "
|
||||
style 0-0 fg=bright-blue
|
||||
9| "▌ "
|
||||
style 0-0 fg=bright-blue
|
||||
10| <blank>
|
||||
11| " Retrying model request (1/2) in 500ms: provider rate limit "
|
||||
style 1-58 fg=yellow
|
||||
12| "────────────────────────────────────────────────────────────────────────────────────────────────"
|
||||
style 0-95 dim
|
||||
13| " "
|
||||
style 1-1 inverse
|
||||
14| "────────────────────────────────────────────────────────────────────────────────────────────────"
|
||||
style 0-95 dim
|
||||
15| "/workspace/project ↑0 ↓0 idle reasoning:on tools:compact"
|
||||
style 0-24 dim
|
||||
style 63-95 dim
|
||||
16-35| <blank>
|
||||
@@ -4,6 +4,7 @@ import { fileURLToPath } from 'node:url'
|
||||
import { afterAll, describe, expect, it } from 'vitest'
|
||||
import type { Context } from 'cordis'
|
||||
import { CallId, type ContentBlock } from '@deepseek-ai/dsh-llm'
|
||||
import type {} from '@deepseek-ai/dsh-llm-retry'
|
||||
import type { Session } from '@deepseek-ai/dsh-session'
|
||||
import SystemPrompt from '@deepseek-ai/dsh-system-prompt'
|
||||
import ToolRegistry, { type ToolDefinition, type ToolResultView } from '@deepseek-ai/dsh-tools'
|
||||
@@ -24,6 +25,10 @@ const REFRESHING = process.env.DSH_SNAPSHOT === 'refresh'
|
||||
|
||||
const CHECKPOINTS = [
|
||||
'conversation-streaming',
|
||||
'retry-scheduled',
|
||||
'retry-recovered',
|
||||
'retry-cancelled',
|
||||
'retry-exhausted',
|
||||
'code-mode-pending',
|
||||
'dynamic-workflow-pending',
|
||||
'cordis-tools-pending',
|
||||
@@ -222,6 +227,82 @@ describe('TUI terminal-state snapshots', () => {
|
||||
await disposeSnapshot(harness)
|
||||
})
|
||||
|
||||
it('pins failed-stream retraction, scheduled retry, and eventual success', async () => {
|
||||
const harness = await setupSnapshot()
|
||||
await renderAfter(harness, () => {
|
||||
appendUser(harness.session, 'Recover this request.')
|
||||
harness.session.append('assistant/chunk', {
|
||||
turn: 1,
|
||||
step: 1,
|
||||
chunk: { type: 'text-delta', index: 0, text: 'discarded partial output' },
|
||||
})
|
||||
harness.session.append('llm/retry', {
|
||||
turn: 1,
|
||||
step: 1,
|
||||
retry: 1,
|
||||
maxRetries: 2,
|
||||
delayMs: 500,
|
||||
failure: { message: 'provider rate limit', code: 'RATE_LIMIT', status: 429 },
|
||||
})
|
||||
})
|
||||
await checkpoint('retry-scheduled', harness.terminal, { includeScrollback: true })
|
||||
|
||||
await renderAfter(harness, () => {
|
||||
harness.session.append('assistant/message', {
|
||||
turn: 1,
|
||||
step: 2,
|
||||
provenance: { provider: 'mock', model: 'deepseek-v4-flash' },
|
||||
content: [{ type: 'text', text: 'Recovered on the next bounded attempt.' }],
|
||||
}, { surfaceOp: 'append' })
|
||||
harness.session.append('turn/end', { turn: 1, reason: { kind: 'completed' } })
|
||||
})
|
||||
await checkpoint('retry-recovered', harness.terminal, { includeScrollback: true })
|
||||
await disposeSnapshot(harness)
|
||||
})
|
||||
|
||||
it('pins cancellation during a scheduled retry delay', async () => {
|
||||
const harness = await setupSnapshot()
|
||||
await renderAfter(harness, () => {
|
||||
appendUser(harness.session, 'Start then cancel.')
|
||||
harness.session.append('llm/retry', {
|
||||
turn: 1,
|
||||
step: 1,
|
||||
retry: 1,
|
||||
maxRetries: 2,
|
||||
delayMs: 1_000,
|
||||
failure: { message: 'temporary transport failure', code: 'TRANSPORT' },
|
||||
})
|
||||
harness.session.append('turn/end', {
|
||||
turn: 1,
|
||||
reason: { kind: 'aborted', reason: 'cancelled during retry delay' },
|
||||
})
|
||||
})
|
||||
await checkpoint('retry-cancelled', harness.terminal, { includeScrollback: true })
|
||||
await disposeSnapshot(harness)
|
||||
})
|
||||
|
||||
it('pins terminal exhaustion after retracting a failed partial stream', async () => {
|
||||
const harness = await setupSnapshot()
|
||||
await renderAfter(harness, () => {
|
||||
appendUser(harness.session, 'Let the bounded policy exhaust.')
|
||||
harness.session.append('assistant/chunk', {
|
||||
turn: 1,
|
||||
step: 3,
|
||||
chunk: { type: 'text-delta', index: 0, text: 'discarded terminal partial output' },
|
||||
})
|
||||
harness.session.append('turn/end', {
|
||||
turn: 1,
|
||||
reason: {
|
||||
kind: 'error',
|
||||
step: 3,
|
||||
failure: { message: 'provider still unavailable', code: 'SERVER', status: 503 },
|
||||
},
|
||||
})
|
||||
})
|
||||
await checkpoint('retry-exhausted', harness.terminal, { includeScrollback: true })
|
||||
await disposeSnapshot(harness)
|
||||
})
|
||||
|
||||
it('pins Code Mode run_code with its production presenter', async () => {
|
||||
const harness = await setupSnapshot({ configureContext: configureAdvancedTools })
|
||||
const call = {
|
||||
|
||||
@@ -7,6 +7,7 @@ import AgentRegistry, { type Agent } from '@deepseek-ai/dsh-agent'
|
||||
import SessionStore, { SessionId } from '@deepseek-ai/dsh-session'
|
||||
import type { ToolDefinition } from '@deepseek-ai/dsh-tools'
|
||||
import UserInteractionService from '@deepseek-ai/dsh-user-interaction'
|
||||
import type {} from '@deepseek-ai/dsh-llm-retry'
|
||||
import {
|
||||
createTuiChat,
|
||||
mountTui,
|
||||
@@ -244,7 +245,12 @@ describe('pi-tui chat lifecycle and transcript', () => {
|
||||
expect(result.terminal.output).toContain('live thought')
|
||||
result.terminal.send('\x12')
|
||||
await tick()
|
||||
appendAssistant(result.session, [{ type: 'text', text: 'final live answer' }], { inputTokens: 500, outputTokens: 8 })
|
||||
appendAssistant(
|
||||
result.session,
|
||||
[{ type: 'text', text: 'final live answer' }],
|
||||
{ inputTokens: 500, outputTokens: 8 },
|
||||
{ turn: 2, step: 0 },
|
||||
)
|
||||
await tick()
|
||||
|
||||
expect(result.terminal.output).toContain('Working')
|
||||
@@ -276,6 +282,68 @@ describe('pi-tui chat lifecycle and transcript', () => {
|
||||
expect(result.terminal.drainInput).toHaveBeenCalledWith(100, 20)
|
||||
})
|
||||
|
||||
it('counts failed and recovered request usage once per step', async () => {
|
||||
const result = await setup()
|
||||
result.session.append('assistant/chunk', {
|
||||
turn: 1,
|
||||
step: 1,
|
||||
chunk: { type: 'usage', usage: { inputTokens: 10, outputTokens: 2 } },
|
||||
})
|
||||
result.session.append('llm/retry', {
|
||||
turn: 1,
|
||||
step: 1,
|
||||
retry: 1,
|
||||
maxRetries: 2,
|
||||
delayMs: 500,
|
||||
failure: { message: 'temporary', code: 'SERVER' },
|
||||
})
|
||||
result.session.append('assistant/chunk', {
|
||||
turn: 1,
|
||||
step: 2,
|
||||
chunk: { type: 'usage', usage: { inputTokens: 7, outputTokens: 3 } },
|
||||
})
|
||||
appendAssistant(
|
||||
result.session,
|
||||
[{ type: 'text', text: 'recovered' }],
|
||||
{ inputTokens: 7, outputTokens: 3 },
|
||||
{ turn: 1, step: 2 },
|
||||
)
|
||||
await tick()
|
||||
|
||||
expect(result.terminal.output).toContain('↑17 ↓5')
|
||||
await dispose(result)
|
||||
})
|
||||
|
||||
it('retracts a failed live stream and renders its durable retry status', async () => {
|
||||
const result = await setup()
|
||||
result.session.append('assistant/chunk', {
|
||||
turn: 1,
|
||||
step: 1,
|
||||
chunk: { type: 'text-delta', index: 0, text: 'discarded partial answer' },
|
||||
})
|
||||
result.session.append('llm/retry', {
|
||||
turn: 1,
|
||||
step: 1,
|
||||
retry: 1,
|
||||
maxRetries: 2,
|
||||
delayMs: 500,
|
||||
failure: { message: 'rate limited', code: 'RATE_LIMIT', status: 429 },
|
||||
})
|
||||
result.session.append('llm/retry', {
|
||||
turn: 1,
|
||||
step: 2,
|
||||
retry: 2,
|
||||
maxRetries: 2,
|
||||
delayMs: 1_000,
|
||||
failure: { message: 'failed before chunks', code: 'SERVER', status: 503 },
|
||||
})
|
||||
await tick()
|
||||
|
||||
expect(result.terminal.output).toContain('Retrying model request (1/2) in 500ms: rate limited')
|
||||
expect(result.terminal.output).toContain('Retrying model request (2/2) in 1000ms: failed before chunks')
|
||||
await dispose(result)
|
||||
})
|
||||
|
||||
it('renders the ANSI palette and every markdown/content style', async () => {
|
||||
const result = await setup({
|
||||
config: { color: true },
|
||||
@@ -453,10 +521,15 @@ describe('pi-tui chat lifecycle and transcript', () => {
|
||||
events.session.append('turn/end', { turn: 6, reason: { kind: 'max-tokens' } })
|
||||
events.session.append('turn/end', { turn: 7, reason: { kind: 'rejected', reason: 'policy' } })
|
||||
events.session.append('turn/end', { turn: 8, reason: { kind: 'interrupted' } })
|
||||
events.session.append('turn/end', {
|
||||
turn: 9,
|
||||
reason: { kind: 'error', step: 1, failure: { message: 'structured provider failure', code: 'SERVER' } },
|
||||
})
|
||||
events.ctx.emit('agent/disposed', events.agent)
|
||||
await tick()
|
||||
expect(events.terminal.output).toContain('live failure')
|
||||
expect(events.terminal.output).toContain('durable failure')
|
||||
expect(events.terminal.output).toContain('structured provider failure')
|
||||
expect(events.terminal.output).toContain('stopped')
|
||||
expect(events.terminal.output).toContain('output-token limit')
|
||||
expect(events.terminal.output).toContain('Turn rejected')
|
||||
|
||||
@@ -26,6 +26,9 @@
|
||||
{
|
||||
"path": "../../llm/llm"
|
||||
},
|
||||
{
|
||||
"path": "../../llm/llm-retry"
|
||||
},
|
||||
{
|
||||
"path": "../../core/tools"
|
||||
},
|
||||
|
||||
Reference in New Issue
Block a user