fix(preset): align minimal agent with RL composition
This commit is contained in:
Yichen Jiang
@@ -1,137 +0,0 @@
|
||||
import { writeFile } from 'node:fs/promises'
|
||||
import { join } from 'node:path'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import { afterAll, beforeAll, describe, expect, it } from 'vitest'
|
||||
import type { AgentHandle } from '@deepseek-ai/dsh-agent'
|
||||
import { CallId, createUserMessage } from '@deepseek-ai/dsh-llm'
|
||||
import { SessionId } from '@deepseek-ai/dsh-session'
|
||||
import { assertFixtureInventory, launchWebScaffold, type WebScaffold } from './scaffold.ts'
|
||||
|
||||
const CORE_WEB_OVERLAY = fileURLToPath(new URL('../../cli/config/core-web.cordis.yml', import.meta.url))
|
||||
const SNAPSHOT_DIR = fileURLToPath(new URL('./snapshots/core-web-profile', import.meta.url))
|
||||
const FIXTURE = join(SNAPSHOT_DIR, 'session.jsonl')
|
||||
const PROMPT = 'Reply exactly CORE_WEB_REQUEST_OK and stop.'
|
||||
|
||||
describe('core Web profile', () => {
|
||||
let scaffold: WebScaffold
|
||||
let agentHandle: AgentHandle
|
||||
|
||||
beforeAll(async () => {
|
||||
const systemPrompt = process.env.DSH_SYSTEM_PROMPT
|
||||
Reflect.deleteProperty(process.env, 'DSH_SYSTEM_PROMPT')
|
||||
try {
|
||||
scaffold = await launchWebScaffold({ extraOverlayPath: CORE_WEB_OVERLAY, replayFixture: FIXTURE })
|
||||
} finally {
|
||||
if (systemPrompt !== undefined) process.env.DSH_SYSTEM_PROMPT = systemPrompt
|
||||
}
|
||||
agentHandle = await scaffold.ctx.agents.create({
|
||||
sessionId: SessionId('core-web-profile-smoke'),
|
||||
meta: { cwd: scaffold.workspaceCwd },
|
||||
agentOptions: { provider: 'deepseek-official', model: 'deepseek-v4-flash' },
|
||||
})
|
||||
})
|
||||
|
||||
afterAll(async () => {
|
||||
const failures: unknown[] = []
|
||||
await agentHandle?.dispose().catch((error: unknown) => failures.push(error))
|
||||
await scaffold?.close().catch((error: unknown) => failures.push(error))
|
||||
if (failures.length === 1) throw failures[0]
|
||||
if (failures.length > 1) throw new AggregateError(failures, 'core Web profile smoke teardown failed')
|
||||
})
|
||||
|
||||
it('sends the RL prompt and tool schemas through a real request, then executes both tools', async () => {
|
||||
agentHandle.agent.followup(createUserMessage({
|
||||
content: [{ type: 'text', text: PROMPT }],
|
||||
source: { kind: 'user' },
|
||||
}))
|
||||
await agentHandle.agent.whenIdle()
|
||||
|
||||
const requestHeader = agentHandle.agent.session.requestHeader()
|
||||
if (requestHeader === undefined) throw new Error('the core Web agent issued no model request')
|
||||
|
||||
const seedPath = join(scaffold.workspaceCwd, 'profile-smoke.txt')
|
||||
await writeFile(seedPath, 'CORE_WEB_EDITOR_OK\n')
|
||||
const signal = new AbortController().signal
|
||||
const bash = await scaffold.ctx.tools.execute({
|
||||
signal,
|
||||
callId: CallId('core-web-bash-smoke'),
|
||||
name: 'bash',
|
||||
arguments: { command: "printf 'CORE_WEB_BASH_OK\\n'" },
|
||||
agent: agentHandle.agent,
|
||||
})
|
||||
const editor = await scaffold.ctx.tools.execute({
|
||||
signal,
|
||||
callId: CallId('core-web-editor-smoke'),
|
||||
name: 'str_replace_editor',
|
||||
arguments: { command: 'view', path: seedPath },
|
||||
agent: agentHandle.agent,
|
||||
})
|
||||
|
||||
const text = (result: typeof bash): string => result.content
|
||||
.filter(block => block.type === 'text')
|
||||
.map(block => block.text)
|
||||
.join('')
|
||||
.replaceAll(scaffold.workspaceCwd, '{{cwd}}')
|
||||
.trimEnd()
|
||||
|
||||
expect({
|
||||
prompt: requestHeader.system,
|
||||
tools: requestHeader.tools?.map(tool => tool.name),
|
||||
bash: text(bash),
|
||||
editor: text(editor),
|
||||
}).toMatchInlineSnapshot(`
|
||||
{
|
||||
"bash": "CORE_WEB_BASH_OK",
|
||||
"editor": "Here's the content of {{cwd}}/profile-smoke.txt with line numbers (which has a total of 2 lines):
|
||||
1 CORE_WEB_EDITOR_OK
|
||||
2",
|
||||
"prompt": "You are a helpful software engineer assistant.",
|
||||
"tools": [
|
||||
"bash",
|
||||
"str_replace_editor",
|
||||
],
|
||||
}
|
||||
`)
|
||||
expect(requestHeader.tools).toEqual(scaffold.ctx.tools.schemas(agentHandle.agent))
|
||||
|
||||
const entries = [...scaffold.ctx.loader.entries()]
|
||||
expect(entries.find(entry => entry.options.id === 'persistent-bash')?.fiber).toBeDefined()
|
||||
expect(entries.find(entry => entry.options.id === 'pty-local')?.fiber).toBeDefined()
|
||||
expect(entries.find(entry => entry.options.id === 'str-replace-editor')?.fiber).toBeDefined()
|
||||
expect(entries.find(entry => entry.options.id === 'web-runtime')?.fiber).toBeDefined()
|
||||
expect(entries.find(entry => entry.options.id === 'workspace-context')?.fiber).toBeUndefined()
|
||||
await assertFixtureInventory(SNAPSHOT_DIR, ['session.jsonl'])
|
||||
})
|
||||
|
||||
it('uses DSH_SYSTEM_PROMPT as the complete prompt when configured', async () => {
|
||||
const previous = process.env.DSH_SYSTEM_PROMPT
|
||||
process.env.DSH_SYSTEM_PROMPT = 'RL prompt override'
|
||||
let overrideScaffold: WebScaffold | undefined
|
||||
let overrideAgent: AgentHandle | undefined
|
||||
try {
|
||||
overrideScaffold = await launchWebScaffold({ extraOverlayPath: CORE_WEB_OVERLAY, replayFixture: FIXTURE })
|
||||
overrideAgent = await overrideScaffold.ctx.agents.create({
|
||||
sessionId: SessionId('core-web-profile-override'),
|
||||
meta: { cwd: overrideScaffold.workspaceCwd },
|
||||
agentOptions: { provider: 'deepseek-official', model: 'deepseek-v4-flash' },
|
||||
})
|
||||
overrideAgent.agent.followup(createUserMessage({
|
||||
content: [{ type: 'text', text: PROMPT }],
|
||||
source: { kind: 'user' },
|
||||
}))
|
||||
await overrideAgent.agent.whenIdle()
|
||||
expect(overrideAgent.agent.session.requestHeader()?.system).toBe('RL prompt override')
|
||||
} finally {
|
||||
try {
|
||||
await overrideAgent?.dispose()
|
||||
} finally {
|
||||
try {
|
||||
await overrideScaffold?.close()
|
||||
} finally {
|
||||
if (previous === undefined) Reflect.deleteProperty(process.env, 'DSH_SYSTEM_PROMPT')
|
||||
else process.env.DSH_SYSTEM_PROMPT = previous
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
})
|
||||
115
apps/web/tests/minimal-preset.snapshot.ts
Normal file
115
apps/web/tests/minimal-preset.snapshot.ts
Normal file
@@ -0,0 +1,115 @@
|
||||
import { mkdir, writeFile } from 'node:fs/promises'
|
||||
import { join } from 'node:path'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import { afterAll, beforeAll, describe, expect, it } from 'vitest'
|
||||
import type { AgentHandle } from '@deepseek-ai/dsh-agent'
|
||||
import { CallId, createUserMessage } from '@deepseek-ai/dsh-llm'
|
||||
import { SessionId } from '@deepseek-ai/dsh-session'
|
||||
import type {} from '@deepseek-ai/dsh-agent-presets'
|
||||
import type {} from '@deepseek-ai/dsh-system-prompt'
|
||||
import { assertFixtureInventory, launchWebScaffold, type WebScaffold } from './scaffold.ts'
|
||||
|
||||
const SNAPSHOT_DIR = fileURLToPath(new URL('./snapshots/minimal-preset', import.meta.url))
|
||||
const FIXTURE = join(SNAPSHOT_DIR, 'session.jsonl')
|
||||
const PROMPT = 'Reply exactly MINIMAL_PRESET_REQUEST_OK and stop.'
|
||||
|
||||
describe('minimal agent preset', () => {
|
||||
let scaffold: WebScaffold
|
||||
let agentHandle: AgentHandle
|
||||
let disposeInjectedPrompt: () => void
|
||||
|
||||
beforeAll(async () => {
|
||||
scaffold = await launchWebScaffold({ replayFixture: FIXTURE })
|
||||
disposeInjectedPrompt = scaffold.ctx.systemPrompt.section({
|
||||
name: 'test:injected-prompt',
|
||||
order: 999,
|
||||
text: 'THIS TEXT MUST NOT REACH THE MODEL.',
|
||||
})
|
||||
agentHandle = await scaffold.ctx.agents.create({
|
||||
sessionId: SessionId('minimal-preset-smoke'),
|
||||
meta: { cwd: scaffold.workspaceCwd, agentPreset: 'minimal' },
|
||||
agentOptions: { provider: 'deepseek-official', model: 'deepseek-v4-flash' },
|
||||
setup: agentCtx => scaffold.ctx.agentPresets.mount(agentCtx, 'minimal').then(() => undefined),
|
||||
})
|
||||
})
|
||||
|
||||
afterAll(async () => {
|
||||
const failures: unknown[] = []
|
||||
await agentHandle?.dispose().catch((error: unknown) => failures.push(error))
|
||||
try {
|
||||
disposeInjectedPrompt?.()
|
||||
} catch (error: unknown) {
|
||||
failures.push(error)
|
||||
}
|
||||
await scaffold?.close().catch((error: unknown) => failures.push(error))
|
||||
if (failures.length === 1) throw failures[0]
|
||||
if (failures.length > 1) throw new AggregateError(failures, 'minimal preset smoke teardown failed')
|
||||
})
|
||||
|
||||
it('sends the exact RL prompt and schemas, then executes the persistent shell and editor', async () => {
|
||||
agentHandle.agent.followup(createUserMessage({
|
||||
content: [{ type: 'text', text: PROMPT }],
|
||||
source: { kind: 'user' },
|
||||
}))
|
||||
await agentHandle.agent.whenIdle()
|
||||
|
||||
const requestHeader = agentHandle.agent.session.requestHeader()
|
||||
if (requestHeader === undefined) throw new Error('the minimal agent issued no model request')
|
||||
|
||||
const stateDir = join(scaffold.workspaceCwd, 'persistent-state')
|
||||
await mkdir(stateDir)
|
||||
const signal = new AbortController().signal
|
||||
await scaffold.ctx.tools.execute({
|
||||
signal,
|
||||
callId: CallId('minimal-bash-state-setup'),
|
||||
name: 'bash',
|
||||
arguments: { command: `cd ${JSON.stringify(stateDir)} && export DSH_MINIMAL_STATE=PERSISTED` },
|
||||
agent: agentHandle.agent,
|
||||
})
|
||||
const bash = await scaffold.ctx.tools.execute({
|
||||
signal,
|
||||
callId: CallId('minimal-bash-state-read'),
|
||||
name: 'bash',
|
||||
arguments: { command: 'printf \'%s:%s\n\' "$DSH_MINIMAL_STATE" "$PWD"' },
|
||||
agent: agentHandle.agent,
|
||||
})
|
||||
const seedPath = join(scaffold.workspaceCwd, 'preset-smoke.txt')
|
||||
await writeFile(seedPath, 'MINIMAL_EDITOR_OK\n')
|
||||
const editor = await scaffold.ctx.tools.execute({
|
||||
signal,
|
||||
callId: CallId('minimal-editor-smoke'),
|
||||
name: 'str_replace_editor',
|
||||
arguments: { command: 'view', path: seedPath },
|
||||
agent: agentHandle.agent,
|
||||
})
|
||||
|
||||
const text = (result: typeof bash): string => result.content
|
||||
.filter(block => block.type === 'text')
|
||||
.map(block => block.text)
|
||||
.join('')
|
||||
.replaceAll(scaffold.workspaceCwd, '{{cwd}}')
|
||||
.trimEnd()
|
||||
|
||||
expect({
|
||||
prompt: requestHeader.system,
|
||||
tools: requestHeader.tools?.map(tool => tool.name),
|
||||
bash: text(bash),
|
||||
editor: text(editor),
|
||||
}).toMatchInlineSnapshot(`
|
||||
{
|
||||
"bash": "PERSISTED:{{cwd}}/persistent-state",
|
||||
"editor": "Here's the content of {{cwd}}/preset-smoke.txt with line numbers (which has a total of 2 lines):
|
||||
1 MINIMAL_EDITOR_OK
|
||||
2",
|
||||
"prompt": "You are a helpful software engineer assistant.",
|
||||
"tools": [
|
||||
"bash",
|
||||
"str_replace_editor",
|
||||
],
|
||||
}
|
||||
`)
|
||||
expect(requestHeader.tools?.toSorted((left, right) => left.name.localeCompare(right.name)))
|
||||
.toEqual(scaffold.ctx.tools.schemas(agentHandle.agent).toSorted((left, right) => left.name.localeCompare(right.name)))
|
||||
await assertFixtureInventory(SNAPSHOT_DIR, ['session.jsonl'])
|
||||
})
|
||||
})
|
||||
@@ -1,7 +1,7 @@
|
||||
{"type":"session","version":0,"id":"{{sessionId}}","createdAt":1785974400000,"cwd":"{{cwd}}"}
|
||||
{"type":"user/message","seq":0,"time":1785974400001,"data":{"content":[{"type":"text","text":"Reply exactly CORE_WEB_REQUEST_OK and stop."}],"source":{"kind":"user","rpcId":"{{rpcId}}"}},"surfaceOp":"append"}
|
||||
{"type":"session","version":0,"id":"{{sessionId}}","createdAt":1785974400000,"cwd":"{{cwd}}","agentPreset":"minimal"}
|
||||
{"type":"user/message","seq":0,"time":1785974400001,"data":{"content":[{"type":"text","text":"Reply exactly MINIMAL_PRESET_REQUEST_OK and stop."}],"source":{"kind":"user","rpcId":"{{rpcId}}"}},"surfaceOp":"append"}
|
||||
{"type":"assistant/chunk","seq":1,"time":1785974400002,"data":{"turn":1,"step":1,"chunk":{"type":"block-start","index":0,"blockType":"text"}}}
|
||||
{"type":"assistant/chunk","seq":2,"time":1785974400003,"data":{"turn":1,"step":1,"chunk":{"type":"text-delta","index":0,"text":"CORE_WEB_REQUEST_OK"}}}
|
||||
{"type":"assistant/chunk","seq":3,"time":1785974400004,"data":{"turn":1,"step":1,"chunk":{"type":"block-end","index":0,"block":{"type":"text","text":"CORE_WEB_REQUEST_OK"}}}}
|
||||
{"type":"assistant/chunk","seq":2,"time":1785974400003,"data":{"turn":1,"step":1,"chunk":{"type":"text-delta","index":0,"text":"MINIMAL_PRESET_REQUEST_OK"}}}
|
||||
{"type":"assistant/chunk","seq":3,"time":1785974400004,"data":{"turn":1,"step":1,"chunk":{"type":"block-end","index":0,"block":{"type":"text","text":"MINIMAL_PRESET_REQUEST_OK"}}}}
|
||||
{"type":"assistant/chunk","seq":4,"time":1785974400005,"data":{"turn":1,"step":1,"chunk":{"type":"usage","usage":{"inputTokens":10,"outputTokens":4}}}}
|
||||
{"type":"assistant/chunk","seq":5,"time":1785974400006,"data":{"turn":1,"step":1,"chunk":{"type":"finish","reason":{"kind":"stop"}}}}
|
||||
@@ -24,7 +24,7 @@
|
||||
"exclude": [
|
||||
"tests/scaffold.ts",
|
||||
"tests/scaffold-hermetic.e2e.ts",
|
||||
"tests/core-web-profile.snapshot.ts",
|
||||
"tests/minimal-preset.snapshot.ts",
|
||||
"tests/live-interactions.e2e.ts",
|
||||
"tests/question-composer.e2e.ts",
|
||||
"tests/approval-composer.e2e.ts",
|
||||
|
||||
Reference in New Issue
Block a user