Coder/deepseek-harness
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

test(skill): close invocation policy coverage gaps

Browse Source
This commit is contained in:
Tianyi Cui
2026-07-29 23:17:07 +08:00
parent 976114becb
commit 1619d7686b
10 changed files with 170 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
examples/tui-agent/tests/snapshots/skill-invocation-policy/session.jsonl
View File

@@ -1 +1,5 @@
{"type":"session","version":0,"id":"31f63cc0-0198-4ab2-bfde-79a4eb4f1867","createdAt":1783352180000,"cwd":"{{cwd}}","delegationDepth":0}
{"type":"assistant/chunk","seq":0,"time":1783352180001,"data":{"turn":1,"step":1,"chunk":{"type":"block-start","index":0,"blockType":"text"}}}
{"type":"assistant/chunk","seq":1,"time":1783352180002,"data":{"turn":1,"step":1,"chunk":{"type":"text-delta","index":0,"text":"USER-ONLY SKILL LOADED"}}}
{"type":"assistant/chunk","seq":2,"time":1783352180003,"data":{"turn":1,"step":1,"chunk":{"type":"block-end","index":0,"block":{"type":"text","text":"USER-ONLY SKILL LOADED"}}}}
{"type":"assistant/chunk","seq":3,"time":1783352180004,"data":{"turn":1,"step":1,"chunk":{"type":"finish","reason":{"kind":"stop"}}}}

122
examples/tui-agent/tests/snapshots/skill-invocation-policy/terminal.expected.txt
View File

@@ -26,30 +26,132 @@ buffer
7-35| <blank>
=== denied exact invocation ===
=== loaded exact invocation ===
terminal 100x36 buffer=normal length=36 base=0 viewport=0
lifecycle started=1 stopped=0 progress=inactive
title "DSH TUI snapshot"
cursor hidden column=7 viewportRow=7 bufferRow=7
title "<skill name=\"user-only-skill\"> Reference — DSH TUI snapshot"
cursor hidden column=7 viewportRow=30 bufferRow=30
buffer
0| " DEEPSEEK HARNESS"
style 1-8 fg=bright-magenta bold
style 10-16 bold
1| " Recorded replay: skill-invocation-policy"
1| " <skill name=\"user-only-skill\"> Reference"
style 1-40 dim
2| " main-session"
style 1-12 dim
3| <blank>
4| "Skill \"model-only-skill\" is not available for user invocation. "
style 0-61 fg=yellow
5| <blank>
6| "/workspace/project deepseek-v4-flash ↑0 ↓0 0% context"
4| "You "
style 0-2 fg=bright-magenta bold underline
5| "<skill name=\"user-only-skill\"> "
6| "References in this skill are relative to "
7| "/workspace/project/.agents/skills/user-only-skill. "
8| " "
9| "USER-ONLY BODY "
10| "</skill> "
11| <blank>
12| "Context · dsh-tool-skill"
style 0-23 dim
13| "A skill is a reusable set of task-specific instructions. The following skills are available in this "
style 0-99 dim
14| "session: "
style 0-7 dim
15| " "
16| "<available_skills> "
style 0-17 dim
17| "- `model-only-skill`: Model-only assembled snapshot skill. "
style 0-57 dim
18| "</available_skills> "
style 0-18 dim
19| " "
20| "If the user names a skill, or the task clearly matches a skill's description, call the `skill` tool "
style 0-99 dim
21| "with the exact skill name before taking task actions. Load all applicable skills, then follow their "
style 0-99 dim
22| "full instructions. This catalog contains summaries only; do not infer or follow a skill's "
style 0-99 dim
23| "instructions until it has been loaded. "
style 0-37 dim
24| <blank>
25| "Assistant "
style 0-8 fg=bright-magenta bold underline
26| "USER-ONLY SKILL LOADED "
27| "Model wait 0.0s · Completed 2026-07-21 12:00:00 "
style 0-46 dim
28| <blank>
29| "/workspace/project deepseek-v4-flash ↑0 ↓0 3% context"
style 0-51 fg=bright-magenta bold
style 54-70 dim
style 73-77 dim
style 80-89 dim
7| " dsh > "
30| " dsh ◍ "
style 1-3 fg=bright-magenta bold
style 5-6 dim
style 7-7 inverse
8-35| <blank>
31-35| <blank>
=== denied exact invocation ===
terminal 100x36 buffer=normal length=36 base=0 viewport=0
lifecycle started=1 stopped=0 progress=inactive
title "<skill name=\"user-only-skill\"> Reference — DSH TUI snapshot"
cursor hidden column=7 viewportRow=32 bufferRow=32
buffer
0| " DEEPSEEK HARNESS"
style 1-8 fg=bright-magenta bold
style 10-16 bold
1| " <skill name=\"user-only-skill\"> Reference"
style 1-40 dim
2| " main-session"
style 1-12 dim
3| <blank>
4| "You "
style 0-2 fg=bright-magenta bold underline
5| "<skill name=\"user-only-skill\"> "
6| "References in this skill are relative to "
7| "/workspace/project/.agents/skills/user-only-skill. "
8| " "
9| "USER-ONLY BODY "
10| "</skill> "
11| <blank>
12| "Context · dsh-tool-skill"
style 0-23 dim
13| "A skill is a reusable set of task-specific instructions. The following skills are available in this "
style 0-99 dim
14| "session: "
style 0-7 dim
15| " "
16| "<available_skills> "
style 0-17 dim
17| "- `model-only-skill`: Model-only assembled snapshot skill. "
style 0-57 dim
18| "</available_skills> "
style 0-18 dim
19| " "
20| "If the user names a skill, or the task clearly matches a skill's description, call the `skill` tool "
style 0-99 dim
21| "with the exact skill name before taking task actions. Load all applicable skills, then follow their "
style 0-99 dim
22| "full instructions. This catalog contains summaries only; do not infer or follow a skill's "
style 0-99 dim
23| "instructions until it has been loaded. "
style 0-37 dim
24| <blank>
25| "Assistant "
style 0-8 fg=bright-magenta bold underline
26| "USER-ONLY SKILL LOADED "
27| "Model wait 0.0s · Completed 2026-07-21 12:00:00 "
style 0-46 dim
28| <blank>
29| "Skill \"model-only-skill\" is not available for user invocation. "
style 0-61 fg=yellow
30| <blank>
31| "/workspace/project deepseek-v4-flash ↑0 ↓0 3% context"
style 0-51 fg=bright-magenta bold
style 54-70 dim
style 73-77 dim
style 80-89 dim
32| " dsh ◍ "
style 1-3 fg=bright-magenta bold
style 5-6 dim
style 7-7 inverse
33-35| <blank>

1
examples/tui-agent/tests/snapshots/skill-invocation-policy/workspace/.agents/skills/model-only-skill/SKILL.md
View File

@@ -1,7 +1,6 @@
---
name: model-only-skill
description: Model-only assembled snapshot skill.
disable-model-invocation: false
user-invocable: false
---

1
examples/tui-agent/tests/snapshots/skill-invocation-policy/workspace/.agents/skills/user-only-skill/SKILL.md
View File

@@ -2,7 +2,6 @@
name: user-only-skill
description: User-only assembled snapshot skill.
disable-model-invocation: true
user-invocable: true
---
USER-ONLY BODY

10
examples/tui-agent/tests/tui-keyless-smoke.e2e.ts
View File

@@ -217,11 +217,12 @@ describe('tui-agent keyless smoke (real Loader tree in a PTY)', () => {
}, LOADER_SMOKE_TEST_TIMEOUT_MS)
it('loads a local skill via /skill: and delivers its body to the model as a user turn', async () => {
// The whole manual-invocation path in one keyless boot: `ctx.get('skills')`
// The whole user-only invocation path in one keyless boot: `ctx.get('skills')`
// resolves in the shipped tree, the client-side `/skill:` command parses,
// the local provider loads `scripted-skill` from the agents home, and the
// rendered `<skill name="…">` block reaches the model — proven by the
// scripted adapter echoing the fixture's body marker only when it arrives.
// and the local provider admits a model-disabled skill by the omitted
// `user-invocable` default. The rendered `<skill name="…">` block reaches
// the model — proven by the scripted adapter echoing the fixture's body
// marker only when it arrives.
const output = await smoke({
label: 'tui-agent skill',
tempDirPrefix: 'tui-agent-skill-',
@@ -233,7 +234,6 @@ describe('tui-agent keyless smoke (real Loader tree in a PTY)', () => {
'name: scripted-skill',
'description: Keyless PTY proof that the skill command loads a local skill into the conversation.',
'disable-model-invocation: true',
'user-invocable: true',
'---',
'',
'SCRIPTED SKILL BODY MARKER',

46
examples/tui-agent/tests/tui.snapshot.ts
View File

@@ -41,6 +41,7 @@ const UUID_RE = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/gi
type SnapshotMode = 'replay' | 'record' | 'refresh'
type Composition = 'native' | 'code' | 'advanced'
type ScenarioInteraction = 'skill-invocation-policy'
interface Scenario {
name: string
@@ -65,8 +66,8 @@ interface Scenario {
* preview + locator while the program value stays whole.
*/
spillMaxInlineBytes?: number
/** Exercise user-facing skill discovery and exact policy denial without opening a model turn. */
skillInvocationPolicy?: boolean
/** Run scenario-specific terminal input instead of replaying recorded user prompts. */
interaction?: ScenarioInteraction
}
const SCENARIOS: Scenario[] = [
@@ -106,7 +107,7 @@ const SCENARIOS: Scenario[] = [
expectedTools: [],
recorded: false,
seedWorkspace: true,
skillInvocationPolicy: true,
interaction: 'skill-invocation-policy',
},
{
name: 'code-mode',
@@ -280,7 +281,7 @@ async function runScenario(scenario: Scenario): Promise<ScenarioResult> {
const fixtureFile = join(dir, 'session.jsonl')
const childFiles = childFixturePaths(scenario)
const prompts = userPrompts(await readFile(fixtureFile, 'utf8'))
if (scenario.skillInvocationPolicy !== true) {
if (scenario.interaction === undefined) {
expect(prompts.length, `${scenario.name} must carry at least one recorded user prompt`).toBeGreaterThan(0)
}
@@ -321,8 +322,8 @@ async function runScenario(scenario: Scenario): Promise<ScenarioResult> {
})
await settleTerminal(terminal)
let skillPolicySnapshot: string | undefined
if (scenario.skillInvocationPolicy === true) {
let interactionSnapshot: string | undefined
if (scenario.interaction === 'skill-invocation-policy') {
terminal.send('/skill')
await settleTerminal(terminal)
const discovery = normalizeTerminalSnapshot(
@@ -335,6 +336,25 @@ async function runScenario(scenario: Scenario): Promise<ScenarioResult> {
terminal.send('\x03')
await settleTerminal(terminal)
const skillTurnEnded = new Promise<void>((resolve) => {
const detach = ctx.on('session/event', (session, event) => {
if (session !== agent.session || event.type !== 'turn/end') return
detach()
resolve()
})
})
terminal.send('/skill:user-only-skill')
terminal.send('\r')
await skillTurnEnded
await agent.whenIdle()
await settleTerminal(terminal)
const loaded = normalizeTerminalSnapshot(
await terminal.snapshot({ includeScrollback: true }),
cwd,
displayCwd,
)
expect(loaded).toContain('USER-ONLY SKILL LOADED')
terminal.send('/skill:model-only-skill')
terminal.send('\r')
await settleTerminal(terminal)
@@ -346,10 +366,13 @@ async function runScenario(scenario: Scenario): Promise<ScenarioResult> {
expect(denied).toContain('model-only-skill')
expect(denied).toContain('not available for user invocation.')
expect(denied).not.toContain('MODEL-ONLY BODY MUST NOT LOAD')
skillPolicySnapshot = [
interactionSnapshot = [
'=== skill autocomplete ===',
discovery,
'',
'=== loaded exact invocation ===',
loaded,
'',
'=== denied exact invocation ===',
denied,
].join('\n')
@@ -380,10 +403,8 @@ async function runScenario(scenario: Scenario): Promise<ScenarioResult> {
const events: SessionEvent[] = [...agent.session.events]
const firstHeader = events.find(event => event.type === 'request/header')
if (scenario.skillInvocationPolicy !== true) {
expect(firstHeader?.type === 'request/header' && firstHeader.data.header.system)
.toContain(FILE_REFERENCE_PROMPT)
}
expect(firstHeader?.type === 'request/header' && firstHeader.data.header.system)
.toContain(FILE_REFERENCE_PROMPT)
expect(events.filter(event => event.type === 'tool/call').map(event => event.data.name)).toEqual(scenario.expectedTools)
for (const [type, count] of Object.entries(scenario.expectedEventCounts ?? {})) {
expect(events.filter(event => event.type === type), `${scenario.name} must emit ${type}`).toHaveLength(count)
@@ -439,12 +460,11 @@ async function runScenario(scenario: Scenario): Promise<ScenarioResult> {
}
expect(terminal.themeViolations(), `${scenario.name} must remain theme-agnostic`).toEqual([])
const terminalSnapshot = normalizeTerminalSnapshot(
const snapshot = interactionSnapshot ?? normalizeTerminalSnapshot(
await terminal.snapshot({ includeScrollback: true }),
cwd,
displayCwd,
)
const snapshot = skillPolicySnapshot ?? terminalSnapshot
await handle.dispose()
const children = disposedSessions
.filter(session => session !== agent.session)