test(skill): snapshot invocation policy surfaces
This commit is contained in:
Tianyi Cui
@@ -0,0 +1 @@
|
||||
{"type":"session","version":0,"id":"31f63cc0-0198-4ab2-bfde-79a4eb4f1867","createdAt":1783352180000,"cwd":"{{cwd}}","delegationDepth":0}
|
||||
@@ -0,0 +1,55 @@
|
||||
=== skill autocomplete ===
|
||||
terminal 100x36 buffer=normal length=36 base=0 viewport=0
|
||||
lifecycle started=1 stopped=0 progress=inactive
|
||||
title "DSH TUI snapshot"
|
||||
cursor hidden column=13 viewportRow=5 bufferRow=5
|
||||
buffer
|
||||
0| " DEEPSEEK HARNESS"
|
||||
style 1-8 fg=bright-magenta bold
|
||||
style 10-16 bold
|
||||
1| " Recorded replay: skill-invocation-policy"
|
||||
style 1-40 dim
|
||||
2| " main-session"
|
||||
style 1-12 dim
|
||||
3| <blank>
|
||||
4| "/workspace/project deepseek-v4-flash ↑0 ↓0 0% context"
|
||||
style 0-51 fg=bright-magenta bold
|
||||
style 54-70 dim
|
||||
style 73-77 dim
|
||||
style 80-89 dim
|
||||
5| " dsh > /skill "
|
||||
style 1-3 fg=bright-magenta bold
|
||||
style 5-6 dim
|
||||
style 13-13 inverse
|
||||
6| " → skill:user-only-skill (project) — User-only assembled snapshot skill. "
|
||||
style 7-78 fg=bright-magenta
|
||||
7-35| <blank>
|
||||
|
||||
|
||||
=== denied exact invocation ===
|
||||
terminal 100x36 buffer=normal length=36 base=0 viewport=0
|
||||
lifecycle started=1 stopped=0 progress=inactive
|
||||
title "DSH TUI snapshot"
|
||||
cursor hidden column=7 viewportRow=7 bufferRow=7
|
||||
buffer
|
||||
0| " DEEPSEEK HARNESS"
|
||||
style 1-8 fg=bright-magenta bold
|
||||
style 10-16 bold
|
||||
1| " Recorded replay: skill-invocation-policy"
|
||||
style 1-40 dim
|
||||
2| " main-session"
|
||||
style 1-12 dim
|
||||
3| <blank>
|
||||
4| "Skill \"model-only-skill\" is not available for user invocation. "
|
||||
style 0-61 fg=yellow
|
||||
5| <blank>
|
||||
6| "/workspace/project deepseek-v4-flash ↑0 ↓0 0% context"
|
||||
style 0-51 fg=bright-magenta bold
|
||||
style 54-70 dim
|
||||
style 73-77 dim
|
||||
style 80-89 dim
|
||||
7| " dsh > "
|
||||
style 1-3 fg=bright-magenta bold
|
||||
style 5-6 dim
|
||||
style 7-7 inverse
|
||||
8-35| <blank>
|
||||
@@ -0,0 +1,8 @@
|
||||
---
|
||||
name: model-only-skill
|
||||
description: Model-only assembled snapshot skill.
|
||||
disable-model-invocation: false
|
||||
user-invocable: false
|
||||
---
|
||||
|
||||
MODEL-ONLY BODY MUST NOT LOAD
|
||||
@@ -0,0 +1,8 @@
|
||||
---
|
||||
name: user-only-skill
|
||||
description: User-only assembled snapshot skill.
|
||||
disable-model-invocation: true
|
||||
user-invocable: true
|
||||
---
|
||||
|
||||
USER-ONLY BODY
|
||||
@@ -65,6 +65,8 @@ interface Scenario {
|
||||
* preview + locator while the program value stays whole.
|
||||
*/
|
||||
spillMaxInlineBytes?: number
|
||||
/** Exercise user-facing skill discovery and exact policy denial without opening a model turn. */
|
||||
skillInvocationPolicy?: boolean
|
||||
}
|
||||
|
||||
const SCENARIOS: Scenario[] = [
|
||||
@@ -98,6 +100,14 @@ const SCENARIOS: Scenario[] = [
|
||||
recorded: true,
|
||||
seedWorkspace: true,
|
||||
},
|
||||
{
|
||||
name: 'skill-invocation-policy',
|
||||
composition: 'native',
|
||||
expectedTools: [],
|
||||
recorded: false,
|
||||
seedWorkspace: true,
|
||||
skillInvocationPolicy: true,
|
||||
},
|
||||
{
|
||||
name: 'code-mode',
|
||||
composition: 'code',
|
||||
@@ -269,9 +279,10 @@ async function runScenario(scenario: Scenario): Promise<ScenarioResult> {
|
||||
const dir = scenarioDir(scenario)
|
||||
const fixtureFile = join(dir, 'session.jsonl')
|
||||
const childFiles = childFixturePaths(scenario)
|
||||
const fixture = await readFile(fixtureFile, 'utf8')
|
||||
const prompts = userPrompts(fixture)
|
||||
expect(prompts.length, `${scenario.name} must carry at least one recorded user prompt`).toBeGreaterThan(0)
|
||||
const prompts = userPrompts(await readFile(fixtureFile, 'utf8'))
|
||||
if (scenario.skillInvocationPolicy !== true) {
|
||||
expect(prompts.length, `${scenario.name} must carry at least one recorded user prompt`).toBeGreaterThan(0)
|
||||
}
|
||||
|
||||
const cwd = await mkdtemp(join(SNAPSHOT_TMP_ROOT, `dsh-tui-snapshot-${scenario.name}-`))
|
||||
const displayCwd = `/tmp/${basename(cwd)}`
|
||||
@@ -310,6 +321,40 @@ async function runScenario(scenario: Scenario): Promise<ScenarioResult> {
|
||||
})
|
||||
await settleTerminal(terminal)
|
||||
|
||||
let skillPolicySnapshot: string | undefined
|
||||
if (scenario.skillInvocationPolicy === true) {
|
||||
terminal.send('/skill')
|
||||
await settleTerminal(terminal)
|
||||
const discovery = normalizeTerminalSnapshot(
|
||||
await terminal.snapshot({ includeScrollback: true }),
|
||||
cwd,
|
||||
displayCwd,
|
||||
)
|
||||
expect(discovery).toContain('user-only-skill')
|
||||
expect(discovery).not.toContain('model-only-skill')
|
||||
|
||||
terminal.send('\x03')
|
||||
await settleTerminal(terminal)
|
||||
terminal.send('/skill:model-only-skill')
|
||||
terminal.send('\r')
|
||||
await settleTerminal(terminal)
|
||||
const denied = normalizeTerminalSnapshot(
|
||||
await terminal.snapshot({ includeScrollback: true }),
|
||||
cwd,
|
||||
displayCwd,
|
||||
)
|
||||
expect(denied).toContain('model-only-skill')
|
||||
expect(denied).toContain('not available for user invocation.')
|
||||
expect(denied).not.toContain('MODEL-ONLY BODY MUST NOT LOAD')
|
||||
skillPolicySnapshot = [
|
||||
'=== skill autocomplete ===',
|
||||
discovery,
|
||||
'',
|
||||
'=== denied exact invocation ===',
|
||||
denied,
|
||||
].join('\n')
|
||||
}
|
||||
|
||||
let remainingPrompts = prompts
|
||||
if (scenario.enterPlanMode === true) {
|
||||
const firstPrompt = prompts[0]!
|
||||
@@ -335,8 +380,10 @@ async function runScenario(scenario: Scenario): Promise<ScenarioResult> {
|
||||
|
||||
const events: SessionEvent[] = [...agent.session.events]
|
||||
const firstHeader = events.find(event => event.type === 'request/header')
|
||||
expect(firstHeader?.type === 'request/header' && firstHeader.data.header.system)
|
||||
.toContain(FILE_REFERENCE_PROMPT)
|
||||
if (scenario.skillInvocationPolicy !== true) {
|
||||
expect(firstHeader?.type === 'request/header' && firstHeader.data.header.system)
|
||||
.toContain(FILE_REFERENCE_PROMPT)
|
||||
}
|
||||
expect(events.filter(event => event.type === 'tool/call').map(event => event.data.name)).toEqual(scenario.expectedTools)
|
||||
for (const [type, count] of Object.entries(scenario.expectedEventCounts ?? {})) {
|
||||
expect(events.filter(event => event.type === type), `${scenario.name} must emit ${type}`).toHaveLength(count)
|
||||
@@ -392,11 +439,12 @@ async function runScenario(scenario: Scenario): Promise<ScenarioResult> {
|
||||
}
|
||||
|
||||
expect(terminal.themeViolations(), `${scenario.name} must remain theme-agnostic`).toEqual([])
|
||||
const snapshot = normalizeTerminalSnapshot(
|
||||
const terminalSnapshot = normalizeTerminalSnapshot(
|
||||
await terminal.snapshot({ includeScrollback: true }),
|
||||
cwd,
|
||||
displayCwd,
|
||||
)
|
||||
const snapshot = skillPolicySnapshot ?? terminalSnapshot
|
||||
await handle.dispose()
|
||||
const children = disposedSessions
|
||||
.filter(session => session !== agent.session)
|
||||
|
||||
Reference in New Issue
Block a user