Merge remote-tracking branch 'origin/master' into worktree/locale-browser-default
This commit is contained in:
171
apps/web/tests/permission-policy-context.e2e.ts
Normal file
171
apps/web/tests/permission-policy-context.e2e.ts
Normal file
@@ -0,0 +1,171 @@
|
||||
// Web acceptance for current sandbox-policy context. A real Chromium drives
|
||||
// the shipped /permission command through all three presets; record mode uses
|
||||
// the real provider, while replay keeps the same provider-authored behavior
|
||||
// keyless. Assertions read the exact durable header, runtime-context messages,
|
||||
// and tool calls, so assistant prose alone cannot satisfy the scenario.
|
||||
import { readFile } from 'node:fs/promises'
|
||||
import { join } from 'node:path'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import type { Browser, Page } from 'playwright'
|
||||
import { chromium } from 'playwright'
|
||||
import { afterAll, beforeAll, describe, expect, it, onTestFailed } from 'vitest'
|
||||
import { canonicalPath } from '@deepseek-ai/dsh-sandbox'
|
||||
import type { SessionEvent } from '@deepseek-ai/dsh-session'
|
||||
import {
|
||||
assertFixtureInventory, fixtureUserPrompts, launchWebScaffold, recordFixture,
|
||||
watchConsole, webSnapshotMode, type WebScaffold,
|
||||
} from './scaffold.ts'
|
||||
import { connectFreshWorkspace, newEnglishPage, saveFailureShot } from './support.ts'
|
||||
|
||||
const SNAPSHOT_DIR = fileURLToPath(new URL('./snapshots/permission-policy-context', import.meta.url))
|
||||
const FIXTURE = fileURLToPath(new URL('./snapshots/permission-policy-context/session.jsonl', import.meta.url))
|
||||
const MODE = webSnapshotMode()
|
||||
|
||||
const PROMPTS = [
|
||||
'Can you create or edit a normal file right now under the current policy? Answer directly in one sentence. Do not call a tool just to discover the policy.',
|
||||
'Does the DSH file sandbox currently restrict file operations? Answer directly in one sentence. Do not call tools.',
|
||||
'Reply with exactly WORKSPACE_POLICY_SEEN. Do not call tools.',
|
||||
'Create the relative path policy-neutral.txt in the current workspace containing exactly POLICY_NEUTRAL_OK, verify its contents, then report completion.',
|
||||
] as const
|
||||
|
||||
const PRESET_LABELS = ['Read Only', 'Full access', 'Workspace Write'] as const
|
||||
|
||||
function requestSystems(events: readonly SessionEvent[]): string[] {
|
||||
return events.flatMap((event) => {
|
||||
if (event.type !== 'request/header') return []
|
||||
return typeof event.data.header.system === 'string' ? [event.data.header.system] : []
|
||||
})
|
||||
}
|
||||
|
||||
function runtimeContexts(events: readonly SessionEvent[]): string[] {
|
||||
return events.flatMap((event) => {
|
||||
if (event.type !== 'user/message'
|
||||
|| event.data.source.kind !== 'plugin'
|
||||
|| event.data.source.plugin !== '@deepseek-ai/dsh-system-prompt') return []
|
||||
return event.data.content.flatMap(block => block.type === 'text' ? [block.text] : [])
|
||||
})
|
||||
}
|
||||
|
||||
function assistantTexts(events: readonly SessionEvent[]): string[] {
|
||||
return events.flatMap((event) => {
|
||||
if (event.type !== 'assistant/message') return []
|
||||
const text = event.data.message.content.flatMap(block => block.type === 'text' ? [block.text] : []).join('').replaceAll('**', '')
|
||||
return text.length === 0 ? [] : [text]
|
||||
})
|
||||
}
|
||||
|
||||
function callArgs(event: Extract<SessionEvent, { type: 'tool/call' }>): Record<string, unknown> {
|
||||
return JSON.parse(event.data.arguments) as Record<string, unknown>
|
||||
}
|
||||
|
||||
describe('web e2e: current sandbox policy reaches the model before tools', () => {
|
||||
let scaffold: WebScaffold
|
||||
let browser: Browser
|
||||
let page: Page
|
||||
let tripwire: ReturnType<typeof watchConsole>
|
||||
let disposeApproval: (() => void) | undefined
|
||||
let sessionWorkspace: string | undefined
|
||||
const sessionEvents: SessionEvent[] = []
|
||||
|
||||
beforeAll(async () => {
|
||||
scaffold = await launchWebScaffold(MODE === 'record' ? {} : { replayFixture: FIXTURE })
|
||||
disposeApproval = scaffold.ctx.on('approval/request', () => Promise.resolve('allowed-once'), { prepend: true })
|
||||
scaffold.ctx.on('session/event', (session, event: SessionEvent) => {
|
||||
sessionWorkspace = session.header.cwd
|
||||
sessionEvents.push(event)
|
||||
})
|
||||
browser = await chromium.launch()
|
||||
page = await newEnglishPage(browser)
|
||||
tripwire = watchConsole(page)
|
||||
await page.goto(scaffold.baseUrl, { waitUntil: 'load' })
|
||||
await page.waitForSelector('[class*="frame"]', { timeout: 30_000 })
|
||||
await connectFreshWorkspace(page)
|
||||
}, 120_000)
|
||||
|
||||
afterAll(async () => {
|
||||
await browser?.close()
|
||||
disposeApproval?.()
|
||||
await scaffold?.close()
|
||||
})
|
||||
|
||||
it('switches read-only, danger-full-access, and workspace-write through the real GUI command path', async () => {
|
||||
onTestFailed(() => saveFailureShot(page, 'web-e2e-permission-policy-context'))
|
||||
if (MODE !== 'record') {
|
||||
expect(fixtureUserPrompts(await readFile(FIXTURE, 'utf8'))).toEqual(PROMPTS)
|
||||
}
|
||||
|
||||
const input = page.locator('textarea').first()
|
||||
let sessionId: Awaited<ReturnType<WebScaffold['whenTurnSettled']>> | undefined
|
||||
for (const [index, preset] of ['read-only', 'danger-full-access', 'workspace-write'].entries()) {
|
||||
await input.fill(`/permission ${preset}`)
|
||||
await input.press('Enter')
|
||||
await page.getByRole('button', { name: `Access mode, current: ${PRESET_LABELS[index]}` })
|
||||
.waitFor({ timeout: 10_000 })
|
||||
|
||||
const settled = scaffold.whenTurnSettled()
|
||||
await input.fill(PROMPTS[index] as string)
|
||||
await input.press('Enter')
|
||||
sessionId = await settled
|
||||
await expect.poll(() => input.isEnabled(), { timeout: 10_000 }).toBe(true)
|
||||
}
|
||||
|
||||
await input.fill('/permission read-only')
|
||||
await input.press('Enter')
|
||||
await page.getByRole('button', { name: 'Access mode, current: Read Only' }).waitFor({ timeout: 10_000 })
|
||||
const settled = scaffold.whenTurnSettled()
|
||||
await input.fill(PROMPTS[3])
|
||||
await input.press('Enter')
|
||||
sessionId = await settled
|
||||
|
||||
if (sessionId === undefined) throw new Error('permission-policy scenario completed no model turn')
|
||||
if (MODE === 'record') await recordFixture(scaffold, sessionId, FIXTURE)
|
||||
}, 240_000)
|
||||
|
||||
it.skipIf(MODE === 'record')('records cache-safe current policy before the corresponding model behavior', async () => {
|
||||
const systems = requestSystems(sessionEvents)
|
||||
expect(systems).toHaveLength(1)
|
||||
expect(systems[0]).not.toContain('Current DSH file policy:')
|
||||
expect(systems[0]).not.toContain('Approval policy:')
|
||||
expect(systems[0]).not.toContain('Approval prompts are disabled in this session')
|
||||
|
||||
const contexts = runtimeContexts(sessionEvents)
|
||||
expect(contexts).toHaveLength(4)
|
||||
expect(contexts[0]).toContain('Current DSH file policy: read-only. Any available operation enforced by the DSH file sandbox cannot modify files in the standing mode.')
|
||||
expect(contexts[0]).toContain('Do not refuse a required modification from this policy alone')
|
||||
expect(contexts[0]).toContain('Approval policy: ask.')
|
||||
expect(contexts[1]).toContain('Current DSH file policy: danger-full-access. The DSH file sandbox does not restrict file modifications by available operations.')
|
||||
expect(contexts[1]).toContain('Approval prompts are disabled in this session')
|
||||
|
||||
if (sessionWorkspace === undefined) throw new Error('permission-policy scenario observed no session workspace')
|
||||
expect(contexts[2]).toContain(`Current DSH file policy: workspace-write. Any available operation enforced by the DSH file sandbox may modify files under the session workspace: ${JSON.stringify(canonicalPath(sessionWorkspace))}. Some platform temporary areas may also be writable.`)
|
||||
expect(contexts[2]).toContain('Approval policy: ask.')
|
||||
expect(contexts[2]).not.toContain('Approval prompts are disabled in this session')
|
||||
expect(contexts[3]).toContain('Current DSH file policy: read-only.')
|
||||
|
||||
const answers = assistantTexts(sessionEvents)
|
||||
expect(answers.length).toBeGreaterThanOrEqual(4)
|
||||
expect(answers[0]).toMatch(/read-only.*(?:denied|cannot modify|cannot create or edit)/i)
|
||||
expect(answers[1]).toMatch(/does not restrict.*(?:file operations|(?:write\/edit tools|write and edit tools).*one-shot bash commands)/i)
|
||||
expect(answers[2]).toBe('WORKSPACE_POLICY_SEEN')
|
||||
const calls = sessionEvents.filter(
|
||||
(event): event is Extract<SessionEvent, { type: 'tool/call' }> => event.type === 'tool/call',
|
||||
)
|
||||
expect(calls.every(call => call.data.turn === 4)).toBe(true)
|
||||
expect(calls.length).toBeGreaterThanOrEqual(2)
|
||||
const firstCall = calls[0]
|
||||
if (firstCall === undefined) throw new Error('neutral policy task produced no tool call')
|
||||
expect(callArgs(firstCall)['sandbox_permissions']).toBeUndefined()
|
||||
expect(calls.some(call => callArgs(call)['sandbox_permissions'] !== undefined)).toBe(true)
|
||||
expect(sessionEvents.some(event => event.type === 'tool/result'
|
||||
&& JSON.stringify(event.data).includes('[sandbox: file access denied under read-only mode]'))).toBe(true)
|
||||
expect(sessionEvents.some(event => event.type === 'approval/asked')).toBe(true)
|
||||
if (sessionWorkspace === undefined) throw new Error('permission-policy scenario observed no session workspace')
|
||||
expect(await readFile(join(sessionWorkspace, 'policy-neutral.txt'), 'utf8')).toBe('POLICY_NEUTRAL_OK')
|
||||
})
|
||||
|
||||
it.skipIf(MODE === 'record')('stays clean and keeps the fixture inventory closed', async () => {
|
||||
expect(tripwire.pageErrors).toEqual([])
|
||||
expect(tripwire.warnings).toEqual([])
|
||||
await assertFixtureInventory(SNAPSHOT_DIR, ['session.jsonl'])
|
||||
})
|
||||
})
|
||||
@@ -131,7 +131,7 @@ describe('web e2e: queue row actions', () => {
|
||||
|
||||
const snapshot = await captureStableAria(page, '[class*="centerCol"]', scaffold.workspaceCwd)
|
||||
await compareOrRefreshGolden(UI_EXPECTED, snapshot, MODE)
|
||||
expect(sessionEvents.filter(event => event.type === 'user/message')).toHaveLength(1)
|
||||
expect(sessionEvents.filter(event => event.type === 'user/message' && event.data.source.kind === 'user')).toHaveLength(1)
|
||||
expect(tripwire.pageErrors).toEqual([])
|
||||
expect(tripwire.warnings).toEqual([])
|
||||
|
||||
|
||||
@@ -11,6 +11,10 @@
|
||||
- img
|
||||
- button "Edit":
|
||||
- img
|
||||
- button "Context injection":
|
||||
- img
|
||||
- img
|
||||
- text: Context injection
|
||||
- 'button "Think The user wants me to write a single `run_code` program that:"':
|
||||
- img
|
||||
- img
|
||||
|
||||
@@ -11,6 +11,10 @@
|
||||
- img
|
||||
- button "Edit":
|
||||
- img
|
||||
- button "Context injection":
|
||||
- img
|
||||
- img
|
||||
- text: Context injection
|
||||
- button "Think The user wants me to:":
|
||||
- img
|
||||
- img
|
||||
|
||||
@@ -11,6 +11,10 @@
|
||||
- img
|
||||
- button "Edit":
|
||||
- img
|
||||
- button "Context injection":
|
||||
- img
|
||||
- img
|
||||
- text: Context injection
|
||||
- button "Think The user wants me to run a simple bash command and reply with \"DONE\".":
|
||||
- img
|
||||
- img
|
||||
|
||||
@@ -11,6 +11,10 @@
|
||||
- img
|
||||
- button "Edit":
|
||||
- img
|
||||
- button "Context injection":
|
||||
- img
|
||||
- img
|
||||
- text: Context injection
|
||||
- button "Think The user wants me to reply with a single word. Let me comply.":
|
||||
- img
|
||||
- img
|
||||
|
||||
@@ -11,6 +11,10 @@
|
||||
- img
|
||||
- button "Edit":
|
||||
- img
|
||||
- button "Context injection":
|
||||
- img
|
||||
- img
|
||||
- text: Context injection
|
||||
- paragraph: partial
|
||||
- text: Stopped
|
||||
- button "Copy":
|
||||
|
||||
@@ -11,6 +11,10 @@
|
||||
- img
|
||||
- button "Edit":
|
||||
- img
|
||||
- button "Context injection":
|
||||
- img
|
||||
- img
|
||||
- text: Context injection
|
||||
- textbox "Message the agent"
|
||||
- button "Commands":
|
||||
- img
|
||||
|
||||
@@ -11,6 +11,10 @@
|
||||
- img
|
||||
- button "Edit":
|
||||
- img
|
||||
- button "Context injection":
|
||||
- img
|
||||
- img
|
||||
- text: Context injection
|
||||
- paragraph: partial
|
||||
- status: Deep diving...
|
||||
- textbox "Message the agent"
|
||||
|
||||
@@ -11,6 +11,10 @@
|
||||
- img
|
||||
- button "Edit":
|
||||
- img
|
||||
- button "Context injection":
|
||||
- img
|
||||
- img
|
||||
- text: Context injection
|
||||
- group:
|
||||
- status: Retried model request (1/2) · {{duration}}
|
||||
- button "Think The user is asking for a one-sentence description of event sourcing. This is a straightforward knowledge question that doesn't require any skill loading or tool calls.":
|
||||
|
||||
124
apps/web/tests/snapshots/permission-policy-context/session.jsonl
Normal file
124
apps/web/tests/snapshots/permission-policy-context/session.jsonl
Normal file
File diff suppressed because one or more lines are too long
@@ -12,6 +12,10 @@
|
||||
- img
|
||||
- button "Edit":
|
||||
- img
|
||||
- button "Context injection":
|
||||
- img
|
||||
- img
|
||||
- text: Context injection
|
||||
- 'button "Think The user wants me to plan a small change to add a `--greeting` flag to a CLI. They explicitly told me not to read or write any files, and to call exit_plan_mode with a short plan. Let me do that directly."':
|
||||
- img
|
||||
- img
|
||||
|
||||
@@ -11,6 +11,10 @@
|
||||
- img
|
||||
- button "Edit":
|
||||
- img
|
||||
- button "Context injection":
|
||||
- img
|
||||
- img
|
||||
- text: Context injection
|
||||
- button "Think The user wants me to use the ask_user_question tool with specific parameters. Let me do exactly that.":
|
||||
- img
|
||||
- img
|
||||
|
||||
@@ -11,6 +11,10 @@
|
||||
- img
|
||||
- button "Edit":
|
||||
- img
|
||||
- button "Context injection":
|
||||
- img
|
||||
- img
|
||||
- text: Context injection
|
||||
- paragraph: partial
|
||||
- status: Deep diving...
|
||||
- button "2 queued messages"
|
||||
|
||||
@@ -11,6 +11,10 @@
|
||||
- img
|
||||
- button "Edit":
|
||||
- img
|
||||
- button "Context injection":
|
||||
- img
|
||||
- img
|
||||
- text: Context injection
|
||||
- paragraph: partial
|
||||
- status: Deep diving...
|
||||
- button "2 queued messages" [disabled] [expanded]
|
||||
|
||||
@@ -11,6 +11,10 @@
|
||||
- img
|
||||
- button "Edit":
|
||||
- img
|
||||
- button "Context injection":
|
||||
- img
|
||||
- img
|
||||
- text: Context injection
|
||||
- paragraph: partial
|
||||
- status: Deep diving...
|
||||
- list:
|
||||
|
||||
@@ -11,6 +11,10 @@
|
||||
- img
|
||||
- button "Edit":
|
||||
- img
|
||||
- button "Context injection":
|
||||
- img
|
||||
- img
|
||||
- text: Context injection
|
||||
- button "Think The user wants me to use the ask_user_question tool to ask them a specific question with the given parameters. Let me do exactly that.":
|
||||
- img
|
||||
- img
|
||||
|
||||
@@ -11,6 +11,10 @@
|
||||
- img
|
||||
- button "Edit":
|
||||
- img
|
||||
- button "Context injection":
|
||||
- img
|
||||
- img
|
||||
- text: Context injection
|
||||
- button "Think The user wants me to use the ask_user_question tool to ask them a specific question with the given parameters. Let me do exactly that.":
|
||||
- img
|
||||
- img
|
||||
|
||||
@@ -11,6 +11,10 @@
|
||||
- img
|
||||
- button "Edit":
|
||||
- img
|
||||
- button "Context injection":
|
||||
- img
|
||||
- img
|
||||
- text: Context injection
|
||||
- img
|
||||
- text: Search DeepSeek Harness snapshot search
|
||||
- list:
|
||||
|
||||
@@ -46,6 +46,7 @@
|
||||
"tests/message-actions.e2e.ts",
|
||||
"tests/queue-actions.e2e.ts",
|
||||
"tests/skill-invocation-policy.e2e.ts",
|
||||
"tests/permission-policy-context.e2e.ts",
|
||||
"tests/access-confirmation.e2e.ts"
|
||||
],
|
||||
"references": [
|
||||
|
||||
Reference in New Issue
Block a user