8c2970e70e
Review found five real defects in the configuration-source work, all confirmed against the code rather than argued: 1. The note claimed --config outranks settings.yaml. It does not: the settings seam registers a plugin's cordis entry config as the `base` layer and the user section layers over it, and the seam cannot tell a shipped value from a --config one. The note now states shipped reality and names --config-replace as the lever for a deployment that must win. Separately, a literal `apiKey` in settings outranked both the environment and .credentials.yaml — the field is removed, so configuration carries a reference and nothing else. 2. DEEPSEEK_SEARCH_BASE_URL was functionally deleted: the shipped inline went away without the provider learning to read it. It now resolves from the environment snapshot, as the README always claimed. 3. The bootstrap deny list missed the interpreter start-up hooks. BASH_ENV is the sharpest: `bash -c` sources it on every bash tool call, so a project .env could run a file of its choosing before every command. The list now covers BASH_ENV and its per-language siblings, the Git hook commands, and the remaining preload and CA variables, organised by what a variable does rather than which runtime owns it. 4. YAML parse errors quoted the offending source line — which in a credentials document is the secret — into boot stderr and the watcher's logger. Only the error code and position are reported now, in credentials-local and settings-local alike, pinned by a test that asserts the secret is absent. 5. 0600 governed only files the harness wrote. A hand-created 0644 document was read normally. POSIX now checks the mode before reading contents, at boot and on every reload; Windows has no mode to inspect and is skipped rather than faked. The project a session is launched in is trusted by default, with no prompt and no stored trust record: it may supply its own endpoint, ordinary variables, and a key ranked below the managed store. Trust stops at the harness itself — a discovered file still cannot set DSH_PERMISSION_MODE, PATH, BASH_ENV, or the rest, because those take effect with no user action, before any turn, outside the permission policy and the sandbox.
621 lines
28 KiB
TypeScript
621 lines
28 KiB
TypeScript
import { readFile, readdir, writeFile } from 'node:fs/promises'
|
|
import { createServer } from 'node:http'
|
|
import type { IncomingMessage, ServerResponse } from 'node:http'
|
|
import { delimiter, dirname, join } from 'node:path'
|
|
import { fileURLToPath } from 'node:url'
|
|
import {
|
|
normalizeSessionLog,
|
|
normalizeStdout,
|
|
refreshFixtureReplacements,
|
|
scrubRequestHeaders,
|
|
stabilizeRefreshLog,
|
|
tokenizeSessionFixtureCwd,
|
|
type HarvestedLog,
|
|
type NormalizeContext,
|
|
} from '@deepseek-ai/dsh-acp-snapshot'
|
|
import { LOADER_SMOKE_TEST_TIMEOUT_MS, runLoaderSmoke } from '@deepseek-ai/dsh-loader-smoke'
|
|
import { describe, expect, it } from 'vitest'
|
|
|
|
const snapshotsDir = join(dirname(fileURLToPath(import.meta.url)), 'snapshots')
|
|
const advancedScenarioDir = join(snapshotsDir, 'advanced-toolchain')
|
|
const advancedSessionFixture = join(advancedScenarioDir, 'session.jsonl')
|
|
const advancedStreamExpected = join(advancedScenarioDir, 'stream-json.expected.jsonl')
|
|
const advancedConfigPath = fileURLToPath(new URL('../advanced.cordis.snapshot.yml', import.meta.url))
|
|
const ptyScenarioDir = join(snapshotsDir, 'pty-tools')
|
|
const ptySessionFixture = join(ptyScenarioDir, 'session.jsonl')
|
|
const ptyStreamExpected = join(ptyScenarioDir, 'stream-json.expected.jsonl')
|
|
const ptyConfigPath = fileURLToPath(new URL('../pty.cordis.snapshot.yml', import.meta.url))
|
|
const goalScenarioDir = join(snapshotsDir, 'goal-tools')
|
|
const goalConfigPath = fileURLToPath(new URL('../goal.cordis.snapshot.yml', import.meta.url))
|
|
const retryScenarioDir = join(snapshotsDir, 'provider-retry')
|
|
const retryConfigPath = fileURLToPath(new URL('../retry.cordis.snapshot.yml', import.meta.url))
|
|
const credentialsScenarioDir = join(snapshotsDir, 'missing-credential')
|
|
const credentialsConfigPath = fileURLToPath(new URL('../credentials.cordis.snapshot.yml', import.meta.url))
|
|
const ralphScenarioDir = join(snapshotsDir, 'ralph-loop')
|
|
const ralphConfigPath = fileURLToPath(new URL('../ralph.cordis.snapshot.yml', import.meta.url))
|
|
const startupFailureConfigPath = fileURLToPath(new URL('./fixtures/startup-activation-error/cordis.yml', import.meta.url))
|
|
const startupFailureExpected = join(snapshotsDir, 'startup-activation-error', 'stderr.expected.txt')
|
|
const binScript = fileURLToPath(new URL('../../../packages/examples/cli-demo/src/bin.ts', import.meta.url))
|
|
const tsconfigPath = fileURLToPath(new URL('../../../tsconfig.json', import.meta.url))
|
|
const reasoningConfigPath = fileURLToPath(new URL('./fixtures/cli.cordis.yml', import.meta.url))
|
|
const deepseekDefaultsConfigPath = fileURLToPath(new URL('./fixtures/deepseek-defaults.cordis.yml', import.meta.url))
|
|
const refreshing = process.env.DSH_SNAPSHOT === 'refresh'
|
|
|
|
interface JsonObject {
|
|
[key: string]: unknown
|
|
}
|
|
|
|
interface PersistedLog {
|
|
readonly content: string
|
|
readonly header: JsonObject
|
|
}
|
|
|
|
interface DeepSeekDefaultsServer {
|
|
readonly url: string
|
|
readonly requests: JsonObject[]
|
|
close(): Promise<void>
|
|
}
|
|
|
|
/** Serve one deterministic DeepSeek-compatible response while retaining its request body. */
|
|
async function deepseekDefaultsServer(): Promise<DeepSeekDefaultsServer> {
|
|
const requests: JsonObject[] = []
|
|
const server = createServer((request: IncomingMessage, response: ServerResponse) => {
|
|
let body = ''
|
|
request.setEncoding('utf8')
|
|
request.on('data', (chunk: string) => { body += chunk })
|
|
request.on('end', () => {
|
|
requests.push(JSON.parse(body) as JsonObject)
|
|
response.writeHead(200, { 'content-type': 'text/event-stream' })
|
|
response.end([
|
|
'data: {"choices":[{"delta":{"content":"DEFAULTS_OK"}}]}',
|
|
'data: {"choices":[{"delta":{},"finish_reason":"stop"}],"usage":{"prompt_tokens":3,"completion_tokens":1}}',
|
|
'data: [DONE]',
|
|
'',
|
|
].join('\n\n'))
|
|
})
|
|
})
|
|
await new Promise<void>(resolve => server.listen(0, '127.0.0.1', resolve))
|
|
const address = server.address()
|
|
if (address === null || typeof address === 'string') throw new Error('DeepSeek defaults snapshot server has no port')
|
|
return {
|
|
url: `http://127.0.0.1:${address.port}`,
|
|
requests,
|
|
close: () => new Promise(resolve => server.close(() => { resolve() })),
|
|
}
|
|
}
|
|
|
|
function parseJsonl(content: string): JsonObject[] {
|
|
return content.split('\n')
|
|
.filter(line => line.trim().length > 0)
|
|
.map(line => JSON.parse(line) as JsonObject)
|
|
}
|
|
|
|
function contextFromLogs(contents: readonly string[]): NormalizeContext {
|
|
const headers = contents.map(content => parseJsonl(content)[0])
|
|
return {
|
|
sessionIds: headers.flatMap(header => typeof header?.id === 'string' ? [header.id] : []),
|
|
cwd: typeof headers[0]?.cwd === 'string' ? headers[0].cwd : '\0no-cwd\0',
|
|
}
|
|
}
|
|
|
|
function normalizeHeadlessStream(rawStdout: string, cwd: string): string {
|
|
const records = parseJsonl(rawStdout)
|
|
if (records.length === 0) throw new Error('headless snapshot emitted no stream-json records')
|
|
const final = records.at(-1)
|
|
if (final?.type !== 'result') throw new Error('headless snapshot did not end with a result record')
|
|
if (records.slice(0, -1).some(record => record.type !== 'session_event')) {
|
|
throw new Error('headless snapshot emitted a non-event record before its result')
|
|
}
|
|
|
|
const sessionIds = [...new Set(records.flatMap(record => typeof record.sessionId === 'string' ? [record.sessionId] : []))]
|
|
if (sessionIds.length !== 1) throw new Error(`headless snapshot streamed ${sessionIds.length} main session ids`)
|
|
const context: NormalizeContext = { sessionIds, cwd }
|
|
const events = records.slice(0, -1).map((record) => {
|
|
if (record.event === null || typeof record.event !== 'object' || Array.isArray(record.event)) {
|
|
throw new Error('headless snapshot emitted an invalid session event')
|
|
}
|
|
return record.event as JsonObject
|
|
})
|
|
const normalizedEvents = parseJsonl(scrubRequestHeaders(normalizeSessionLog(
|
|
`${events.map(event => JSON.stringify(event)).join('\n')}\n`,
|
|
context,
|
|
)))
|
|
const normalizedRecords = records.map((record, index) => index < normalizedEvents.length
|
|
? { ...record, event: normalizedEvents[index] }
|
|
: record)
|
|
return normalizeStdout(`${normalizedRecords.map(record => JSON.stringify(record)).join('\n')}\n`, context)
|
|
}
|
|
|
|
/** Zero durable goal timestamps inside both metadata records and rendered XML JSON. */
|
|
function normalizeGoalTimestamps(value: unknown): unknown {
|
|
if (typeof value === 'string') {
|
|
return value.replace(/("(?:createdAt|updatedAt|clearedAt)":)\d+/g, '$10')
|
|
}
|
|
if (Array.isArray(value)) return value.map(normalizeGoalTimestamps)
|
|
if (value !== null && typeof value === 'object') {
|
|
return Object.fromEntries(Object.entries(value).map(([key, item]) => [
|
|
key,
|
|
['createdAt', 'updatedAt', 'clearedAt'].includes(key) && typeof item === 'number'
|
|
? 0
|
|
: normalizeGoalTimestamps(item),
|
|
]))
|
|
}
|
|
return value
|
|
}
|
|
|
|
/** Normalize the stream's durable goal timestamps after the shared scrubbers. */
|
|
function normalizeGoalStream(rawStdout: string, cwd: string): string {
|
|
return parseJsonl(normalizeHeadlessStream(rawStdout, cwd))
|
|
.map(record => JSON.stringify(normalizeGoalTimestamps(record)))
|
|
.join('\n') + '\n'
|
|
}
|
|
|
|
async function scenarioPrompt(dir: string, label: string): Promise<string> {
|
|
const input = JSON.parse(await readFile(join(dir, 'input.json'), 'utf8')) as {
|
|
steps?: { op?: unknown; text?: unknown }[]
|
|
}
|
|
const prompt = input.steps?.find(step => step.op === 'prompt')?.text
|
|
if (typeof prompt !== 'string') throw new Error(`${label} input has no prompt step`)
|
|
return prompt
|
|
}
|
|
|
|
async function persistedLogs(cwd: string): Promise<PersistedLog[]> {
|
|
const root = join(cwd, '.sessions')
|
|
const files = (await readdir(root, { recursive: true })).filter(file => file.endsWith('.jsonl'))
|
|
return Promise.all(files.map(async (file) => {
|
|
const content = await readFile(join(root, file), 'utf8')
|
|
return { content, header: parseJsonl(content)[0] ?? {} }
|
|
}))
|
|
}
|
|
|
|
describe('headless stream-json snapshots', () => {
|
|
it('prints the original Loader activation error through the assembled one-shot app', async () => {
|
|
const result = await runLoaderSmoke({
|
|
label: 'headless startup activation error snapshot',
|
|
tempDirPrefix: 'headless-snapshot-startup-error-',
|
|
binScript,
|
|
configPath: startupFailureConfigPath,
|
|
binArgs: ['--config', startupFailureConfigPath, '--output-format', 'stream-json', 'unreachable task'],
|
|
tsconfigPath,
|
|
expectedExitCode: 1,
|
|
})
|
|
expect(result.stdout).toBe('')
|
|
await expect(result.stderr).toMatchFileSnapshot(startupFailureExpected)
|
|
}, LOADER_SMOKE_TEST_TIMEOUT_MS)
|
|
|
|
it('retries a transient provider failure through the one-shot app', async () => {
|
|
const prompt = await scenarioPrompt(retryScenarioDir, 'provider-retry')
|
|
const streamExpected = join(retryScenarioDir, 'stream-json.expected.jsonl')
|
|
let runCwd = ''
|
|
const result = await runLoaderSmoke({
|
|
label: 'provider retry headless stream-json snapshot',
|
|
tempDirPrefix: 'headless-snapshot-provider-retry-',
|
|
binScript,
|
|
configPath: retryConfigPath,
|
|
binArgs: ['--config', retryConfigPath, '--output-format', 'stream-json', prompt],
|
|
tsconfigPath,
|
|
env: {
|
|
DSH_SNAPSHOT: 'replay',
|
|
NODE_OPTIONS: [process.env.NODE_OPTIONS, '--disable-warning=ExperimentalWarning'].filter(Boolean).join(' '),
|
|
},
|
|
prepare: (cwd) => { runCwd = cwd },
|
|
inspect: async (cwd) => {
|
|
const logs = await persistedLogs(cwd)
|
|
expect(logs).toHaveLength(1)
|
|
const records = parseJsonl(logs[0]?.content ?? '')
|
|
const retries = records.filter(record => record.type === 'llm/retry')
|
|
expect(retries).toHaveLength(1)
|
|
expect(retries[0]?.data).toMatchObject({
|
|
provider: 'deepseek-official',
|
|
mode: 'normal',
|
|
policyKey: '["normal",1,["RATE_LIMIT"],1,1,0]',
|
|
retry: 1,
|
|
maxRetries: 1,
|
|
delayMs: 1,
|
|
failure: { message: 'snapshot transient failure', code: 'RATE_LIMIT', status: 429 },
|
|
})
|
|
},
|
|
})
|
|
|
|
expect(result.stderr).toBe('')
|
|
const normalized = normalizeHeadlessStream(result.stdout, runCwd)
|
|
if (refreshing) await writeFile(streamExpected, normalized)
|
|
expect(normalized).toBe(await readFile(streamExpected, 'utf8'))
|
|
}, LOADER_SMOKE_TEST_TIMEOUT_MS)
|
|
|
|
it('surfaces actionable missing-credential guidance through the one-shot app', async () => {
|
|
const streamExpected = join(credentialsScenarioDir, 'stream-json.expected.jsonl')
|
|
let runCwd = ''
|
|
const result = await runLoaderSmoke({
|
|
label: 'missing-credential headless stream-json snapshot',
|
|
tempDirPrefix: 'headless-snapshot-missing-credential-',
|
|
binScript,
|
|
configPath: credentialsConfigPath,
|
|
binArgs: ['--config', credentialsConfigPath, '--output-format', 'stream-json', 'say pong'],
|
|
tsconfigPath,
|
|
env: {
|
|
// First-run posture: no key in the environment, none under ./.dsh.
|
|
DEEPSEEK_API_KEY: '',
|
|
DEEPSEEK_BASE_URL: '',
|
|
NODE_OPTIONS: [process.env.NODE_OPTIONS, '--disable-warning=ExperimentalWarning'].filter(Boolean).join(' '),
|
|
},
|
|
// The designed failure surface: the one-shot app reports the failed turn.
|
|
expectedExitCode: 1,
|
|
prepare: (cwd) => { runCwd = cwd },
|
|
})
|
|
|
|
// The guidance names both places a credential can come from, and nothing
|
|
// else: configuration carries the reference, never a literal key.
|
|
expect(result.stderr).toBe(
|
|
'dsh-cli-demo: turn 1 failed at step 1: llm-deepseek: no API key for provider route "deepseek-official";'
|
|
+ ' store DEEPSEEK_API_KEY through the credentials service (the web Models page writes it),'
|
|
+ ' or export DEEPSEEK_API_KEY in the launching environment\n',
|
|
)
|
|
const normalized = normalizeHeadlessStream(result.stdout, runCwd)
|
|
if (refreshing) await writeFile(streamExpected, normalized)
|
|
expect(normalized).toBe(await readFile(streamExpected, 'utf8'))
|
|
}, LOADER_SMOKE_TEST_TIMEOUT_MS)
|
|
|
|
it('logs the model default and a dynamic next-step reasoning effort', async () => {
|
|
const result = await runLoaderSmoke({
|
|
label: 'reasoning effort headless stream-json snapshot',
|
|
tempDirPrefix: 'headless-snapshot-reasoning-effort-',
|
|
binScript,
|
|
configPath: reasoningConfigPath,
|
|
binArgs: ['--config', reasoningConfigPath, '--output-format', 'stream-json', 'prove dynamic reasoning effort'],
|
|
tsconfigPath,
|
|
})
|
|
|
|
expect(result.stderr).toBe('')
|
|
const headers = parseJsonl(result.stdout)
|
|
.map(record => record.event)
|
|
.filter((event): event is JsonObject => (
|
|
event !== null
|
|
&& typeof event === 'object'
|
|
&& !Array.isArray(event)
|
|
&& 'type' in event
|
|
&& event.type === 'request/header'
|
|
))
|
|
.map((event) => {
|
|
const data = event.data as JsonObject
|
|
return (data.header as JsonObject).config
|
|
})
|
|
expect(headers).toMatchInlineSnapshot(`
|
|
[
|
|
{
|
|
"model": "cli-mock",
|
|
"provider": "cli-mock",
|
|
"reasoningEffort": "high",
|
|
},
|
|
{
|
|
"model": "cli-mock",
|
|
"provider": "cli-mock",
|
|
"reasoningEffort": "off",
|
|
},
|
|
]
|
|
`)
|
|
}, LOADER_SMOKE_TEST_TIMEOUT_MS)
|
|
|
|
it('logs and sends the DeepSeek adapter maxTokens default through the one-shot app', async () => {
|
|
const server = await deepseekDefaultsServer()
|
|
try {
|
|
const result = await runLoaderSmoke({
|
|
label: 'DeepSeek adapter defaults headless stream-json snapshot',
|
|
tempDirPrefix: 'headless-snapshot-deepseek-defaults-',
|
|
binScript,
|
|
configPath: deepseekDefaultsConfigPath,
|
|
binArgs: [
|
|
'--config',
|
|
deepseekDefaultsConfigPath,
|
|
'--output-format',
|
|
'stream-json',
|
|
'return the deterministic response',
|
|
],
|
|
tsconfigPath,
|
|
env: {
|
|
// Configuration carries only the reference; the key rides the
|
|
// launching environment, which is the whole credential plane here.
|
|
DEEPSEEK_API_KEY: 'snapshot-key',
|
|
DSH_SNAPSHOT_BASE_URL: server.url,
|
|
NODE_OPTIONS: [process.env.NODE_OPTIONS, '--disable-warning=ExperimentalWarning'].filter(Boolean).join(' '),
|
|
},
|
|
})
|
|
|
|
expect(result.stderr).toBe('')
|
|
expect(server.requests).toHaveLength(1)
|
|
expect(server.requests[0]?.max_tokens).toBe(256_000)
|
|
const header = (parseJsonl(result.stdout)
|
|
.map(record => record.event)
|
|
.find((event): event is JsonObject => (
|
|
event !== null
|
|
&& typeof event === 'object'
|
|
&& !Array.isArray(event)
|
|
&& 'type' in event
|
|
&& event.type === 'request/header'
|
|
))?.data as JsonObject | undefined)?.header as JsonObject | undefined
|
|
expect(header?.config).toMatchInlineSnapshot(`
|
|
{
|
|
"maxTokens": 256000,
|
|
"model": "deepseek-v4-flash",
|
|
"provider": "deepseek-official",
|
|
"reasoningEffort": "off",
|
|
}
|
|
`)
|
|
expect(header?.adapterDefaults).toEqual({
|
|
maxTokens: true,
|
|
reasoningEffort: true,
|
|
})
|
|
} finally {
|
|
await server.close()
|
|
}
|
|
}, LOADER_SMOKE_TEST_TIMEOUT_MS)
|
|
|
|
it('replays the advanced toolchain through the one-shot app', async () => {
|
|
const prompt = await scenarioPrompt(advancedScenarioDir, 'advanced-toolchain')
|
|
const fixtureFiles = [
|
|
advancedSessionFixture,
|
|
join(advancedScenarioDir, 'session.1.jsonl'),
|
|
join(advancedScenarioDir, 'session.2.jsonl'),
|
|
]
|
|
let expectedSessions = await Promise.all(fixtureFiles.map(file => readFile(file, 'utf8')))
|
|
let runCwd = ''
|
|
const result = await runLoaderSmoke({
|
|
label: 'advanced headless stream-json snapshot',
|
|
tempDirPrefix: 'headless-snapshot-advanced-',
|
|
binScript,
|
|
configPath: advancedConfigPath,
|
|
binArgs: ['--config', advancedConfigPath, '--output-format', 'stream-json', prompt],
|
|
tsconfigPath,
|
|
env: {
|
|
DSH_SNAPSHOT: 'replay',
|
|
DSH_SNAPSHOT_FILE: advancedSessionFixture,
|
|
DSH_SNAPSHOT_CHILD_FILES: [
|
|
join(advancedScenarioDir, 'session.1.jsonl'),
|
|
join(advancedScenarioDir, 'session.2.jsonl'),
|
|
].join(delimiter),
|
|
NODE_OPTIONS: [process.env.NODE_OPTIONS, '--disable-warning=ExperimentalWarning'].filter(Boolean).join(' '),
|
|
},
|
|
prepare: (cwd) => { runCwd = cwd },
|
|
inspect: async (cwd) => {
|
|
const logs = await persistedLogs(cwd)
|
|
expect(logs).toHaveLength(3)
|
|
const parents = logs.filter(log => typeof log.header.parentSession !== 'string')
|
|
expect(parents).toHaveLength(1)
|
|
const parent = parents[0]
|
|
if (parent === undefined) throw new Error('headless snapshot did not persist its main session')
|
|
const children = logs.filter(log => typeof log.header.parentSession === 'string')
|
|
.sort((left, right) => Number(left.header.createdAt) - Number(right.header.createdAt))
|
|
const actualSessions = [parent, ...children]
|
|
const actualContext = contextFromLogs(actualSessions.map(log => log.content))
|
|
if (refreshing) {
|
|
const harvested = actualSessions.map((log): HarvestedLog => ({
|
|
id: String(log.header.id),
|
|
createdAt: Number(log.header.createdAt),
|
|
...typeof log.header.parentSession === 'string'
|
|
? { parentSession: log.header.parentSession }
|
|
: {},
|
|
content: log.content,
|
|
}))
|
|
const replacements = refreshFixtureReplacements(harvested, expectedSessions)
|
|
expectedSessions = await Promise.all(actualSessions.map(async (actual, index) => {
|
|
const existing = expectedSessions[index]
|
|
const file = fixtureFiles[index]
|
|
if (existing === undefined || file === undefined) {
|
|
throw new Error(`headless snapshot has no fixture for persisted log ${index}`)
|
|
}
|
|
const stable = tokenizeSessionFixtureCwd(
|
|
stabilizeRefreshLog(actual.content, existing, replacements, actualContext),
|
|
)
|
|
await writeFile(file, stable)
|
|
return stable
|
|
}))
|
|
}
|
|
const expectedContext = contextFromLogs(expectedSessions)
|
|
for (const [index, actual] of actualSessions.entries()) {
|
|
const expected = expectedSessions[index]
|
|
if (expected === undefined) throw new Error(`headless snapshot has no fixture for persisted log ${index}`)
|
|
expect(scrubRequestHeaders(normalizeSessionLog(actual.content, actualContext)))
|
|
.toBe(scrubRequestHeaders(normalizeSessionLog(expected, expectedContext)))
|
|
}
|
|
},
|
|
})
|
|
|
|
expect(result.stderr).toBe('')
|
|
const normalized = normalizeHeadlessStream(result.stdout, runCwd)
|
|
if (refreshing) await writeFile(advancedStreamExpected, normalized)
|
|
expect(normalized).toBe(await readFile(advancedStreamExpected, 'utf8'))
|
|
}, LOADER_SMOKE_TEST_TIMEOUT_MS)
|
|
|
|
it('replays persisted goal tools through the one-shot app', async () => {
|
|
const prompt = await scenarioPrompt(goalScenarioDir, 'goal-tools')
|
|
const streamExpected = join(goalScenarioDir, 'stream-json.expected.jsonl')
|
|
let runCwd = ''
|
|
const result = await runLoaderSmoke({
|
|
label: 'goal tools headless stream-json snapshot',
|
|
tempDirPrefix: 'headless-snapshot-goal-tools-',
|
|
binScript,
|
|
configPath: goalConfigPath,
|
|
binArgs: ['--config', goalConfigPath, '--output-format', 'stream-json', prompt],
|
|
tsconfigPath,
|
|
env: {
|
|
DSH_SNAPSHOT: 'replay',
|
|
DSH_SNAPSHOT_FILE: join(goalScenarioDir, 'session.jsonl'),
|
|
DSH_SNAPSHOT_OVERRIDE: join(goalScenarioDir, 'replay.override.json'),
|
|
NODE_OPTIONS: [process.env.NODE_OPTIONS, '--disable-warning=ExperimentalWarning'].filter(Boolean).join(' '),
|
|
},
|
|
prepare: (cwd) => { runCwd = cwd },
|
|
inspect: async (cwd) => {
|
|
const logs = await persistedLogs(cwd)
|
|
expect(logs).toHaveLength(1)
|
|
const records = parseJsonl(logs[0]?.content ?? '')
|
|
const calls = records.filter(record => record.type === 'tool/call')
|
|
.map(record => (record.data as JsonObject | undefined)?.name)
|
|
expect(calls).toEqual(['update_goal', 'create_goal', 'get_goal'])
|
|
const probeResult = records.find((record) => {
|
|
if (record.type !== 'tool/result') return false
|
|
const data = record.data as JsonObject | undefined
|
|
const message = data?.message as JsonObject | undefined
|
|
const source = message?.source as JsonObject | undefined
|
|
return source?.callId === 'call_goal_probe'
|
|
})
|
|
const probeData = probeResult?.data as JsonObject | undefined
|
|
const probeMessage = probeData?.message as JsonObject | undefined
|
|
const probeContent = probeMessage?.content as JsonObject[] | undefined
|
|
expect(probeContent?.[0]?.isError).toBe(true)
|
|
expect((probeData?.error as JsonObject | undefined)?.code).toBe('GOAL_NOT_FOUND')
|
|
const goalChanges = records.filter((record) => {
|
|
if (record.type !== 'user/message') return false
|
|
const data = record.data as JsonObject | undefined
|
|
const source = data?.source as JsonObject | undefined
|
|
const change = source?.change as JsonObject | undefined
|
|
return source?.kind === 'goal' && change?.kind === 'goal/change'
|
|
})
|
|
expect(goalChanges).toHaveLength(1)
|
|
const data = goalChanges[0]?.data as JsonObject | undefined
|
|
const source = data?.source as JsonObject | undefined
|
|
const change = source?.change as JsonObject | undefined
|
|
const goal = change?.goal as JsonObject | undefined
|
|
expect(change?.operation).toBe('create')
|
|
expect(goal).toMatchObject({
|
|
objective: 'Finish the headless goal-tool snapshot proof',
|
|
phase: 'active',
|
|
maxGoalRounds: 7,
|
|
})
|
|
},
|
|
})
|
|
|
|
expect(result.stderr).toBe('')
|
|
const normalized = normalizeGoalStream(result.stdout, runCwd)
|
|
if (refreshing) await writeFile(streamExpected, normalized)
|
|
expect(normalized).toBe(await readFile(streamExpected, 'utf8'))
|
|
}, LOADER_SMOKE_TEST_TIMEOUT_MS)
|
|
|
|
it('replays two fresh Ralph rounds through the one-shot app', async () => {
|
|
const prompt = await scenarioPrompt(ralphScenarioDir, 'ralph-loop')
|
|
const streamExpected = join(ralphScenarioDir, 'stream-json.expected.jsonl')
|
|
let runCwd = ''
|
|
const result = await runLoaderSmoke({
|
|
label: 'Ralph loop headless stream-json snapshot',
|
|
tempDirPrefix: 'headless-snapshot-ralph-loop-',
|
|
binScript,
|
|
configPath: ralphConfigPath,
|
|
binArgs: ['--config', ralphConfigPath, '--output-format', 'stream-json', prompt],
|
|
tsconfigPath,
|
|
env: {
|
|
DSH_SNAPSHOT: 'replay',
|
|
DSH_SNAPSHOT_FILE: join(ralphScenarioDir, 'session.jsonl'),
|
|
DSH_SNAPSHOT_OVERRIDE: join(ralphScenarioDir, 'replay.override.json'),
|
|
DSH_SNAPSHOT_CHILD_FILES: [
|
|
join(ralphScenarioDir, 'session.1.jsonl'),
|
|
join(ralphScenarioDir, 'session.2.jsonl'),
|
|
].join(delimiter),
|
|
NODE_OPTIONS: [process.env.NODE_OPTIONS, '--disable-warning=ExperimentalWarning'].filter(Boolean).join(' '),
|
|
},
|
|
prepare: (cwd) => { runCwd = cwd },
|
|
inspect: async (cwd) => {
|
|
const logs = await persistedLogs(cwd)
|
|
expect(logs).toHaveLength(3)
|
|
const parent = logs.find(log => typeof log.header.parentSession !== 'string')
|
|
if (parent === undefined) throw new Error('Ralph snapshot did not persist its parent session')
|
|
const parentId = parent.header.id
|
|
expect(typeof parentId).toBe('string')
|
|
const children = logs.filter(log => typeof log.header.parentSession === 'string')
|
|
.sort((left, right) => Number(left.header.createdAt) - Number(right.header.createdAt))
|
|
expect(children).toHaveLength(2)
|
|
expect(children.map(child => child.header.parentSession)).toEqual([parentId, parentId])
|
|
expect(children.map(child => child.header.cwd)).toEqual([parent.header.cwd, parent.header.cwd])
|
|
expect(parent.header.delegationDepth).toBe(0)
|
|
expect(children.map(child => child.header.delegationDepth)).toEqual([1, 1])
|
|
expect(children.map(child => child.header.seedLength)).toEqual([undefined, undefined])
|
|
expect(new Set(children.map(child => child.header.id)).size).toBe(2)
|
|
|
|
const parentRecords = parseJsonl(parent.content)
|
|
const parentCalls = parentRecords.filter(record => record.type === 'tool/call')
|
|
expect(parentCalls.map(record => (record.data as JsonObject | undefined)?.name)).toEqual(['ralph'])
|
|
const parentResult = parentRecords.find(record => record.type === 'tool/result')
|
|
const parentResultData = parentResult?.data as JsonObject | undefined
|
|
const parentMessage = parentResultData?.message as JsonObject | undefined
|
|
const parentContent = parentMessage?.content as JsonObject[] | undefined
|
|
expect(parentContent?.[0]?.isError).toBe(false)
|
|
expect(JSON.stringify(parentContent?.[0]?.content)).toContain('reported completion after 2 rounds')
|
|
|
|
const childRecords = children.map(child => parseJsonl(child.content))
|
|
const childPrompts = childRecords.map((records) => {
|
|
const message = records.find(record => record.type === 'user/message')
|
|
return JSON.stringify((message?.data as JsonObject | undefined)?.content)
|
|
})
|
|
expect(childPrompts[0]).toContain('Ralph round: 1 of 2.')
|
|
expect(childPrompts[0]).toContain('(none — this is the first round)')
|
|
expect(childPrompts[0]).not.toContain('ROUND_ONE_HANDOFF')
|
|
expect(childPrompts[1]).toContain('Ralph round: 2 of 2.')
|
|
expect(childPrompts[1]).toContain('ROUND_ONE_HANDOFF')
|
|
for (const childPrompt of childPrompts) {
|
|
expect(childPrompt).toContain('Prove two fresh Ralph rounds through the shipped headless app.')
|
|
expect(childPrompt).not.toContain('Run a two-round fresh-agent Ralph loop')
|
|
}
|
|
for (const records of childRecords) {
|
|
const calls = records.filter(record => record.type === 'tool/call')
|
|
expect(calls.map(record => (record.data as JsonObject | undefined)?.name))
|
|
.toEqual(['structured_output'])
|
|
}
|
|
},
|
|
})
|
|
|
|
expect(result.stderr).toBe('')
|
|
const normalized = normalizeHeadlessStream(result.stdout, runCwd)
|
|
if (refreshing) await writeFile(streamExpected, normalized)
|
|
expect(normalized).toBe(await readFile(streamExpected, 'utf8'))
|
|
}, LOADER_SMOKE_TEST_TIMEOUT_MS)
|
|
|
|
it('replays persistent PTY tools through the one-shot app', async () => {
|
|
const input = JSON.parse(await readFile(join(ptyScenarioDir, 'input.json'), 'utf8')) as {
|
|
steps?: { op?: unknown; text?: unknown }[]
|
|
}
|
|
const prompt = input.steps?.find(step => step.op === 'prompt')?.text
|
|
if (typeof prompt !== 'string') throw new Error('pty-tools input has no prompt step')
|
|
let expectedSession = await readFile(ptySessionFixture, 'utf8')
|
|
let runCwd = ''
|
|
const result = await runLoaderSmoke({
|
|
label: 'headless persistent PTY snapshot',
|
|
tempDirPrefix: 'headless-snapshot-pty-',
|
|
binScript,
|
|
configPath: ptyConfigPath,
|
|
binArgs: ['--config', ptyConfigPath, '--output-format', 'stream-json', prompt],
|
|
tsconfigPath,
|
|
env: {
|
|
DSH_SNAPSHOT: 'replay',
|
|
DSH_SNAPSHOT_FILE: ptySessionFixture,
|
|
NODE_OPTIONS: [process.env.NODE_OPTIONS, '--disable-warning=ExperimentalWarning'].filter(Boolean).join(' '),
|
|
},
|
|
prepare: (cwd) => { runCwd = cwd },
|
|
inspect: async (cwd) => {
|
|
const logs = await persistedLogs(cwd)
|
|
expect(logs).toHaveLength(1)
|
|
const actual = logs[0]
|
|
if (actual === undefined) throw new Error('headless PTY snapshot did not persist its session')
|
|
const actualContext = contextFromLogs([actual.content])
|
|
if (refreshing) {
|
|
const harvested: HarvestedLog = {
|
|
id: String(actual.header.id),
|
|
createdAt: Number(actual.header.createdAt),
|
|
content: actual.content,
|
|
}
|
|
const replacements = refreshFixtureReplacements([harvested], [expectedSession])
|
|
expectedSession = tokenizeSessionFixtureCwd(
|
|
stabilizeRefreshLog(actual.content, expectedSession, replacements, actualContext),
|
|
)
|
|
await writeFile(ptySessionFixture, expectedSession)
|
|
}
|
|
const expectedContext = contextFromLogs([expectedSession])
|
|
expect(scrubRequestHeaders(normalizeSessionLog(actual.content, actualContext)))
|
|
.toBe(scrubRequestHeaders(normalizeSessionLog(expectedSession, expectedContext)))
|
|
},
|
|
})
|
|
|
|
expect(result.stderr).toBe('')
|
|
const normalized = normalizeHeadlessStream(result.stdout, runCwd)
|
|
if (refreshing) await writeFile(ptyStreamExpected, normalized)
|
|
expect(normalized).toBe(await readFile(ptyStreamExpected, 'utf8'))
|
|
}, LOADER_SMOKE_TEST_TIMEOUT_MS)
|
|
})
|