Coder/deepseek-harness
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

test(ralph): snapshot fresh-agent rounds in headless app

Browse Source
This commit is contained in:
Tianyi Cui
2026-07-20 18:18:08 +08:00
parent 94a6a95c8e
commit 9f6f87cf69
11 changed files with 157 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
examples/headless-agent/ralph.cordis.snapshot.yml Normal file
View File

@@ -0,0 +1,12 @@
# Replay counterpart to cordis.yml for the shipped Ralph-loop snapshot.
- id: base
name: '@cordisjs/plugin-include'
config:
path: ./cordis.yml
patches:
- id: llm-deepseek
name: '@deepseek-ai/dsh-llm-deepseek'
disabled: true
- insert:
- id: llm-replay
name: '@deepseek-ai/dsh-llm-replay'

75
examples/headless-agent/tests/headless.snapshot.ts
View File

@@ -17,6 +17,8 @@ const advancedStreamExpected = join(advancedScenarioDir, 'stream-json.expected.j
const advancedConfigPath = fileURLToPath(new URL('../advanced.cordis.snapshot.yml', import.meta.url))
const goalScenarioDir = join(snapshotsDir, 'goal-tools')
const goalConfigPath = fileURLToPath(new URL('../goal.cordis.snapshot.yml', import.meta.url))
const ralphScenarioDir = join(snapshotsDir, 'ralph-loop')
const ralphConfigPath = fileURLToPath(new URL('../ralph.cordis.snapshot.yml', import.meta.url))
const binScript = fileURLToPath(new URL('../../../packages/examples/cli-demo/src/bin.ts', import.meta.url))
const tsconfigPath = fileURLToPath(new URL('../../../tsconfig.json', import.meta.url))
const refreshing = process.env.DSH_SNAPSHOT === 'refresh'
@@ -216,4 +218,77 @@ describe('headless stream-json snapshots', () => {
if (refreshing) await writeFile(streamExpected, normalized)
expect(normalized).toBe(await readFile(streamExpected, 'utf8'))
}, LOADER_SMOKE_TEST_TIMEOUT_MS)
it('replays two fresh Ralph rounds through the one-shot app', async () => {
const prompt = await scenarioPrompt(ralphScenarioDir, 'ralph-loop')
const streamExpected = join(ralphScenarioDir, 'stream-json.expected.jsonl')
let runCwd = ''
const result = await runLoaderSmoke({
label: 'Ralph loop headless stream-json snapshot',
tempDirPrefix: 'headless-snapshot-ralph-loop-',
binScript,
configPath: ralphConfigPath,
binArgs: ['--config', ralphConfigPath, '--output-format', 'stream-json', prompt],
tsconfigPath,
env: {
DSH_SNAPSHOT: 'replay',
DSH_SNAPSHOT_FILE: join(ralphScenarioDir, 'session.jsonl'),
DSH_SNAPSHOT_OVERRIDE: join(ralphScenarioDir, 'replay.override.json'),
DSH_SNAPSHOT_CHILD_FILES: [
join(ralphScenarioDir, 'session.1.jsonl'),
join(ralphScenarioDir, 'session.2.jsonl'),
].join(delimiter),
NODE_OPTIONS: [process.env.NODE_OPTIONS, '--disable-warning=ExperimentalWarning'].filter(Boolean).join(' '),
},
prepare: (cwd) => { runCwd = cwd },
inspect: async (cwd) => {
const logs = await persistedLogs(cwd)
expect(logs).toHaveLength(3)
const parent = logs.find(log => typeof log.header.parentSession !== 'string')
if (parent === undefined) throw new Error('Ralph snapshot did not persist its parent session')
const parentId = parent.header.id
expect(typeof parentId).toBe('string')
const children = logs.filter(log => typeof log.header.parentSession === 'string')
.sort((left, right) => Number(left.header.createdAt) - Number(right.header.createdAt))
expect(children).toHaveLength(2)
expect(children.map(child => child.header.parentSession)).toEqual([parentId, parentId])
expect(children.map(child => child.header.cwd)).toEqual([parent.header.cwd, parent.header.cwd])
expect(children.map(child => child.header.seedLength)).toEqual([undefined, undefined])
expect(new Set(children.map(child => child.header.id)).size).toBe(2)
const parentRecords = parseJsonl(parent.content)
const parentCalls = parentRecords.filter(record => record.type === 'tool/call')
expect(parentCalls.map(record => (record.data as JsonObject | undefined)?.name)).toEqual(['ralph'])
const parentResult = parentRecords.find(record => record.type === 'tool/result')
const parentResultData = parentResult?.data as JsonObject | undefined
expect(parentResultData?.isError).toBe(false)
expect(JSON.stringify(parentResultData?.content)).toContain('reported completion after 2 rounds')
const childRecords = children.map(child => parseJsonl(child.content))
const childPrompts = childRecords.map((records) => {
const message = records.find(record => record.type === 'user/message')
return JSON.stringify((message?.data as JsonObject | undefined)?.content)
})
expect(childPrompts[0]).toContain('Ralph round: 1 of 2.')
expect(childPrompts[0]).toContain('(none — this is the first round)')
expect(childPrompts[0]).not.toContain('ROUND_ONE_HANDOFF')
expect(childPrompts[1]).toContain('Ralph round: 2 of 2.')
expect(childPrompts[1]).toContain('ROUND_ONE_HANDOFF')
for (const childPrompt of childPrompts) {
expect(childPrompt).toContain('Prove two fresh Ralph rounds through the shipped headless app.')
expect(childPrompt).not.toContain('Run a two-round fresh-agent Ralph loop')
}
for (const records of childRecords) {
const calls = records.filter(record => record.type === 'tool/call')
expect(calls.map(record => (record.data as JsonObject | undefined)?.name))
.toEqual(['structured_output'])
}
},
})
expect(result.stderr).toBe('')
const normalized = normalizeHeadlessStream(result.stdout, runCwd)
if (refreshing) await writeFile(streamExpected, normalized)
expect(normalized).toBe(await readFile(streamExpected, 'utf8'))
}, LOADER_SMOKE_TEST_TIMEOUT_MS)
})

8
examples/headless-agent/tests/snapshots/ralph-loop/input.json Normal file
View File

@@ -0,0 +1,8 @@
{
"steps": [
{
"op": "prompt",
"text": "Run a two-round fresh-agent Ralph loop to prove the shipped headless integration."
}
]
}

22
examples/headless-agent/tests/snapshots/ralph-loop/replay.override.json Normal file
View File

@@ -0,0 +1,22 @@
[
{
"kind": "chunks",
"chunks": [
{ "type": "block-start", "index": 0, "blockType": "tool-call" },
{ "type": "tool-call-delta", "index": 0, "id": "call_ralph", "name": "ralph", "argumentsDelta": "{\"objective\":\"Prove two fresh Ralph rounds through the shipped headless app.\",\"maxRounds\":2}" },
{ "type": "block-end", "index": 0, "block": { "type": "tool-call", "id": "call_ralph", "name": "ralph", "arguments": "{\"objective\":\"Prove two fresh Ralph rounds through the shipped headless app.\",\"maxRounds\":2}" } },
{ "type": "usage", "usage": { "inputTokens": 20, "outputTokens": 8 } },
{ "type": "finish", "reason": { "kind": "tool-calls" } }
]
},
{
"kind": "chunks",
"chunks": [
{ "type": "block-start", "index": 0, "blockType": "text" },
{ "type": "text-delta", "index": 0, "text": "RALPH SNAPSHOT COMPLETE" },
{ "type": "block-end", "index": 0, "block": { "type": "text", "text": "RALPH SNAPSHOT COMPLETE" } },
{ "type": "usage", "usage": { "inputTokens": 30, "outputTokens": 4 } },
{ "type": "finish", "reason": { "kind": "stop" } }
]
}
]

6
examples/headless-agent/tests/snapshots/ralph-loop/session.1.jsonl Normal file
View File

@@ -0,0 +1,6 @@
{"type":"session","version":0,"id":"42222222-2222-4222-8222-222222222222","createdAt":1783951001000,"cwd":"/tmp/ralph-headless","parentSession":"41111111-1111-4111-8111-111111111111"}
{"type":"assistant/chunk","seq":0,"time":1783951001001,"data":{"turn":1,"step":1,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}}}
{"type":"assistant/chunk","seq":1,"time":1783951001002,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":0,"id":"round-one-report","name":"structured_output","argumentsDelta":"{\"status\":\"continue\",\"summary\":\"ROUND_ONE_HANDOFF\",\"evidence\":[\"Round one inspected the workspace.\"],\"nextSteps\":[\"Finish the snapshot objective.\"],\"blocker\":\"\"}"}}}
{"type":"assistant/chunk","seq":2,"time":1783951001003,"data":{"turn":1,"step":1,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"round-one-report","name":"structured_output","arguments":"{\"status\":\"continue\",\"summary\":\"ROUND_ONE_HANDOFF\",\"evidence\":[\"Round one inspected the workspace.\"],\"nextSteps\":[\"Finish the snapshot objective.\"],\"blocker\":\"\"}"}}}}
{"type":"assistant/chunk","seq":3,"time":1783951001004,"data":{"turn":1,"step":1,"chunk":{"type":"usage","usage":{"inputTokens":30,"outputTokens":12}}}}
{"type":"assistant/chunk","seq":4,"time":1783951001005,"data":{"turn":1,"step":1,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}}

6
examples/headless-agent/tests/snapshots/ralph-loop/session.2.jsonl Normal file
View File

@@ -0,0 +1,6 @@
{"type":"session","version":0,"id":"43333333-3333-4333-8333-333333333333","createdAt":1783951002000,"cwd":"/tmp/ralph-headless","parentSession":"41111111-1111-4111-8111-111111111111"}
{"type":"assistant/chunk","seq":0,"time":1783951002001,"data":{"turn":1,"step":1,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}}}
{"type":"assistant/chunk","seq":1,"time":1783951002002,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":0,"id":"round-two-report","name":"structured_output","argumentsDelta":"{\"status\":\"complete\",\"summary\":\"The Ralph snapshot objective is complete.\",\"evidence\":[\"Two fresh rounds completed through the shipped app.\"],\"nextSteps\":[],\"blocker\":\"\"}"}}}
{"type":"assistant/chunk","seq":2,"time":1783951002003,"data":{"turn":1,"step":1,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"round-two-report","name":"structured_output","arguments":"{\"status\":\"complete\",\"summary\":\"The Ralph snapshot objective is complete.\",\"evidence\":[\"Two fresh rounds completed through the shipped app.\"],\"nextSteps\":[],\"blocker\":\"\"}"}}}}
{"type":"assistant/chunk","seq":3,"time":1783951002004,"data":{"turn":1,"step":1,"chunk":{"type":"usage","usage":{"inputTokens":40,"outputTokens":12}}}}
{"type":"assistant/chunk","seq":4,"time":1783951002005,"data":{"turn":1,"step":1,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}}

1
examples/headless-agent/tests/snapshots/ralph-loop/session.jsonl Normal file
View File

@@ -0,0 +1 @@
{"type":"session","version":0,"id":"41111111-1111-4111-8111-111111111111","createdAt":1783951000000,"cwd":"/tmp/ralph-headless"}

23
examples/headless-agent/tests/snapshots/ralph-loop/stream-json.expected.jsonl Normal file
View File

@@ -0,0 +1,23 @@
{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"turn/start","seq":0,"time":0,"data":{"turn":1,"trigger":{"kind":"message","source":{"kind":"user"}}}}}
{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"user/message","seq":1,"time":0,"data":{"content":[{"type":"text","text":"Run a two-round fresh-agent Ralph loop to prove the shipped headless integration."}],"source":{"kind":"user"}},"surfaceOp":"append"}}
{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"step/start","seq":2,"time":0,"data":{"turn":1,"step":1}}}
{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"request/header","seq":3,"time":0,"data":{"header":{"config":{"provider":"deepseek","model":"deepseek-v4-flash"},"system":"{{system}}","tools":"{{tools}}"},"reason":"initial"}}}
{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"assistant/chunk","seq":4,"time":0,"data":{"turn":1,"step":1,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}}}}
{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"assistant/chunk","seq":5,"time":0,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":0,"id":"call_ralph","name":"ralph","argumentsDelta":"{\"objective\":\"Prove two fresh Ralph rounds through the shipped headless app.\",\"maxRounds\":2}"}}}}
{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"assistant/chunk","seq":6,"time":0,"data":{"turn":1,"step":1,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"call_ralph","name":"ralph","arguments":"{\"objective\":\"Prove two fresh Ralph rounds through the shipped headless app.\",\"maxRounds\":2}"}}}}}
{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"assistant/chunk","seq":7,"time":0,"data":{"turn":1,"step":1,"chunk":{"type":"usage","usage":{"inputTokens":20,"outputTokens":8}}}}}
{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"assistant/chunk","seq":8,"time":0,"data":{"turn":1,"step":1,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}}}
{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"assistant/message","seq":9,"time":0,"data":{"turn":1,"step":1,"content":[{"type":"tool-call","id":"call_ralph","name":"ralph","arguments":"{\"objective\":\"Prove two fresh Ralph rounds through the shipped headless app.\",\"maxRounds\":2}"}],"provenance":{"provider":"deepseek","model":"deepseek-v4-flash"},"usage":{"inputTokens":20,"outputTokens":8}},"sourceEventSeqs":[4,5,6,7,8],"surfaceOp":"append"}}
{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"tool/call","seq":10,"time":0,"data":{"turn":1,"step":1,"callId":"call_ralph","name":"ralph","arguments":"{\"objective\":\"Prove two fresh Ralph rounds through the shipped headless app.\",\"maxRounds\":2}"}}}
{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"tool/result","seq":11,"time":0,"data":{"turn":1,"step":1,"callId":"call_ralph","content":[{"type":"text","text":"Ralph worker reported completion after 2 rounds.\nFinal report:\n{\n \"status\": \"complete\",\n \"summary\": \"The Ralph snapshot objective is complete.\",\n \"evidence\": [\n \"Two fresh rounds completed through the shipped app.\"\n ],\n \"nextSteps\": [],\n \"blocker\": \"\"\n}"}],"isError":false},"sourceEventSeqs":[10],"surfaceOp":"append"}}
{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"step/end","seq":12,"time":0,"data":{"turn":1,"step":1}}}
{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"step/start","seq":13,"time":0,"data":{"turn":1,"step":2}}}
{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"assistant/chunk","seq":14,"time":0,"data":{"turn":1,"step":2,"chunk":{"type":"block-start","index":0,"blockType":"text"}}}}
{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"assistant/chunk","seq":15,"time":0,"data":{"turn":1,"step":2,"chunk":{"type":"text-delta","index":0,"text":"RALPH SNAPSHOT COMPLETE"}}}}
{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"assistant/chunk","seq":16,"time":0,"data":{"turn":1,"step":2,"chunk":{"type":"block-end","index":0,"block":{"type":"text","text":"RALPH SNAPSHOT COMPLETE"}}}}}
{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"assistant/chunk","seq":17,"time":0,"data":{"turn":1,"step":2,"chunk":{"type":"usage","usage":{"inputTokens":30,"outputTokens":4}}}}}
{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"assistant/chunk","seq":18,"time":0,"data":{"turn":1,"step":2,"chunk":{"type":"finish","reason":{"kind":"stop"}}}}}
{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"assistant/message","seq":19,"time":0,"data":{"turn":1,"step":2,"content":[{"type":"text","text":"RALPH SNAPSHOT COMPLETE"}],"provenance":{"provider":"deepseek","model":"deepseek-v4-flash"},"usage":{"inputTokens":30,"outputTokens":4}},"sourceEventSeqs":[14,15,16,17,18],"surfaceOp":"append"}}
{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"step/end","seq":20,"time":0,"data":{"turn":1,"step":2}}}
{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"turn/end","seq":21,"time":0,"data":{"turn":1,"reason":{"kind":"completed"}}}}
{"type":"result","success":true,"sessionId":"{{sessionId}}","turn":1,"result":"RALPH SNAPSHOT COMPLETE","reason":{"kind":"completed"},"usage":{"inputTokens":50,"outputTokens":12}}