fix(goal): require explained model blockers

2026-07-20 16:53:10 +08:00
parent 97e144fff6
commit 15640ca697
17 changed files with 289 additions and 287 deletions
--- a/examples/headless-agent/tests/headless.snapshot.ts
+++ b/examples/headless-agent/tests/headless.snapshot.ts
@@ -11,10 +11,12 @@ import { LOADER_SMOKE_TEST_TIMEOUT_MS, runLoaderSmoke } from '@deepseek-ai/dsh-l
 import { describe, expect, it } from 'vitest'

 const snapshotsDir = join(dirname(fileURLToPath(import.meta.url)), 'snapshots')
-const scenarioDir = join(snapshotsDir, 'advanced-toolchain')
-const sessionFixture = join(scenarioDir, 'session.jsonl')
-const streamExpected = join(scenarioDir, 'stream-json.expected.jsonl')
-const configPath = fileURLToPath(new URL('../advanced.cordis.snapshot.yml', import.meta.url))
+const advancedScenarioDir = join(snapshotsDir, 'advanced-toolchain')
+const advancedSessionFixture = join(advancedScenarioDir, 'session.jsonl')
+const advancedStreamExpected = join(advancedScenarioDir, 'stream-json.expected.jsonl')
+const advancedConfigPath = fileURLToPath(new URL('../advanced.cordis.snapshot.yml', import.meta.url))
+const goalScenarioDir = join(snapshotsDir, 'goal-tools')
+const goalConfigPath = fileURLToPath(new URL('../goal.cordis.snapshot.yml', import.meta.url))
 const binScript = fileURLToPath(new URL('../../../packages/examples/cli-demo/src/bin.ts', import.meta.url))
 const tsconfigPath = fileURLToPath(new URL('../../../tsconfig.json', import.meta.url))
 const refreshing = process.env.DSH_SNAPSHOT === 'refresh'
@@ -70,12 +72,36 @@ function normalizeHeadlessStream(rawStdout: string, cwd: string): string {
  return normalizeStdout(`${normalizedRecords.map(record => JSON.stringify(record)).join('\n')}\n`, context)
 }

-async function advancedPrompt(): Promise<string> {
-  const input = JSON.parse(await readFile(join(scenarioDir, 'input.json'), 'utf8')) as {
+/** Zero durable goal timestamps inside both metadata records and rendered XML JSON. */
+function normalizeGoalTimestamps(value: unknown): unknown {
+  if (typeof value === 'string') {
+    return value.replace(/("(?:createdAt|updatedAt|clearedAt)":)\d+/g, '$10')
+  }
+  if (Array.isArray(value)) return value.map(normalizeGoalTimestamps)
+  if (value !== null && typeof value === 'object') {
+    return Object.fromEntries(Object.entries(value).map(([key, item]) => [
+      key,
+      ['createdAt', 'updatedAt', 'clearedAt'].includes(key) && typeof item === 'number'
+        ? 0
+        : normalizeGoalTimestamps(item),
+    ]))
+  }
+  return value
+}
+
+/** Normalize the stream's durable goal timestamps after the shared scrubbers. */
+function normalizeGoalStream(rawStdout: string, cwd: string): string {
+  return parseJsonl(normalizeHeadlessStream(rawStdout, cwd))
+    .map(record => JSON.stringify(normalizeGoalTimestamps(record)))
+    .join('\n') + '\n'
+}
+
+async function scenarioPrompt(dir: string, label: string): Promise<string> {
+  const input = JSON.parse(await readFile(join(dir, 'input.json'), 'utf8')) as {
    steps?: { op?: unknown; text?: unknown }[]
  }
  const prompt = input.steps?.find(step => step.op === 'prompt')?.text
-  if (typeof prompt !== 'string') throw new Error('advanced-toolchain input has no prompt step')
+  if (typeof prompt !== 'string') throw new Error(`${label} input has no prompt step`)
  return prompt
 }

@@ -90,24 +116,27 @@ async function persistedLogs(cwd: string): Promise<PersistedLog[]> {

 describe('headless stream-json snapshots', () => {
  it('replays the advanced toolchain through the one-shot app', async () => {
-    const prompt = await advancedPrompt()
+    const prompt = await scenarioPrompt(advancedScenarioDir, 'advanced-toolchain')
    const expectedSessions = await Promise.all([
-      sessionFixture,
-      join(scenarioDir, 'session.1.jsonl'),
-      join(scenarioDir, 'session.2.jsonl'),
+      advancedSessionFixture,
+      join(advancedScenarioDir, 'session.1.jsonl'),
+      join(advancedScenarioDir, 'session.2.jsonl'),
    ].map(file => readFile(file, 'utf8')))
    let runCwd = ''
    const result = await runLoaderSmoke({
      label: 'advanced headless stream-json snapshot',
      tempDirPrefix: 'headless-snapshot-advanced-',
      binScript,
-      configPath,
-      binArgs: ['--config', configPath, '--output-format', 'stream-json', prompt],
+      configPath: advancedConfigPath,
+      binArgs: ['--config', advancedConfigPath, '--output-format', 'stream-json', prompt],
      tsconfigPath,
      env: {
        DSH_SNAPSHOT: 'replay',
-        DSH_SNAPSHOT_FILE: sessionFixture,
-        DSH_SNAPSHOT_CHILD_FILES: [join(scenarioDir, 'session.1.jsonl'), join(scenarioDir, 'session.2.jsonl')].join(delimiter),
+        DSH_SNAPSHOT_FILE: advancedSessionFixture,
+        DSH_SNAPSHOT_CHILD_FILES: [
+          join(advancedScenarioDir, 'session.1.jsonl'),
+          join(advancedScenarioDir, 'session.2.jsonl'),
+        ].join(delimiter),
        NODE_OPTIONS: [process.env.NODE_OPTIONS, '--disable-warning=ExperimentalWarning'].filter(Boolean).join(' '),
      },
      prepare: (cwd) => { runCwd = cwd },
@@ -134,6 +163,56 @@ describe('headless stream-json snapshots', () => {

    expect(result.stderr).toBe('')
    const normalized = normalizeHeadlessStream(result.stdout, runCwd)
+    if (refreshing) await writeFile(advancedStreamExpected, normalized)
+    expect(normalized).toBe(await readFile(advancedStreamExpected, 'utf8'))
+  }, LOADER_SMOKE_TEST_TIMEOUT_MS)
+
+  it('replays persisted goal tools through the one-shot app', async () => {
+    const prompt = await scenarioPrompt(goalScenarioDir, 'goal-tools')
+    const streamExpected = join(goalScenarioDir, 'stream-json.expected.jsonl')
+    let runCwd = ''
+    const result = await runLoaderSmoke({
+      label: 'goal tools headless stream-json snapshot',
+      tempDirPrefix: 'headless-snapshot-goal-tools-',
+      binScript,
+      configPath: goalConfigPath,
+      binArgs: ['--config', goalConfigPath, '--output-format', 'stream-json', prompt],
+      tsconfigPath,
+      env: {
+        DSH_SNAPSHOT: 'replay',
+        DSH_SNAPSHOT_FILE: join(goalScenarioDir, 'session.jsonl'),
+        DSH_SNAPSHOT_OVERRIDE: join(goalScenarioDir, 'replay.override.json'),
+        NODE_OPTIONS: [process.env.NODE_OPTIONS, '--disable-warning=ExperimentalWarning'].filter(Boolean).join(' '),
+      },
+      prepare: (cwd) => { runCwd = cwd },
+      inspect: async (cwd) => {
+        const logs = await persistedLogs(cwd)
+        expect(logs).toHaveLength(1)
+        const records = parseJsonl(logs[0]?.content ?? '')
+        const calls = records.filter(record => record.type === 'tool/call')
+          .map(record => (record.data as JsonObject | undefined)?.name)
+        expect(calls).toEqual(['create_goal', 'get_goal'])
+        const goalChanges = records.filter((record) => {
+          if (record.type !== 'context/message') return false
+          const data = record.data as JsonObject | undefined
+          const meta = data?.meta as JsonObject | undefined
+          return meta?.kind === 'goal/change'
+        })
+        expect(goalChanges).toHaveLength(1)
+        const data = goalChanges[0]?.data as JsonObject | undefined
+        const meta = data?.meta as JsonObject | undefined
+        const goal = meta?.goal as JsonObject | undefined
+        expect(meta?.operation).toBe('create')
+        expect(goal).toMatchObject({
+          objective: 'Finish the headless goal-tool snapshot proof',
+          phase: 'active',
+          maxGoalRounds: 7,
+        })
+      },
+    })
+
+    expect(result.stderr).toBe('')
+    const normalized = normalizeGoalStream(result.stdout, runCwd)
    if (refreshing) await writeFile(streamExpected, normalized)
    expect(normalized).toBe(await readFile(streamExpected, 'utf8'))
  }, LOADER_SMOKE_TEST_TIMEOUT_MS)
--- a/examples/headless-agent/tests/snapshots/goal-tools/input.json
+++ b/examples/headless-agent/tests/snapshots/goal-tools/input.json
@@ -0,0 +1,9 @@
+{
+  "steps": [
+    {
+      "op": "prompt",
+      "text": "Create a durable goal to finish the snapshot proof, then inspect it."
+    }
+  ]
+}
+
--- a/examples/headless-agent/tests/snapshots/goal-tools/replay.override.json
+++ b/examples/headless-agent/tests/snapshots/goal-tools/replay.override.json
@@ -0,0 +1,33 @@
+[
+  {
+    "kind": "chunks",
+    "chunks": [
+      { "type": "block-start", "index": 0, "blockType": "tool-call" },
+      { "type": "tool-call-delta", "index": 0, "id": "call_goal_create", "name": "create_goal", "argumentsDelta": "{\"objective\":\"Finish the headless goal-tool snapshot proof\",\"max_goal_rounds\":7}" },
+      { "type": "block-end", "index": 0, "block": { "type": "tool-call", "id": "call_goal_create", "name": "create_goal", "arguments": "{\"objective\":\"Finish the headless goal-tool snapshot proof\",\"max_goal_rounds\":7}" } },
+      { "type": "usage", "usage": { "inputTokens": 20, "outputTokens": 8 } },
+      { "type": "finish", "reason": { "kind": "tool-calls" } }
+    ]
+  },
+  {
+    "kind": "chunks",
+    "chunks": [
+      { "type": "block-start", "index": 0, "blockType": "tool-call" },
+      { "type": "tool-call-delta", "index": 0, "id": "call_goal_get", "name": "get_goal", "argumentsDelta": "{}" },
+      { "type": "block-end", "index": 0, "block": { "type": "tool-call", "id": "call_goal_get", "name": "get_goal", "arguments": "{}" } },
+      { "type": "usage", "usage": { "inputTokens": 30, "outputTokens": 4 } },
+      { "type": "finish", "reason": { "kind": "tool-calls" } }
+    ]
+  },
+  {
+    "kind": "chunks",
+    "chunks": [
+      { "type": "block-start", "index": 0, "blockType": "text" },
+      { "type": "text-delta", "index": 0, "text": "GOAL READY" },
+      { "type": "block-end", "index": 0, "block": { "type": "text", "text": "GOAL READY" } },
+      { "type": "usage", "usage": { "inputTokens": 35, "outputTokens": 2 } },
+      { "type": "finish", "reason": { "kind": "stop" } }
+    ]
+  }
+]
+
--- a/examples/headless-agent/tests/snapshots/goal-tools/stream-json.expected.jsonl
+++ b/examples/headless-agent/tests/snapshots/goal-tools/stream-json.expected.jsonl
@@ -0,0 +1,34 @@
+{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"turn/start","seq":0,"time":0,"data":{"turn":1,"trigger":{"kind":"message","source":{"kind":"user"}}}}}
+{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"user/message","seq":1,"time":0,"data":{"content":[{"type":"text","text":"Create a durable goal to finish the snapshot proof, then inspect it."}],"source":{"kind":"user"}},"surfaceOp":"append"}}
+{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"step/start","seq":2,"time":0,"data":{"turn":1,"step":1}}}
+{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"request/header","seq":3,"time":0,"data":{"header":{"config":{"provider":"deepseek","model":"deepseek-v4-flash"},"system":"{{system}}","tools":"{{tools}}"},"reason":"initial"}}}
+{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"assistant/chunk","seq":4,"time":0,"data":{"turn":1,"step":1,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}}}}
+{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"assistant/chunk","seq":5,"time":0,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":0,"id":"call_goal_create","name":"create_goal","argumentsDelta":"{\"objective\":\"Finish the headless goal-tool snapshot proof\",\"max_goal_rounds\":7}"}}}}
+{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"assistant/chunk","seq":6,"time":0,"data":{"turn":1,"step":1,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"call_goal_create","name":"create_goal","arguments":"{\"objective\":\"Finish the headless goal-tool snapshot proof\",\"max_goal_rounds\":7}"}}}}}
+{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"assistant/chunk","seq":7,"time":0,"data":{"turn":1,"step":1,"chunk":{"type":"usage","usage":{"inputTokens":20,"outputTokens":8}}}}}
+{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"assistant/chunk","seq":8,"time":0,"data":{"turn":1,"step":1,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}}}
+{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"assistant/message","seq":9,"time":0,"data":{"turn":1,"step":1,"content":[{"type":"tool-call","id":"call_goal_create","name":"create_goal","arguments":"{\"objective\":\"Finish the headless goal-tool snapshot proof\",\"max_goal_rounds\":7}"}],"provenance":{"provider":"deepseek","model":"deepseek-v4-flash"},"usage":{"inputTokens":20,"outputTokens":8}},"sourceEventSeqs":[4,5,6,7,8],"surfaceOp":"append"}}
+{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"tool/call","seq":10,"time":0,"data":{"turn":1,"step":1,"callId":"call_goal_create","name":"create_goal","arguments":"{\"objective\":\"Finish the headless goal-tool snapshot proof\",\"max_goal_rounds\":7}"}}}
+{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"tool/result","seq":11,"time":0,"data":{"turn":1,"step":1,"callId":"call_goal_create","content":[{"type":"text","text":"{\"goal\":{\"id\":\"goal-{{sessionId}}\",\"revision\":1,\"objective\":\"Finish the headless goal-tool snapshot proof\",\"phase\":\"active\",\"roundsStarted\":0,\"maxGoalRounds\":7},\"activation\":\"armed\"}"}],"isError":false},"sourceEventSeqs":[10],"surfaceOp":"append"}}
+{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"context/message","seq":12,"time":0,"data":{"content":[{"type":"text","text":"<goal_state>{\"goal\":{\"id\":\"goal-{{sessionId}}\",\"revision\":1,\"objective\":\"Finish the headless goal-tool snapshot proof\",\"phase\":\"active\",\"maxGoalRounds\":7},\"roundsStarted\":0,\"createdAt\":0,\"updatedAt\":0}</goal_state>"}],"source":{"kind":"goal","goalId":"goal-{{sessionId}}","revision":1,"round":0},"envelope":"raw","meta":{"kind":"goal/change","version":1,"operation":"create","goal":{"id":"goal-{{sessionId}}","revision":1,"objective":"Finish the headless goal-tool snapshot proof","phase":"active","maxGoalRounds":7},"roundsStarted":0,"createdAt":0,"updatedAt":0}},"surfaceOp":"append"}}
+{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"step/end","seq":13,"time":0,"data":{"turn":1,"step":1}}}
+{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"step/start","seq":14,"time":0,"data":{"turn":1,"step":2}}}
+{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"assistant/chunk","seq":15,"time":0,"data":{"turn":1,"step":2,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}}}}
+{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"assistant/chunk","seq":16,"time":0,"data":{"turn":1,"step":2,"chunk":{"type":"tool-call-delta","index":0,"id":"call_goal_get","name":"get_goal","argumentsDelta":"{}"}}}}
+{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"assistant/chunk","seq":17,"time":0,"data":{"turn":1,"step":2,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"call_goal_get","name":"get_goal","arguments":"{}"}}}}}
+{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"assistant/chunk","seq":18,"time":0,"data":{"turn":1,"step":2,"chunk":{"type":"usage","usage":{"inputTokens":30,"outputTokens":4}}}}}
+{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"assistant/chunk","seq":19,"time":0,"data":{"turn":1,"step":2,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}}}
+{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"assistant/message","seq":20,"time":0,"data":{"turn":1,"step":2,"content":[{"type":"tool-call","id":"call_goal_get","name":"get_goal","arguments":"{}"}],"provenance":{"provider":"deepseek","model":"deepseek-v4-flash"},"usage":{"inputTokens":30,"outputTokens":4}},"sourceEventSeqs":[15,16,17,18,19],"surfaceOp":"append"}}
+{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"tool/call","seq":21,"time":0,"data":{"turn":1,"step":2,"callId":"call_goal_get","name":"get_goal","arguments":"{}"}}}
+{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"tool/result","seq":22,"time":0,"data":{"turn":1,"step":2,"callId":"call_goal_get","content":[{"type":"text","text":"{\"goal\":{\"id\":\"goal-{{sessionId}}\",\"revision\":1,\"objective\":\"Finish the headless goal-tool snapshot proof\",\"phase\":\"active\",\"roundsStarted\":0,\"maxGoalRounds\":7},\"activation\":\"armed\"}"}],"isError":false},"sourceEventSeqs":[21],"surfaceOp":"append"}}
+{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"step/end","seq":23,"time":0,"data":{"turn":1,"step":2}}}
+{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"step/start","seq":24,"time":0,"data":{"turn":1,"step":3}}}
+{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"assistant/chunk","seq":25,"time":0,"data":{"turn":1,"step":3,"chunk":{"type":"block-start","index":0,"blockType":"text"}}}}
+{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"assistant/chunk","seq":26,"time":0,"data":{"turn":1,"step":3,"chunk":{"type":"text-delta","index":0,"text":"GOAL READY"}}}}
+{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"assistant/chunk","seq":27,"time":0,"data":{"turn":1,"step":3,"chunk":{"type":"block-end","index":0,"block":{"type":"text","text":"GOAL READY"}}}}}
+{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"assistant/chunk","seq":28,"time":0,"data":{"turn":1,"step":3,"chunk":{"type":"usage","usage":{"inputTokens":35,"outputTokens":2}}}}}
+{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"assistant/chunk","seq":29,"time":0,"data":{"turn":1,"step":3,"chunk":{"type":"finish","reason":{"kind":"stop"}}}}}
+{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"assistant/message","seq":30,"time":0,"data":{"turn":1,"step":3,"content":[{"type":"text","text":"GOAL READY"}],"provenance":{"provider":"deepseek","model":"deepseek-v4-flash"},"usage":{"inputTokens":35,"outputTokens":2}},"sourceEventSeqs":[25,26,27,28,29],"surfaceOp":"append"}}
+{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"step/end","seq":31,"time":0,"data":{"turn":1,"step":3}}}
+{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"turn/end","seq":32,"time":0,"data":{"turn":1,"reason":{"kind":"completed"}}}}
+{"type":"result","success":true,"sessionId":"{{sessionId}}","turn":1,"result":"GOAL READY","reason":{"kind":"completed"},"usage":{"inputTokens":85,"outputTokens":14}}