Merge master into codex/simp-session-log-representation

packages/core/session/src/index.ts

@@ -13,9 +13,9 @@ import { scopeOf, scopeTarget } from '@deepseek-ai/dsh-scope'
 import type { Scoped } from '@deepseek-ai/dsh-scope'
 import type { ContentBlock, Message, MessageSource } from '@deepseek-ai/dsh-llm'
 import { SESSION_FORMAT_VERSION, SessionId } from './types.ts'
-import type { CreateSessionOptions, EpochHeader, SessionEvent, SessionEventMap, SessionEventType, SessionHeader, SurfaceIntent, SurfaceEventType } from './types.ts'
+import type { ContextEnvelope, CreateSessionOptions, EpochHeader, SessionEvent, SessionEventMap, SessionEventType, SessionHeader, SurfaceIntent, SurfaceEventType } from './types.ts'
 import { snapshotJsonValue } from './json.ts'
-import { SurfaceManager, isSurfaceEligibleType } from './surface.ts'
+import { SurfaceManager } from './surface.ts'
 import { foldRequestHeader } from './request-header.ts'
 
 export * from './types.ts'
@@ -131,43 +131,6 @@ function snapshotSessionHeader(id: SessionId, source?: SessionHeader): SessionHe
   return deepFreeze(record as unknown as SessionHeader)
 }
 
-/** Validate the runtime shape of surface metadata after its JSON snapshot. */
-function assertSurfaceMetadataShape(
-  type: string,
-  surfaceOp: unknown,
-  sourceEventSeqs: unknown,
-): void {
-  const eligible = isSurfaceEligibleType(type)
-  if (!eligible) {
-    if (surfaceOp !== undefined || sourceEventSeqs !== undefined) {
-      throw new Error(`session event "${type}" is not surface-eligible and cannot carry surface metadata`)
-    }
-    return
-  }
-  if (surfaceOp === undefined) {
-    throw new Error(`session event "${type}" is surface-eligible and requires a surfaceOp marker`)
-  }
-  if (surfaceOp !== 'append') {
-    if (surfaceOp === null || typeof surfaceOp !== 'object' || Array.isArray(surfaceOp)) {
-      throw new Error(`session event "${type}" carries an invalid surfaceOp`)
-    }
-    const op = surfaceOp as Record<string, unknown>
-    const keys = Object.keys(op)
-    if (keys.length !== 3 || !Object.hasOwn(op, 'op') || !Object.hasOwn(op, 'start') || !Object.hasOwn(op, 'end')
-      || op['op'] !== 'replace'
-      || typeof op['start'] !== 'number' || !Number.isSafeInteger(op['start']) || op['start'] < 0
-      || typeof op['end'] !== 'number' || !Number.isSafeInteger(op['end']) || op['end'] < 0) {
-      throw new Error(`session event "${type}" carries an invalid replace surfaceOp`)
-    }
-  }
-  if (sourceEventSeqs !== undefined) {
-    if (!Array.isArray(sourceEventSeqs)
-      || sourceEventSeqs.some(seq => typeof seq !== 'number' || !Number.isSafeInteger(seq) || seq < 0)) {
-      throw new Error(`session event "${type}" sourceEventSeqs must contain non-negative safe integers`)
-    }
-  }
-}
-
 /** Validate the fixed event envelope after one-pass JSON materialization. */
 function assertSessionEventEnvelope(value: Record<string, unknown>, index: number): asserts value is SessionEvent {
   const event = value
@@ -241,6 +204,22 @@ interface SessionEntry {
 /** Store attachment for the append path; module-private to keep Session store-agnostic publicly. */
 const attachments = new WeakMap<Session, SessionEntry>()
 
+/**
+ * Render one context contribution exactly as it will appear in model history.
+ * @param content - content blocks supplied by the context producer.
+ * @param source - attribution used by the canonical context envelope.
+ * @param envelope - canonical tagged framing or caller-owned raw framing.
+ * @returns a detached block list ready for the derived model transcript.
+ */
+export function renderContextContent(
+  content: ContentBlock[],
+  source: MessageSource,
+  envelope: ContextEnvelope = 'context',
+): ContentBlock[] {
+  const cloned = structuredClone(content)
+  return envelope === 'raw' ? cloned : renderTagged('context', cloned, source)
+}
+
 /**
  * An event-sourced session: an append-only log of {@link SessionEvent}s.
  *
@@ -249,13 +228,15 @@ const attachments = new WeakMap<Session, SessionEntry>()
  */
 export class Session {
   private log: SessionEvent[] = []
+  /** Incremental acceptance state, kept separate from the public lazy view. */
+  private readonly surfaceValidator = new SurfaceManager(this.log)
 
   /**
    * Derived surface — a cached order of message-producing event sequences.
    * Lazily rebuilt from `surfaceOp` markers in the log; processes only new
    * events (delta) on each access — the log is append-only, so prior events
    * never change.
-   * `append`. Undefined until first accessed (including after fork/seed).
+   * Undefined until first accessed (including after fork/seed).
    */
   private _surface: SurfaceManager | undefined
 
@@ -284,7 +265,7 @@ export class Session {
       // `seq = log.length` contract the whole system relies on). Without this,
       // a bad seed would surface only later as a backend rejection or a silent
       // divergence between the live log and disk.
-      this.log = Array.from(seed, (source, index) => {
+      for (const [index, source] of seed.entries()) {
         // The seed is a persistence/replay boundary: validate and detach the
         // complete event in one lossless-JSON pass.
         const snapshot = snapshotJsonValue(source)
@@ -296,20 +277,16 @@ export class Session {
         if (snapshot.seq !== index) {
           throw new Error(`seed event at index ${index} has seq ${snapshot.seq} (expected ${index}); seed must be contiguous from 0`)
         }
-        // Surface-eligible events MUST carry a surfaceOp marker — the surface is
-        // the sole source of derived history, so a marker-less message event
-        // would load fine yet vanish from deriveMessages(). `append` enforces
-        // this at compile time via its typed overload; a seed arrives as raw
-        // SessionEvent[] (replay/fork/load), bypassing that, so re-check at
-        // runtime here rather than silently resuming with empty history.
-        const structural = snapshot as SessionEvent & { surfaceOp?: unknown; sourceEventSeqs?: unknown }
+        // A seed is accepted incrementally through the same transition as a
+        // live append and a full-log fold. The candidate is planned before it
+        // enters `log`, so a failure cannot partially mutate the surface.
         try {
-          assertSurfaceMetadataShape(snapshot.type, structural.surfaceOp, structural.sourceEventSeqs)
+          this.surfaceValidator.validateNext(snapshot)
         } catch (error: unknown) {
           throw new Error(`invalid seed event at index ${index}: ${error instanceof Error ? error.message : 'invalid surface metadata'}`)
         }
-        return deepFreeze(snapshot)
-      })
+        this.log.push(deepFreeze(snapshot))
+      }
     }
     this.header = snapshotSessionHeader(id, header)
   }
@@ -356,7 +333,10 @@ export class Session {
    * @throws if `data` or surface metadata is not losslessly JSON-serializable
    *   (BigInt, function, symbol, undefined, negative zero, non-finite number,
    *   circular reference, sparse array, or an exotic object such as
-   *   Map/Set/Date/class instance). One recursive pass reads, validates, and
+   *   Map/Set/Date/class instance), or when the candidate violates the
+   *   canonical surface contract (marker shape and eligibility, unique
+   *   earlier provenance, positional replacement validity, and complete
+   *   shadowed-node coverage). One recursive pass reads, validates, and
    *   copies each nested value once, so a stateful getter cannot supply one value
    *   to validation and another to storage. The event log is the durable source
    *   of truth, so a bad event fails at the append site rather than later during
@@ -383,25 +363,21 @@ export class Session {
     if (surfaceMetadataSnapshot === undefined) {
       throw new Error(`session event "${type}" carries non-JSON-serializable surface metadata`)
     }
-    assertSurfaceMetadataShape(
-      type,
-      (surfaceMetadataSnapshot as { surfaceOp?: unknown }).surfaceOp,
-      (surfaceMetadataSnapshot as { sourceEventSeqs?: unknown }).sourceEventSeqs,
-    )
-
     const entry = attachments.get(this)
     if (entry?.appending) {
       throw new Error('session append cannot reenter while another append is being published')
     }
+    const event = deepFreeze({
+      type,
+      seq: this.log.length,
+      time: Date.now(),
+      data: dataSnapshot,
+      ...(surfaceMetadataSnapshot as { surfaceOp?: unknown; sourceEventSeqs?: unknown }),
+    } as unknown as SessionEvent<T>)
+    this.surfaceValidator.validateNext(event as SessionEvent)
+
     if (entry !== undefined) entry.appending = true
     try {
-      const event = deepFreeze({
-        type,
-        seq: this.log.length,
-        time: Date.now(),
-        data: dataSnapshot,
-        ...surfaceMetadataSnapshot,
-      } as unknown as SessionEvent<T>)
       let callbacks: SessionCallback[] | undefined
       const callbackArgs: unknown[] = [this, event]
       if (entry !== undefined) {
@@ -531,8 +507,8 @@ export class Session {
         }
       }
       case 'context/message': {
-        const { content, source } = event.data
-        return { role: 'user', content: renderTagged('context', content, source) }
+        const { content, source, envelope } = event.data
+        return { role: 'user', content: renderContextContent(content, source, envelope) }
       }
       case 'steering/message': {
         const { content, source } = event.data

packages/core/session/src/surface.ts

@@ -61,40 +61,106 @@ interface SurfaceFoldState {
   replaceGeneration: number
 }
 
-/** Create one empty fold state. */
+/** A validated replacement transition that has not mutated fold state yet. */
+interface SurfaceReplacePlan extends SurfaceFoldReplacement {
+  kind: 'replace'
+  startIdx: number
+  endIdx: number
+}
+
+/** One validated surface transition that has not mutated fold state yet. */
+type SurfacePlan =
+  | { kind: 'append'; seq: number }
+  | SurfaceReplacePlan
+
+/** Create an empty surface fold state. */
 function createFoldState(): SurfaceFoldState {
   return { nodes: [], replaceGeneration: 0 }
 }
 
-/** Apply one event and return replacement metadata when one occurred. */
-function applySurfaceEvent(
-  state: SurfaceFoldState,
-  event: SessionEvent,
-): SurfaceFoldReplacement | undefined {
-  if (!isSurfaceEligibleType(event.type)) return
-  if (!isSurfaceEvent(event)) {
-    throw new Error(`surface event "${event.type}" (seq ${event.seq}) carries no surfaceOp marker`)
-  }
-  if (event.surfaceOp === 'append') {
-    state.nodes.push(event.seq)
+/** Whether a runtime value is a non-negative safe event sequence. */
+function isEventSeq(value: unknown): value is number {
+  return typeof value === 'number' && Number.isSafeInteger(value) && value >= 0
+}
+
+/** Whether a runtime value is the exact positional-replacement shape. */
+function isReplaceOp(value: object): value is Extract<SurfaceOp, { op: 'replace' }> {
+  const op = value as Record<string, unknown>
+  return Object.keys(op).length === 3
+    && Object.hasOwn(op, 'op')
+    && Object.hasOwn(op, 'start')
+    && Object.hasOwn(op, 'end')
+    && op['op'] === 'replace'
+    && isEventSeq(op['start'])
+    && isEventSeq(op['end'])
+}
+
+/** Validate event-local surface eligibility and return its operation. */
+function surfaceOpOf(event: SessionEvent): SurfaceOp | undefined {
+  const raw = event as SessionEvent & { surfaceOp?: unknown; sourceEventSeqs?: unknown }
+  if (!isSurfaceEligibleType(event.type)) {
+    if (raw.surfaceOp !== undefined) {
+      throw new Error(`session event "${event.type}" is not surface-eligible and cannot carry surfaceOp`)
+    }
+    if (raw.sourceEventSeqs !== undefined) {
+      throw new Error(`session event "${event.type}" is not surface-eligible and cannot carry sourceEventSeqs`)
+    }
     return
   }
+  const op = raw.surfaceOp
+  if (op === undefined) {
+    throw new Error(`session event "${event.type}" is surface-eligible and requires a surfaceOp marker`)
+  }
+  if (op === 'append') return op
+  if (op === null || typeof op !== 'object' || Array.isArray(op)) {
+    throw new Error(`session event "${event.type}" carries an invalid surfaceOp`)
+  }
+  if (!isReplaceOp(op)) {
+    throw new Error(`session event "${event.type}" carries an invalid replace surfaceOp`)
+  }
+  return op
+}
 
-  const shadowedSeqs = replaceSurface(state, event.seq, event.surfaceOp)
-  return {
-    seq: event.seq,
-    start: event.surfaceOp.start,
-    end: event.surfaceOp.end,
-    shadowedSeqs,
+/** Validate provenance against prior log entries and the replacement range. */
+function assertProvenance(
+  event: SessionEvent,
+  shadowedSeqs: readonly number[],
+): void {
+  const raw = (event as SessionEvent & { sourceEventSeqs?: unknown }).sourceEventSeqs
+  const sources = new Set<number>()
+  if (raw !== undefined) {
+    if (!Array.isArray(raw)) {
+      throw new Error(`sourceEventSeqs on event at seq ${event.seq} must be an array when present`)
+    }
+    if (raw.length === 0) {
+      throw new Error('sourceEventSeqs must not be empty when present')
+    }
+    let nonEarlierSource: number | undefined
+    for (const source of raw) {
+      if (!isEventSeq(source)) {
+        throw new Error(`session event "${event.type}" sourceEventSeqs must densely contain non-negative safe integers`)
+      }
+      sources.add(source)
+      if (nonEarlierSource === undefined && source >= event.seq) nonEarlierSource = source
+    }
+    if (sources.size !== raw.length) {
+      throw new Error('sourceEventSeqs must not contain duplicates')
+    }
+    if (nonEarlierSource !== undefined) {
+      throw new Error(`sourceEventSeqs must reference earlier events: ${nonEarlierSource} >= current seq ${event.seq}`)
+    }
+  }
+  const missing = shadowedSeqs.filter(seq => !sources.has(seq))
+  if (missing.length > 0) {
+    throw new Error(`surface replace: sourceEventSeqs must include every shadowed surface node; missing ${missing.join(', ')}`)
   }
 }
 
-/** Replace one inclusive surface range and return the removed sequences. */
-function replaceSurface(
+/** Locate one replacement range without mutating the current fold state. */
+function replacementRange(
   state: SurfaceFoldState,
-  newSeq: number,
   op: Extract<SurfaceOp, { op: 'replace' }>,
-): number[] {
+): Pick<SurfaceReplacePlan, 'startIdx' | 'endIdx' | 'shadowedSeqs'> {
   const startIdx = state.nodes.indexOf(op.start)
   if (startIdx === -1) {
     throw new Error(`surface replace: start seq ${op.start} not found in surface`)
@@ -106,29 +172,78 @@ function replaceSurface(
   if (startIdx > endIdx) {
     throw new Error(`surface replace: start seq ${op.start} (index ${startIdx}) is after end seq ${op.end} (index ${endIdx})`)
   }
+  return {
+    startIdx,
+    endIdx,
+    shadowedSeqs: state.nodes.slice(startIdx, endIdx + 1),
+  }
+}
 
-  const shadowedSeqs = state.nodes.splice(startIdx, endIdx - startIdx + 1, newSeq)
-  state.replaceGeneration += 1
-  return shadowedSeqs
+/** Validate one event at its replay boundary and prepare its atomic fold transition. */
+function planSurfaceEvent(
+  state: SurfaceFoldState,
+  event: SessionEvent,
+  expectedSeq: number,
+): SurfacePlan | undefined {
+  if (event.seq !== expectedSeq) {
+    throw new Error(`session event seq ${event.seq} is not contiguous; expected ${expectedSeq}`)
+  }
+  const surfaceOp = surfaceOpOf(event)
+  if (surfaceOp === undefined) return
+  if (surfaceOp === 'append') {
+    assertProvenance(event, [])
+    return { kind: 'append', seq: event.seq }
+  }
+  const range = replacementRange(state, surfaceOp)
+  assertProvenance(event, range.shadowedSeqs)
+  return {
+    kind: 'replace',
+    seq: event.seq,
+    start: surfaceOp.start,
+    end: surfaceOp.end,
+    ...range,
+  }
+}
+
+/** Apply one event and return replacement metadata only when one occurred. */
+function applySurfaceEvent(
+  state: SurfaceFoldState,
+  event: SessionEvent,
+  expectedSeq: number,
+): SurfaceFoldReplacement | undefined {
+  const plan = planSurfaceEvent(state, event, expectedSeq)
+  if (plan?.kind === 'append') {
+    state.nodes.push(plan.seq)
+  } else if (plan?.kind === 'replace') {
+    state.nodes.splice(plan.startIdx, plan.endIdx - plan.startIdx + 1, plan.seq)
+    state.replaceGeneration += 1
+  }
+  if (plan?.kind !== 'replace') return
+  return {
+    seq: plan.seq,
+    start: plan.start,
+    end: plan.end,
+    shadowedSeqs: plan.shadowedSeqs,
+  }
 }
 
 /**
- * Replay a complete event log through the canonical surface fold.
- * @param events - events in contiguous seq order.
+ * Replay a complete session log through the canonical surface fold.
+ * @param events - session events in contiguous seq order.
  * @returns detached current sequences and replacement history.
- * @throws when a surface marker is missing or names an invalid range.
+ * @throws when an event violates surface metadata, provenance, or range rules.
  */
 export function foldSurface(events: readonly SessionEvent[]): SurfaceFoldResult {
   const state = createFoldState()
   const replacements: SurfaceFoldReplacement[] = []
-  for (const event of events) {
-    const replacement = applySurfaceEvent(state, event)
+  for (const [index, event] of events.entries()) {
+    const replacement = applySurfaceEvent(state, event, index)
     if (replacement !== undefined) replacements.push(replacement)
   }
   return { nodes: [...state.nodes], replacements }
 }
 
-/** Incremental ordered surface view over an append-only session log. */
+/** Incremental ordered surface view and append-boundary validator. */
 export class SurfaceManager {
   /** Shared transition state; replacement history is not retained. */
   private _state = createFoldState()
@@ -137,6 +252,15 @@ export class SurfaceManager {
 
   constructor(private log: readonly SessionEvent[]) {}
 
+  /**
+   * Validate the next candidate without mutating the committed surface.
+   * @param event - candidate event that has not entered the log yet.
+   */
+  validateNext(event: SessionEvent): void {
+    if (this._lastProcessedSeq < this.log.length - 1) this._processDelta()
+    planSurfaceEvent(this._state, event, this.log.length)
+  }
+
   /** Monotonic count of folded positional replacements. */
   get replaceGeneration(): number {
     if (this._lastProcessedSeq < this.log.length - 1) this._processDelta()
@@ -153,8 +277,8 @@ export class SurfaceManager {
   private _processDelta(): void {
     for (let i = this._lastProcessedSeq + 1; i < this.log.length; i++) {
       // eslint-disable-next-line @typescript-eslint/no-non-null-assertion -- bounded by the loop condition
-      applySurfaceEvent(this._state, this.log[i]!)
+      applySurfaceEvent(this._state, this.log[i]!, i)
+      this._lastProcessedSeq = i
     }
-    this._lastProcessedSeq = this.log.length - 1
   }
 }

packages/core/session/src/types.ts

@@ -1,5 +1,9 @@
 import type { Branded } from '@deepseek-ai/dsh-brand'
 import type { CallId, ContentBlock, LlmCallConfig, Message, MessageSource, StreamChunk, TokenUsage, ToolSchema } from '@deepseek-ai/dsh-llm'
+import type { JsonValue } from './json.ts'
+
+/** Canonical context-tag framing, or caller-owned framing rendered verbatim. */
+export type ContextEnvelope = 'context' | 'raw'
 
 /** Identifies one session in the store (and its persistence artifacts). */
 export type SessionId = Branded<'SessionId'>
@@ -201,9 +205,16 @@ export interface SessionEventMap {
   /**
    * In-session context injection (file-change notices, subdir AGENTS.md,
    * skill content, cron notifications, …). Rendered into the derived history
-   * as tagged synthetic context — NOT a user prompt.
+   * as synthetic context — NOT a user prompt. `envelope: 'raw'` lets a caller
+   * own the complete model-facing frame; `meta` is durable JSON state omitted
+   * from the model projection.
    */
-  'context/message': { content: ContentBlock[]; source: MessageSource }
+  'context/message': {
+    content: ContentBlock[]
+    source: MessageSource
+    envelope?: ContextEnvelope
+    meta?: JsonValue
+  }
   /** Raw stream chunk — token-level replay fidelity. */
   'assistant/chunk': { turn: number; step: number; chunk: StreamChunk }
   /**