Merge remote-tracking branch 'origin/master' into jsonl-packed-chunk-rows

Master removed the stdio demo (#702c8cc30) — accept the deletion; this
branch's packChunks passthrough survives in acp-demo (auto-merged), and
cli-demo/tui-demo arrived from master without one (the follow-up snapshot
PR decides which demos expose the switch). Generated catalogs regenerated
over merged sources; the hand-written session.md durability paragraph
re-weaves this branch's lossless-encoding wording with master's invariant-
companion sentence.

packages/core/session/src/index.ts

@@ -13,7 +13,7 @@ import { scopeOf, scopeTarget } from '@deepseek-ai/dsh-scope'
 import type { Scoped } from '@deepseek-ai/dsh-scope'
 import type { Message } from '@deepseek-ai/dsh-llm'
 import { SESSION_FORMAT_VERSION, SessionId } from './types.ts'
-import type { CreateSessionOptions, EpochHeader, SessionEvent, SessionEventMap, SessionEventType, SessionHeader, SurfaceIntent, SurfaceEventType } from './types.ts'
+import type { CreateSessionOptions, EpochHeader, OutOfBandSessionEventType, SessionEvent, SessionEventMap, SessionEventType, SessionHeader, SurfaceIntent, SurfaceEventType, TurnTrigger } from './types.ts'
 import { snapshotJsonValue } from './json.ts'
 import { SurfaceManager } from './surface.ts'
 import type { SessionSurface } from './surface.ts'
@@ -29,6 +29,27 @@ export type { SessionSurface, SurfaceFoldReplacement, SurfaceFoldResult } from '
 export { foldSurface, isSurfaceEvent, isSurfaceEligibleType } from './surface.ts'
 export { canonicalHeader, foldRequestHeader, headerEquals } from './request-header.ts'
 
+/**
+ * Find the latest closed message-triggered turn, excluding injection and
+ * plugin-owned zero-step turns.
+ * @param events - session events, or an owned suffix, to inspect.
+ * @returns the latest matching turn end, or `undefined`.
+ */
+export function findLastMessageTurnEnd(
+  events: readonly SessionEvent[],
+): SessionEvent<'turn/end'> | undefined {
+  const messageTurns = new Set<number>()
+  let latest: SessionEvent<'turn/end'> | undefined
+  for (const event of events) {
+    if (event.type === 'turn/start') {
+      if (event.data.trigger.kind === 'message') messageTurns.add(event.data.turn)
+      continue
+    }
+    if (event.type === 'turn/end' && messageTurns.delete(event.data.turn)) latest = event
+  }
+  return latest
+}
+
 declare module 'cordis' {
   interface Context {
     sessions: SessionStore
@@ -139,6 +160,7 @@ function assertSessionEventEnvelope(value: Record<string, unknown>, index: numbe
     throw new Error(`seed event at index ${index} has an invalid event envelope`)
   }
   assertCurrentLlmShape(event, index)
+  assertCurrentTurnEndShape(event, index)
 }
 
 /** Reject pre-provider request headers and assistant messages at the seed/load boundary. */
@@ -156,6 +178,22 @@ function assertCurrentLlmShape(event: Record<string, unknown>, index: number): v
   }
 }
 
+/** Reject legacy aborted outcomes that persisted caller-owned reason detail. */
+function assertCurrentTurnEndShape(event: Record<string, unknown>, index: number): void {
+  if (event['type'] !== 'turn/end') return
+  const data = event['data']
+  /* v8 ignore next -- this migration recognizes only the legacy object shape; format-wide payload validation is separate. */
+  if (typeof data !== 'object' || data === null) return
+  const reason = (data as Record<string, unknown>)['reason']
+  /* v8 ignore next -- non-object reasons cannot carry the legacy aborted detail this migration removes. */
+  if (typeof reason !== 'object' || reason === null || Array.isArray(reason)) return
+  const record = reason as Record<string, unknown>
+  if (record['kind'] === 'aborted'
+    && (Object.keys(record).length !== 1 || !Object.hasOwn(record, 'kind'))) {
+    throw new Error(`seed turn/end at index ${index} uses unsupported reason-bearing aborted format`)
+  }
+}
+
 /** Whether an unknown value carries the current provider/model pair. */
 function hasProviderModel(value: unknown): boolean {
   if (typeof value !== 'object' || value === null) return false
@@ -212,6 +250,7 @@ interface SessionEntry {
   announced: boolean
   announcing: boolean
   appending: boolean
+  outOfBand: boolean
   detachRequested: boolean
   detach(): void
 }
@@ -386,7 +425,7 @@ export class Session {
     } finally {
       if (entry !== undefined) {
         entry.appending = false
-        if (entry.detachRequested && !entry.announcing) entry.detach()
+        if (entry.detachRequested && !entry.announcing && !entry.outOfBand) entry.detach()
       }
     }
   }
@@ -670,6 +709,7 @@ export class SessionStore extends Service {
       announced: false,
       announcing: false,
       appending: false,
+      outOfBand: false,
       detachRequested: false,
       detach: () => { this.detachEntered(entry) },
     }
@@ -682,7 +722,7 @@ export class SessionStore extends Service {
       // A lifecycle listener may own the advanced detach capability. Keep the
       // entry and its publication hooks live until synchronous creation or append
       // publication unwinds, then publish the paired disposal edge.
-      if (entry.announcing || entry.appending) {
+      if (entry.announcing || entry.appending || entry.outOfBand) {
         entry.detachRequested = true
         return
       }
@@ -736,7 +776,7 @@ export class SessionStore extends Service {
       }
     } finally {
       entry.announcing = false
-      if (entry.detachRequested && !entry.appending) entry.detach()
+      if (entry.detachRequested && !entry.appending && !entry.outOfBand) entry.detach()
     }
   }
 
@@ -780,6 +820,87 @@ export class SessionStore extends Service {
     if (failure !== undefined) throw failure.reason
   }
 
+  /**
+   * Append one plugin-declared log-only event without borrowing the agent
+   * loop's lifecycle. An open turn receives the event directly and remains
+   * responsible for its ordinary checkpoint. A closed log receives one
+   * zero-step turn around the event, followed by an awaited flush.
+   *
+   * Once the synthetic `turn/start` commits, this method always attempts its
+   * matching `turn/end` and flush, including when the target append fails.
+   * Detachment requested by an event or flush listener is deferred until that
+   * sequence settles, so publication cannot switch from a live scoped session
+   * to an unobserved bare `Session` halfway through the update.
+   *
+   * @param session - exact live session that owns the target log.
+   * @param type - event type opted into {@link OutOfBandSessionEventMap} by its owner.
+   * @param data - typed JSON payload for the target event.
+   * @param trigger - plugin-owned turn trigger used only when the log is closed.
+   * @returns the accepted target event with its assigned sequence and timestamp.
+   * @throws when the session is detached, another out-of-band append is active,
+   *   event acceptance fails, the synthetic turn cannot close, or flushing fails.
+   */
+  async appendOutOfBand<T extends OutOfBandSessionEventType>(
+    session: Session,
+    type: T,
+    data: SessionEventMap[T],
+    trigger: TurnTrigger,
+  ): Promise<SessionEvent<T>> {
+    const entry = this.liveEntryFor(session)
+    if (entry.outOfBand) {
+      throw new Error(`session "${session.id}" already has an out-of-band append in progress`)
+    }
+    entry.outOfBand = true
+    // `T` is excluded from SurfaceEventType by OutOfBandSessionEventType, but
+    // TypeScript does not reduce Session.append's conditional rest parameter
+    // through a generic intersection. Preserve that proven two-argument call
+    // shape without widening the public Session.append overload.
+    const appendLogOnly = session.append.bind(session) as unknown as <K extends OutOfBandSessionEventType>(
+      eventType: K,
+      eventData: SessionEventMap[K],
+    ) => SessionEvent<K>
+    try {
+      const lastBoundary = session.events.findLast(event => event.type === 'turn/start' || event.type === 'turn/end')
+      if (lastBoundary?.type === 'turn/start') {
+        return appendLogOnly(type, data)
+      }
+
+      const lastStart = session.events.findLast(event => event.type === 'turn/start')
+      const turn = (lastStart?.data.turn ?? 0) + 1
+      let accepted: SessionEvent<T> | undefined
+      let failure: unknown
+      let opened = false
+      try {
+        session.append('turn/start', { turn, trigger })
+        opened = true
+        accepted = appendLogOnly(type, data)
+      } catch (error: unknown) {
+        failure = error
+      } finally {
+        if (opened) {
+          // The only target types admitted by OutOfBandSessionEventMap are
+          // log-only plugin events, so the synthetic turn remains open here.
+          session.append('turn/end', { turn, reason: { kind: 'completed' } })
+          try {
+            await this.flush(session)
+          } catch (error: unknown) {
+            if (failure === undefined) failure = error
+          }
+        }
+      }
+      if (failure !== undefined) {
+        // eslint-disable-next-line @typescript-eslint/only-throw-error -- preserve an arbitrary flush-listener rejection exactly
+        throw failure
+      }
+      /* v8 ignore next -- accepted is assigned unless an append failure was captured above. */
+      if (accepted === undefined) throw new Error('out-of-band append completed without an accepted event')
+      return accepted
+    } finally {
+      entry.outOfBand = false
+      if (entry.detachRequested && !entry.announcing && !entry.appending) entry.detach()
+    }
+  }
+
   /** Return the exact live entry; detached/prepared objects reject. */
   private liveEntryFor(session: Session): SessionEntry {
     const entry = attachments.get(session)

packages/core/session/src/invariant.ts

@@ -0,0 +1,238 @@
+/**
+ * Package-owned relational invariants for the session event log. Load this
+ * companion beside `@deepseek-ai/dsh-invariants` to enable the checks.
+ *
+ * @module @deepseek-ai/dsh-session/invariant
+ */
+
+import type { Context } from 'cordis'
+import { assertNever } from '@deepseek-ai/dsh-llm'
+import type { CallId } from '@deepseek-ai/dsh-llm'
+import type { InvariantFailure, InvariantInstaller } from '@deepseek-ai/dsh-invariants'
+import type { Session, SessionEvent } from '@deepseek-ai/dsh-session'
+
+const PACKAGE_NAME = '@deepseek-ai/dsh-session'
+
+/** Cordis companion plugin name. */
+export const name = 'session-invariant'
+/** Service required before the companion can reserve package ownership. */
+export const inject = ['invariants']
+
+/** Per-session bookkeeping for relational log checks. */
+interface SessionTrace {
+  lastSeq: number
+  openTurn: number | null
+  openStep: number | null
+  nextTurn: number
+  nextStep: number
+  pendingCalls: Set<CallId>
+}
+
+/** One accepted event's deferred mutation of a committed session trace. */
+interface SessionTraceTransition {
+  scalars: Pick<SessionTrace, 'lastSeq' | 'openTurn' | 'openStep' | 'nextTurn' | 'nextStep'>
+  pendingCalls:
+    | { kind: 'none' }
+    | { kind: 'add' | 'delete'; callId: CallId }
+    | { kind: 'clear' }
+}
+
+/** Assert that a step-scoped event names the currently open turn and step. */
+function requireOpenStep(
+  trace: SessionTrace,
+  kind: string,
+  turn: number,
+  step: number,
+  fail: InvariantFailure,
+): void {
+  if (trace.openTurn !== turn || trace.openStep !== step) {
+    fail(`${kind} names turn ${turn}/step ${step} but open is turn ${trace.openTurn}/step ${trace.openStep}`)
+  }
+}
+
+/** Validate one candidate event without mutating the committed trace. */
+function validateEvent(
+  trace: SessionTrace,
+  event: SessionEvent,
+  fail: InvariantFailure,
+): SessionTraceTransition {
+  if (event.seq <= trace.lastSeq) {
+    fail(`seq must strictly increase: saw ${event.seq} after ${trace.lastSeq}`)
+  }
+  let openTurn = trace.openTurn
+  let openStep = trace.openStep
+  let nextTurn = trace.nextTurn
+  let nextStep = trace.nextStep
+  let pendingCalls: SessionTraceTransition['pendingCalls'] = { kind: 'none' }
+
+  // SessionEventMap is merge-extensible, so the default enforces turn
+  // enclosure for package-added events as well as the built-in variants.
+  switch (event.type) {
+    case 'turn/start': {
+      if (trace.openTurn !== null) {
+        fail(`turn/start ${event.data.turn} while turn ${trace.openTurn} is still open`)
+      }
+      if (event.data.turn !== trace.nextTurn) {
+        fail(`turn/start expected turn ${trace.nextTurn}, got ${event.data.turn}`)
+      }
+      openTurn = event.data.turn
+      nextStep = 1
+      break
+    }
+    case 'turn/end': {
+      if (trace.openTurn !== event.data.turn) {
+        fail(`turn/end ${event.data.turn} does not match open turn ${trace.openTurn}`)
+      }
+      if (trace.openStep !== null) {
+        fail(`turn/end ${event.data.turn} while step ${trace.openStep} is still open`)
+      }
+      openTurn = null
+      nextTurn += 1
+      break
+    }
+    case 'step/start': {
+      if (trace.openTurn !== event.data.turn) {
+        fail(`step/start in turn ${event.data.turn} but open turn is ${trace.openTurn}`)
+      }
+      if (trace.openStep !== null) {
+        fail(`step/start ${event.data.step} while step ${trace.openStep} is still open`)
+      }
+      if (event.data.step !== trace.nextStep) {
+        fail(`step/start expected step ${trace.nextStep} in turn ${event.data.turn}, got ${event.data.step}`)
+      }
+      openStep = event.data.step
+      break
+    }
+    case 'step/end': {
+      requireOpenStep(trace, 'step/end', event.data.turn, event.data.step, fail)
+      pendingCalls = { kind: 'clear' }
+      openStep = null
+      nextStep += 1
+      break
+    }
+    case 'assistant/chunk': {
+      requireOpenStep(trace, 'assistant/chunk', event.data.turn, event.data.step, fail)
+      break
+    }
+    case 'assistant/message': {
+      requireOpenStep(trace, 'assistant/message', event.data.turn, event.data.step, fail)
+      break
+    }
+    case 'tool/call': {
+      requireOpenStep(trace, 'tool/call', event.data.turn, event.data.step, fail)
+      pendingCalls = { kind: 'add', callId: event.data.callId }
+      break
+    }
+    case 'tool/result': {
+      // Session has already validated a provenance-backed content rewrite.
+      // It is durable turn work, not a second execution of the original call.
+      if (event.surfaceOp !== 'append') {
+        if (trace.openTurn === null) {
+          fail('tool/result surface replacement appended outside any open turn')
+        }
+        break
+      }
+      requireOpenStep(trace, 'tool/result', event.data.turn, event.data.step, fail)
+      const syntheticInterrupted = event.data.isError && event.data.error?.code === 'interrupted'
+      if (!trace.pendingCalls.has(event.data.callId) && !syntheticInterrupted) {
+        fail(`tool/result for ${event.data.callId} with no prior tool/call in this step`)
+      }
+      pendingCalls = { kind: 'delete', callId: event.data.callId }
+      break
+    }
+    default: {
+      if (trace.openTurn === null) {
+        fail(`${event.type} appended outside any open turn (every event must be turn-enclosed)`)
+      }
+      break
+    }
+  }
+  return {
+    scalars: { lastSeq: event.seq, openTurn, openStep, nextTurn, nextStep },
+    pendingCalls,
+  }
+}
+
+/** Apply one already-validated transition after its event commits. */
+function applyTransition(trace: SessionTrace, transition: SessionTraceTransition): void {
+  Object.assign(trace, transition.scalars)
+  switch (transition.pendingCalls.kind) {
+    case 'none':
+      break
+    case 'add':
+      trace.pendingCalls.add(transition.pendingCalls.callId)
+      break
+    case 'delete':
+      trace.pendingCalls.delete(transition.pendingCalls.callId)
+      break
+    case 'clear':
+      trace.pendingCalls.clear()
+      break
+    /* v8 ignore next -- validateEvent produces this closed transition union */
+    default:
+      assertNever(transition.pendingCalls, 'session trace pending-call transition')
+  }
+}
+
+/** Install the session contribution into its child registration fiber. */
+const install: InvariantInstaller = Object.assign((ctx: Context, fail: InvariantFailure) => {
+  const traces = new WeakMap<Session, SessionTrace>()
+  const stagedTransitions = new WeakMap<SessionEvent, {
+    session: Session
+    trace: SessionTrace
+    transition: SessionTraceTransition
+  }>()
+
+  const freshTrace = (): SessionTrace => ({
+    lastSeq: -1,
+    openTurn: null,
+    openStep: null,
+    nextTurn: 1,
+    nextStep: 1,
+    pendingCalls: new Set(),
+  })
+
+  const seedSession = (session: Session): SessionTrace => {
+    const trace = freshTrace()
+    traces.set(session, trace)
+    for (const event of session.events) {
+      applyTransition(trace, validateEvent(trace, event, fail))
+    }
+    return trace
+  }
+
+  /* v8 ignore next -- session/event always follows list() or session/created seeding */
+  const traceFor = (session: Session): SessionTrace => traces.get(session) ?? seedSession(session)
+
+  for (const session of ctx.sessions.list()) seedSession(session)
+
+  ctx.on('session/created', (session) => { seedSession(session) }, { global: true })
+
+  ctx.on('session/event', (session, event) => {
+    const staged = stagedTransitions.get(event)
+    /* v8 ignore next 2 -- internal/dispatch stages the exact callback arguments */
+    if (staged === undefined || staged.session !== session) {
+      return fail('session/event reached publication without matching pre-commit validation')
+    }
+    stagedTransitions.delete(event)
+    applyTransition(staged.trace, staged.transition)
+  }, { global: true })
+
+  ctx.on('internal/dispatch', (_mode, eventName, args) => {
+    if (eventName !== 'session/event') return
+    const [session, event] = args as [Session, SessionEvent]
+    const trace = traceFor(session)
+    const transition = validateEvent(trace, event, fail)
+    // A later dispatch listener may veto. Validation is pure, so abandoning
+    // this weakly keyed transition does not advance or retain the session.
+    stagedTransitions.set(event, { session, trace, transition })
+  }, { global: true })
+}, { inject: ['sessions'] })
+
+/**
+ * Register the session invariant companion.
+ * @param ctx - Cordis context carrying the invariant service.
+ * @returns the installed registration's disposer after setup succeeds.
+ */
+export const apply = (ctx: Context): Promise<() => void> =>
+  Promise.resolve(ctx.invariants.register(PACKAGE_NAME, install))

packages/core/session/src/types.ts

@@ -1,5 +1,5 @@
 import type { Branded } from '@deepseek-ai/dsh-brand'
-import type { AssistantProvenance, CallId, ContentBlock, LlmCallConfig, Message, MessageSource, StreamChunk, TokenUsage, ToolSchema } from '@deepseek-ai/dsh-llm'
+import type { AssistantProvenance, CallId, ContentBlock, LlmCallConfig, LlmFailure, Message, MessageSource, StreamChunk, TokenUsage, ToolSchema } from '@deepseek-ai/dsh-llm'
 import type { JsonValue } from './json.ts'
 
 /** Identifies one session in the store (and its persistence artifacts). */
@@ -101,14 +101,19 @@ export type TurnTrigger = TurnTriggerMap[keyof TurnTriggerMap]
  */
 export interface TurnEndReasonMap {
   completed: { kind: 'completed' }
-  aborted: { kind: 'aborted'; reason?: string }
+  /** A cancellation request interrupted the live turn. */
+  aborted: { kind: 'aborted' }
   /**
    * The turn failed: a step threw or the model reported a failure. `step` is the
    * step number the failure occurred on (the operational error's location — the
    * single durable record of an in-turn failure; live diagnostics also fire via
-   * `agent/error`). `code` is the error's code when one was attached.
+   * `agent/error`). Final model-request failures retain their normalized facts
+   * as one `failure`; other turn failures retain their live Error message/code.
    */
-  error: { kind: 'error'; step: number; message: string; code?: string }
+  error: { kind: 'error'; step: number } & (
+    | { failure: LlmFailure; message?: never; code?: never }
+    | { message: string; code?: string; failure?: never }
+  )
   disposed: { kind: 'disposed' }
   /** At least one step reached its output-token ceiling, even if a plugin continued the turn. */
   'max-tokens': { kind: 'max-tokens' }
@@ -259,9 +264,23 @@ export interface SessionEventMap {
   'request/header': { header: EpochHeader; reason: RequestHeaderReason }
 }
 
+/**
+ * Marker map for plugin-owned log-only events accepted by
+ * `SessionStore.appendOutOfBand()`. A plugin extends this map with the same key
+ * it adds to {@link SessionEventMap}; surface and lifecycle events stay
+ * ineligible unless their owner explicitly opts them into this narrow seam.
+ */
+export interface OutOfBandSessionEventMap {}
+
 /** The appendable event-type keys of {@link SessionEventMap}, plugin-merged extensions included. */
 export type SessionEventType = keyof SessionEventMap
 
+/** Plugin-declared non-surface event types accepted by `SessionStore.appendOutOfBand()`. */
+export type OutOfBandSessionEventType = Exclude<
+  Extract<SessionEventType, keyof OutOfBandSessionEventMap>,
+  SurfaceEventType
+>
+
 /**
  * The subset of {@link SessionEventType} values whose events produce LLM
  * messages and are eligible to appear on the ordered surface. Only these