test: real-Loader dynamic composition, keyless onboarding snapshot, and .env-only e2e

llm-deepseek gains a Loader+Include composition spec proving external
settings.yaml/.env edits reach the very next request, and a real-API e2e
where only a credentials-local document holds the key. The headless example
pins the first-run missing-credential UX as a keyless stream-json snapshot
(new credentials.cordis.snapshot.yml scenario); runLoaderSmoke learns
expectedExitCode so a designed failure surface can be pinned instead of
masked.
This commit is contained in:
Yichen Jiang
2026-07-29 13:44:24 +08:00
parent c0426142c5
commit d77db29f01
9 changed files with 290 additions and 7 deletions

View File

@@ -141,6 +141,13 @@ export interface LoaderSmokeOptions {
readonly prepare?: (cwd: string) => Promise<void> | void
/** Optional world-state assertion run in the isolated cwd before cleanup. */
readonly inspect?: (cwd: string) => Promise<void> | void
/**
* Exact process exit code this smoke expects; defaults to `0`. Scenarios
* pinning a designed failure surface (a one-shot turn ending in an error
* result) declare its nonzero exit here, and a run that exits any other
* way — including succeeding — still fails the smoke.
*/
readonly expectedExitCode?: number
}
/** Captured output from a Loader smoke that exited successfully. */
@@ -187,8 +194,9 @@ export async function runLoaderSmoke(options: LoaderSmokeOptions): Promise<Loade
if (result.timedOut) {
throw new Error(`${options.label} did not exit within ${processTimeoutMs / 1_000}s. stdout:\n${result.stdout}\nstderr:\n${result.stderr}`)
}
if (result.failed) {
throw new Error(`${options.label} exited ${String(result.exitCode)}. stdout:\n${result.stdout}\nstderr:\n${result.stderr}`)
const expectedExitCode = options.expectedExitCode ?? 0
if (result.exitCode !== expectedExitCode) {
throw new Error(`${options.label} exited ${String(result.exitCode)} (expected ${expectedExitCode}). stdout:\n${result.stdout}\nstderr:\n${result.stderr}`)
}
await options.inspect?.(cwd)
return { stdout: result.stdout, stderr: result.stderr }

View File

@@ -74,7 +74,32 @@ describe('runLoaderSmoke', () => {
libBinScript: fixture('fail'),
configPath,
tsconfigPath,
})).rejects.toThrow('failure fixture exited 7. stdout:\n\nstderr:\nfixture failed')
})).rejects.toThrow('failure fixture exited 7 (expected 0). stdout:\n\nstderr:\nfixture failed')
})
it('accepts a declared expected failure exit and rejects any other outcome', async () => {
// A scenario pinning a designed failure surface declares its exit code…
const declared = await runLoaderSmoke({
label: 'declared failure fixture',
tempDirPrefix: 'loader-smoke-declared-fail-',
binScript: fixture('fail'),
libBinScript: fixture('fail'),
configPath,
tsconfigPath,
expectedExitCode: 7,
})
expect(declared.stderr).toBe('fixture failed\n')
// …and a run that succeeds instead still fails the smoke.
await expect(runLoaderSmoke({
label: 'unexpectedly clean fixture',
tempDirPrefix: 'loader-smoke-clean-',
binScript: fixture('success'),
libBinScript: fixture('success'),
configPath,
tsconfigPath,
expectedExitCode: 7,
})).rejects.toThrow(/exited 0 \(expected 7\)/)
})
it('kills a process at its deadline and reports captured output', async () => {