From 1fae5dc40ec3cf7e559550a80ba88d5f029f87b0 Mon Sep 17 00:00:00 2001 From: imccyu <276526105+imccyu@users.noreply.github.com> Date: Fri, 14 Aug 2026 22:24:23 +0800 Subject: [PATCH] fix(scripts): disclose browser-bundled packages as shipped MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Moving react, shiki, katex and the markdown pipeline to devDependencies took them out of the notices runtime tier, which tiers by declaring section — yet their code is inside lib/client.js and the shell dist. The generator now learns what the browser artifacts carry from the real build configs: each client bundle through its own tsdown config, the shell through apps/web's Vite config, with a recorder that resolves each bare specifier, notes the package behind it, and stops there. About three seconds, and only packages a resolved file backs, so a bundler's virtual module is not mistaken for a shipped one. Net effect on the file: the type-only packages @types/mdast and micromark-util-types move to the development tier, because neither ships code. --- THIRD_PARTY_NOTICES.md | 8 +- lefthook.yml | 2 +- scripts/browser-bundled-externals.ts | 177 ++++++++++++++++++++++++ scripts/gen-third-party-notices.spec.ts | 24 +++- scripts/gen-third-party-notices.ts | 47 ++++--- 5 files changed, 234 insertions(+), 24 deletions(-) create mode 100644 scripts/browser-bundled-externals.ts diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index 92b218ff33..482d0ea1ce 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -27,7 +27,7 @@ The Cordis framework and its foundation libraries are source-vendored into this ## Runtime npm dependencies -External packages that a workspace package resolves at runtime. The tier covers every plugin a user can mount from `cordis.yml` — not only what the `dsh` CLI, Web UI, and Python SDK runtime load by default. +External packages that reach a user: a workspace package resolves them at runtime, or a published browser artifact carries a copy of their code. The tier covers every plugin a user can mount from `cordis.yml` — not only what the `dsh` CLI, Web UI, and Python SDK runtime load by default — and the packages the client build inlines into a plugin bundle or the shell `dist`, which are declared as `devDependencies` because nothing on a user's machine resolves their specifiers. | Package | License | | --- | --- | @@ -48,7 +48,6 @@ External packages that a workspace package resolves at runtime. The tier covers | [`@shikijs/langs`](https://github.com/shikijs/shiki) | MIT | | [`@standard-schema/spec`](https://github.com/standard-schema/standard-schema) | MIT | | [`@tanstack/react-virtual`](https://github.com/TanStack/virtual) | MIT | -| [`@types/mdast`](https://github.com/DefinitelyTyped/DefinitelyTyped) | MIT | | [`@vscode/ripgrep`](https://github.com/microsoft/vscode-ripgrep) | MIT | | [`anser`](https://github.com/IonicaBizau/anser) | MIT | | [`chokidar`](https://github.com/paulmillr/chokidar) | MIT | @@ -73,7 +72,6 @@ External packages that a workspace package resolves at runtime. The tier covers | [`micromark-util-classify-character`](https://github.com/micromark/micromark/tree/main/packages/micromark-util-classify-character) | MIT | | [`micromark-util-sanitize-uri`](https://github.com/micromark/micromark/tree/main/packages/micromark-util-sanitize-uri) | MIT | | [`micromark-util-symbol`](https://github.com/micromark/micromark/tree/main/packages/micromark-util-symbol) | MIT | -| [`micromark-util-types`](https://github.com/micromark/micromark/tree/main/packages/micromark-util-types) | MIT | | [`node-addon-require-builtin`](https://www.npmjs.com/package/node-addon-require-builtin) | MIT | | [`node-pty`](https://github.com/microsoft/node-pty) | MIT | | [`picomatch`](https://github.com/micromatch/picomatch) | MIT | @@ -115,7 +113,7 @@ The installed SDK 0.3.220 declares the following optional platform packages. Eac ## Development-only npm dependencies -External packages **directly declared** only by repository tooling, test infrastructure, the documentation site, the demo leaves, or the native launcher's build workspace. No shipped surface names them itself. A package here may still be pulled in transitively by a runtime dependency — `pnpm-lock.yaml` is the authority on the full closure — so this tier records who declares a package, not what a build ultimately bundles. +External packages **directly declared** only by repository tooling, test infrastructure, the documentation site, the demo leaves, or the native launcher's build workspace, and carried by no published artifact. No shipped surface names them itself. A package here may still be pulled in transitively by a runtime dependency — `pnpm-lock.yaml` is the authority on the full closure — so this tier records who declares a package, not what a build ultimately bundles. | Package | License | | --- | --- | @@ -129,6 +127,7 @@ External packages **directly declared** only by repository tooling, test infrast | [`@types/babel__code-frame`](https://github.com/DefinitelyTyped/DefinitelyTyped) | MIT | | [`@types/js-yaml`](https://github.com/DefinitelyTyped/DefinitelyTyped) | MIT | | [`@types/jsdom`](https://github.com/DefinitelyTyped/DefinitelyTyped) | MIT | +| [`@types/mdast`](https://github.com/DefinitelyTyped/DefinitelyTyped) | MIT | | [`@types/node`](https://github.com/DefinitelyTyped/DefinitelyTyped) | MIT | | [`@types/picomatch`](https://github.com/DefinitelyTyped/DefinitelyTyped) | MIT | | [`@types/react`](https://github.com/DefinitelyTyped/DefinitelyTyped) | MIT | @@ -154,6 +153,7 @@ External packages **directly declared** only by repository tooling, test infrast | [`lefthook`](https://github.com/evilmartians/lefthook) | MIT | | [`lightningcss`](https://github.com/parcel-bundler/lightningcss) | MPL-2.0 | | [`mermaid`](https://github.com/mermaid-js/mermaid) | MIT | +| [`micromark-util-types`](https://github.com/micromark/micromark/tree/main/packages/micromark-util-types) | MIT | | [`oxlint`](https://github.com/oxc-project/oxc) | MIT | | [`oxlint-tsgolint`](https://github.com/oxc-project/tsgolint) | MIT | | [`playwright`](https://github.com/microsoft/playwright) | Apache-2.0 | diff --git a/lefthook.yml b/lefthook.yml index 1c2a693c88..01c4a5b519 100644 --- a/lefthook.yml +++ b/lefthook.yml @@ -28,7 +28,7 @@ pre-commit: # lefthook only inspects files present on disk — so that one case still # falls through to the freshness assertion in the test lane. - name: third-party notices (staged) - glob: '{package.json,*/package.json,*/*/package.json,*/*/*/package.json,*/*/*/*/package.json,pnpm-workspace.yaml,*/*/pnpm-workspace.yaml,pnpm-lock.yaml,vendor/README.md,python/*/pyproject.toml,scripts/gen-third-party-notices.ts,scripts/build-exe-for-python-sdk.ts}' + glob: '{package.json,*/package.json,*/*/package.json,*/*/*/package.json,*/*/*/*/package.json,pnpm-workspace.yaml,*/*/pnpm-workspace.yaml,pnpm-lock.yaml,vendor/README.md,python/*/pyproject.toml,scripts/gen-third-party-notices.ts,scripts/browser-bundled-externals.ts,packages/client/tsdown.client.ts,scripts/build-exe-for-python-sdk.ts}' run: node_modules/.bin/tsx scripts/gen-third-party-notices.ts && git add THIRD_PARTY_NOTICES.md - name: whitespace (staged) diff --git a/scripts/browser-bundled-externals.ts b/scripts/browser-bundled-externals.ts new file mode 100644 index 0000000000..4ab572f544 --- /dev/null +++ b/scripts/browser-bundled-externals.ts @@ -0,0 +1,177 @@ +/** + * The external packages a published browser artifact carries a copy of. + * + * Read from the real build configurations rather than declared by hand: each + * `lib/client.js` plugin bundle is driven through its own `tsdown.config.ts`, and + * the shell `dist` through `apps/web`'s Vite config. A recording plugin resolves + * every bare specifier as external and notes it, so the pass walks our own source + * and stops at the package boundary — which is both fast (about two seconds for + * the whole repository) and exactly the direct-dependency granularity + * THIRD_PARTY_NOTICES.md discloses. Erased type imports never appear, because the + * transform drops them before resolution. + * + * Workspace names are followed only on the Vite side, where the shell's aliases + * map them to source: that is how a browser-only library's own third-party + * imports — katex and shiki through `ui-primitives`, for one — become visible. A + * plugin bundle keeps them external, matching the frozen module table it is built + * against; the wire layers it inlines are host packages that declare their own + * dependencies, so nothing goes undisclosed. + * + * A specifier is recorded only once the host resolves it to a file inside a + * package. A bundler's own virtual module has no package behind it — + * `vite/modulepreload-polyfill` is generated by a Vite plugin rather than shipped + * as a file, so the polyfill in the published `dist` is build glue in the same + * category as an emitted TypeScript helper, not a redistributed copy of Vite. + * + * rolldown is resolved through tsdown deliberately: the dry run must use the + * exact bundler the real build uses, which a separate root pin could drift from. + */ + +import { globSync, readFileSync } from 'node:fs' +import { createRequire } from 'node:module' +import { dirname, join } from 'node:path' + +/** The plugin-context member the recorder needs to resolve before recording. */ +interface ResolveContext { + resolve: ( + source: string, + importer: string, + options: { skipSelf: boolean }, + ) => Promise<{ id: string } | null> +} + +/** A rolldown/Vite plugin shape, narrowed to what the recorder needs. */ +interface RecorderPlugin { + name: string + enforce?: 'pre' + resolveId: ( + this: ResolveContext, + source: string, + importer: string | undefined, + ) => Promise<{ id: string; external: true } | null> +} + +/** + * The package a resolved module file belongs to. + * @param file - absolute path of a resolved module. + * @returns the package name, or undefined when the file is not inside a package. + */ +function packageOfFile(file: string): string | undefined { + const marker = file.lastIndexOf('node_modules/') + if (marker < 0) return undefined + const rest = file.slice(marker + 'node_modules/'.length) + const parts = rest.split('/') + return rest.startsWith('@') ? parts.slice(0, 2).join('/') : parts[0] +} + +/** + * Build the plugin that records bare specifiers and stops the walk at them. + * @param seen - set the recorder adds package names to. + * @param followWorkspace - leave `@deepseek-ai/*` to the host resolver instead of + * externalizing it, so the walk continues into our own source. + * @returns the recording plugin. + */ +function recorder(seen: Set, followWorkspace: boolean): RecorderPlugin { + return { + name: 'dsh-record-direct-externals', + enforce: 'pre', + async resolveId(source, importer) { + if (importer === undefined) return null // the entry itself + if (source.startsWith('.') || source.startsWith('/') || source.startsWith('\0')) return null + if (source.startsWith('virtual:') || source.includes('?')) return null + if (followWorkspace && source.startsWith('@deepseek-ai/')) return null + if (source.startsWith('node:')) return { id: source, external: true } + if (!source.startsWith('@deepseek-ai/')) { + const resolved = await this.resolve(source, importer, { skipSelf: true }) + const name = resolved === null ? undefined : packageOfFile(resolved.id) + if (name !== undefined) seen.add(name) + } + return { id: source, external: true } + }, + } +} + +interface Manifest { + exports?: Record + files?: string[] +} + +/** Read one workspace manifest. */ +function manifestOf(dir: string): Manifest { + return JSON.parse(readFileSync(join(dir, 'package.json'), 'utf8')) as Manifest +} + +/** Whether a manifest publishes a tsdown browser bundle at `lib/client.js`. */ +function publishesClientBundle(manifest: Manifest): boolean { + const target = manifest.exports?.['./client'] + return typeof target === 'object' && target !== null && target.default === './lib/client.js' +} + +/** + * Record every external package the plugin client bundles carry. + * @param root - repository root. + * @param seen - set the recorder adds package names to. + */ +async function collectFromClientBundles(root: string, seen: Set): Promise { + const requireFromTsdown = createRequire(createRequire(import.meta.url).resolve('tsdown')) + const { rolldown } = await import(requireFromTsdown.resolve('rolldown')) as { + rolldown: (options: Record) => Promise<{ + generate: (output: Record) => Promise + close: () => Promise + }> + } + + for (const relative of globSync('packages/*/*/tsdown.config.ts', { cwd: root }).sort()) { + const dir = join(root, dirname(relative)) + if (!publishesClientBundle(manifestOf(dir))) continue + const loaded = await import(join(root, relative)) as { default: unknown } + const factory = loaded.default + const configs = (typeof factory === 'function' + ? (factory as (inline: { env: Record }) => unknown[])({ env: {} }) + : [factory]) as { name?: string; entry?: unknown; plugins?: unknown[] }[] + // The `/client` config is the browser bundle; its siblings emit the node half. + const client = configs.find(config => config.name?.endsWith('/client') === true) + if (client === undefined) continue + const bundle = await rolldown({ + cwd: dir, + input: client.entry, + plugins: [recorder(seen, false), ...(client.plugins ?? [])], + platform: 'browser', + }) + await bundle.generate({ format: 'cjs', minify: false, sourcemap: false }) + await bundle.close() + } +} + +/** + * Record every external package the prebuilt shell bundle carries. + * @param root - repository root. + * @param seen - set the recorder adds package names to. + */ +async function collectFromShellBundle(root: string, seen: Set): Promise { + for (const relative of globSync('apps/*/vite.config.ts', { cwd: root }).sort()) { + const dir = join(root, dirname(relative)) + // Vite belongs to the app that builds with it, so it resolves from there. + const { build } = await import(createRequire(join(dir, 'package.json')).resolve('vite')) as { + build: (options: Record) => Promise + } + await build({ + root: dir, + logLevel: 'error', + plugins: [recorder(seen, true)], + build: { write: false, minify: false, sourcemap: false, reportCompressedSize: false }, + }) + } +} + +/** + * The external packages a published browser artifact carries a copy of. + * @param root - repository root. + * @returns package names, workspace names excluded. + */ +export async function browserBundledExternals(root: string): Promise> { + const seen = new Set() + await collectFromClientBundles(root, seen) + await collectFromShellBundle(root, seen) + return seen +} diff --git a/scripts/gen-third-party-notices.spec.ts b/scripts/gen-third-party-notices.spec.ts index 479a2f13b1..5cd4f6b7fa 100644 --- a/scripts/gen-third-party-notices.spec.ts +++ b/scripts/gen-third-party-notices.spec.ts @@ -24,11 +24,12 @@ describe('THIRD_PARTY_NOTICES.md', () => { // already runs in the test lane, so the check costs no extra CI process. // Pre-commit regenerates the file whenever a manifest is staged, so reaching // this assertion means the notices were committed without that hook. - it('matches what the generator produces from the current manifests', () => { - const generated = render() + it('matches what the generator produces from the current manifests', async () => { + const generated = await render() expect(generated).toContain('It depends on the third-party software listed below.') expect(readFileSync(resolve(root, 'THIRD_PARTY_NOTICES.md'), 'utf8'), 'stale notices — run `pnpm run gen-third-party-notices`').toBe(generated) - }) + // Driving the two real bundlers to learn what ships costs a few seconds. + }, 60_000) }) /** Build the (manifests, names) pair `tierExternalDeps` consumes. */ @@ -67,6 +68,23 @@ describe('tierExternalDeps', () => { ])) }) + it('keeps a devDependency runtime when a published browser artifact carries it', () => { + const { manifests, names } = workspace({ + // The client build inlines these, so a copy ships even though no manifest + // resolves the specifier at run time. + 'packages/client/ui-primitives/package.json': { + name: '@deepseek-ai/dsh-client-ui-primitives', + devDependencies: { katex: '^0.16', 'test-only-helper': '^1' }, + }, + }) + + expect(tierExternalDeps(manifests, names, new Set(['katex']))).toEqual(new Map([ + ['tsx', true], + ['katex', true], + ['test-only-helper', false], + ])) + }) + it('keeps a package runtime when any shipping area declares it, and excludes workspace links', () => { const { manifests, names } = workspace({ 'package.json': { devDependencies: { shared: '^1' } }, diff --git a/scripts/gen-third-party-notices.ts b/scripts/gen-third-party-notices.ts index f2411f5eff..1762e2d125 100644 --- a/scripts/gen-third-party-notices.ts +++ b/scripts/gen-third-party-notices.ts @@ -13,6 +13,7 @@ import { resolve } from 'node:path' import * as yaml from 'js-yaml' import { parse as parseToml, type TomlTableWithoutBigInt, type TomlValueWithoutBigInt } from 'smol-toml' import parseSpdx from 'spdx-expression-parse' +import { browserBundledExternals } from './browser-bundled-externals.ts' const root = resolve(import.meta.dirname, '..') const OUT = 'THIRD_PARTY_NOTICES.md' @@ -347,15 +348,17 @@ function normalizeRepo(raw: string | undefined): string | undefined { } /** - * External npm dependencies, tiered by which workspace area declares them at - * runtime: a package is runtime when any manifest outside `DEV_ONLY_AREAS` - * names it in `dependencies`/`optionalDependencies`. A package declared only - * by tooling, test infrastructure, the website, or the demo leaves — whatever - * the declaring section is called — is development-only. + * External npm dependencies, tiered by what reaches a user: a package is runtime + * when any manifest outside `DEV_ONLY_AREAS` names it in + * `dependencies`/`optionalDependencies`, or when a published browser artifact + * carries a copy of it. A package declared only by tooling, test infrastructure, + * the website, or the demo leaves — whatever the declaring section is called, and + * with no shipped artifact carrying it — is development-only. + * @returns every external dependency with its tier and metadata. */ -function collectNpmDeps(): ExternalDep[] { +async function collectNpmDeps(): Promise { const { manifests, names } = loadWorkspaceManifests() - return [...tierExternalDeps(manifests, names)] + return [...tierExternalDeps(manifests, names, await browserBundledExternals(root))] .filter(([name]) => !FIRST_PARTY.has(name)) .sort(([a], [b]) => a.localeCompare(b)) .map(([name, runtime]) => ({ name, ...installedMetadata(name), runtime })) @@ -363,11 +366,23 @@ function collectNpmDeps(): ExternalDep[] { /** * Tier every external dependency the workspace declares. + * + * A package a published browser artifact carries is runtime whatever section + * declares it: the client build inlines its code, or the shell `dist` answers it + * from the frozen module table, so a copy is redistributed even though nothing on + * a user's machine resolves the specifier. Those packages are declared as + * `devDependencies` — `verify-client-runtime-deps` owns that rule — and tiering + * them by section alone would understate the notice. * @param manifests - workspace manifests keyed by repository-relative path. * @param names - every workspace package name, which never counts as external. + * @param bundled - external packages a published browser artifact carries. * @returns each external package mapped to whether it is a runtime dependency. */ -export function tierExternalDeps(manifests: Map, names: Set): Map { +export function tierExternalDeps( + manifests: Map, + names: Set, + bundled: ReadonlySet = new Set(), +): Map { const tiers = new Map() // `tsx` is runtime by fiat: the root source-run scripts execute through its ESM hook. tiers.set('tsx', true) @@ -376,7 +391,7 @@ export function tierExternalDeps(manifests: Map, names: Set { verifyBuildTimePins() - const npm = collectNpmDeps() + const npm = await collectNpmDeps() const runtimeDeps = npm.filter(dep => dep.runtime) const devDeps = npm.filter(dep => !dep.runtime) const vendored = collectVendored() @@ -707,7 +722,7 @@ ${vendored.map(row => `| \`${row.npmName}\` | \`${row.upstreamName}\` | [${row.u ## Runtime npm dependencies -External packages that a workspace package resolves at runtime. The tier covers every plugin a user can mount from \`cordis.yml\` — not only what the \`dsh\` CLI, Web UI, and Python SDK runtime load by default. +External packages that reach a user: a workspace package resolves them at runtime, or a published browser artifact carries a copy of their code. The tier covers every plugin a user can mount from \`cordis.yml\` — not only what the \`dsh\` CLI, Web UI, and Python SDK runtime load by default — and the packages the client build inlines into a plugin bundle or the shell \`dist\`, which are declared as \`devDependencies\` because nothing on a user's machine resolves their specifiers. ${renderNpmTable(runtimeDeps)} @@ -718,7 +733,7 @@ ${renderClaudeDistribution(claudeDistribution)} ## Development-only npm dependencies -External packages **directly declared** only by repository tooling, test infrastructure, the documentation site, the demo leaves, or the native launcher's build workspace. No shipped surface names them itself. A package here may still be pulled in transitively by a runtime dependency — \`pnpm-lock.yaml\` is the authority on the full closure — so this tier records who declares a package, not what a build ultimately bundles. +External packages **directly declared** only by repository tooling, test infrastructure, the documentation site, the demo leaves, or the native launcher's build workspace, and carried by no published artifact. No shipped surface names them itself. A package here may still be pulled in transitively by a runtime dependency — \`pnpm-lock.yaml\` is the authority on the full closure — so this tier records who declares a package, not what a build ultimately bundles. ${renderNpmTable(devDeps)} ${renderNonPermissiveNote(nonPermissiveDev)} @@ -746,8 +761,8 @@ ${BUILD_TIME_TOOLS.map(tool => `| [\`${tool.name}\`](${tool.repo}) | ${tool.lice /** CLI entry: default writes the notices, `--check` fails if the committed copy * is stale. Guarded behind an entry-point check so importing this module for * tests neither regenerates the committed file nor calls process.exit. */ -function main(): void { - const content = render() +async function main(): Promise { + const content = await render() if (process.argv.includes('--check')) { let committed: string | null = null try { @@ -771,5 +786,5 @@ function main(): void { // Run only when invoked as a script, not when imported by a test. if (process.argv[1] !== undefined && import.meta.filename === resolve(process.argv[1])) { - main() + await main() }