diff --git a/.changeset/build-memory.md b/.changeset/build-memory.md new file mode 100644 index 000000000..da0d4c7e0 --- /dev/null +++ b/.changeset/build-memory.md @@ -0,0 +1,8 @@ +--- +'@doc-kit/cli': minor +'@doc-kit/core': minor +'@doc-kit/generator-react': minor +'@node-core/doc-kit-legacy': patch +--- + +Build with 30–50% less memory, in less than half the time: Shiki registers each language once code in it is highlighted, highlighted code reaches pages as static markup rather than a tree per token, the default Vite bundler runs in a child process (`createChildProcess`), `all.html` is rendered by the worker pool and no longer minified, idle workers end after 500ms, and each worker's heap is limited to 512MB by default (the new `workerHeapSize` option, or `--worker-heap-size`) diff --git a/docs/cli.md b/docs/cli.md index e82748e7d..26e3070fa 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -39,4 +39,6 @@ Runs the generators and writes their output. Requires a `target` and an - `--type-map ` {string} Type map URL or path (custom type-name → URL links). - `-p, --threads ` {number} Worker threads to use (minimum 1). +- `--worker-heap-size ` {number} Heap size limit of each worker thread, in + MB (minimum 1). - `--chunk-size ` {number} Items per worker thread (minimum 1). diff --git a/docs/configuration.md b/docs/configuration.md index 1865a918c..027270b49 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -135,8 +135,23 @@ Top-level, alongside `target` and `global`: - `threads` {number} Worker threads used for generation. Defaults to your CPU count. +- `workerHeapSize` {number} Heap size limit of each worker thread (V8's old + space), in MB. Defaults to V8's own limit, at most `512`: V8 lets a heap grow + to several times its live data before collecting it, the more the higher its + limit. A worker running out of memory fails the build; raise it then. When + Node's `--max-old-space-size` is set (`NODE_OPTIONS` takes it too), it limits + the workers instead, and the main thread, which runs the generators with + `threads: 1`. - `chunkSize` {number} Items processed per worker thread. **Default:** `10`. +> [!NOTE] +> On Node.js 26, a worker thread that ends leaves its memory for the threads +> started after it to reuse, rather than giving it back. Building the Node.js +> docs, that's ~800MB held while the site is bundled. Passing +> `--no-memory-pool-share-memory-on-teardown` to the `node` running doc-kit +> gives it back. Node.js 24 gives it back by itself after a few seconds, and +> doesn't have the flag. + ## Generator options Each generator documents its own options on its reference page — see the @@ -196,6 +211,7 @@ own. - `rehypePlugins` {Array} Run on the HTML of the generators rendering Markdown, such as `jsx-ast`, before code is highlighted. - `recmaPlugins` {Array} Run on the JavaScript `jsx-ast` compiles the pages to. + Highlighted code is in it as the markup it renders to, rather than as JSX. A generator only takes the plugins its pipeline has a place for: `jsx-ast` takes all three kinds, `ast`, `metadata`, and `json` take remark plugins, and @@ -261,17 +277,18 @@ precedence): CLI options map to configuration properties: -| CLI Option | Config Property | Example | -| ---------------------- | ------------------ | ------------------------- | -| `--input ` | `global.input` | `--input src/` | -| `--output ` | `global.output` | `--output dist/` | -| `--ignore ` | `global.ignore[]` | `--ignore test/` | -| `--minify` | `global.minify` | `--minify` | -| `--git-ref ` | `global.ref` | `--git-ref v20.0.0` | -| `--version ` | `global.version` | `--version 20.0.0` | -| `--changelog ` | `global.changelog` | `--changelog https://...` | -| `--index ` | `global.index` | `--index file://...` | -| `--type-map ` | `metadata.typeMap` | `--type-map file://...` | -| `--target ` | `target` | `--target json` | -| `--threads ` | `threads` | `--threads 4` | -| `--chunk-size ` | `chunkSize` | `--chunk-size 10` | +| CLI Option | Config Property | Example | +| ------------------------- | ------------------ | ------------------------- | +| `--input ` | `global.input` | `--input src/` | +| `--output ` | `global.output` | `--output dist/` | +| `--ignore ` | `global.ignore[]` | `--ignore test/` | +| `--minify` | `global.minify` | `--minify` | +| `--git-ref ` | `global.ref` | `--git-ref v20.0.0` | +| `--version ` | `global.version` | `--version 20.0.0` | +| `--changelog ` | `global.changelog` | `--changelog https://...` | +| `--index ` | `global.index` | `--index file://...` | +| `--type-map ` | `metadata.typeMap` | `--type-map file://...` | +| `--target ` | `target` | `--target json` | +| `--threads ` | `threads` | `--threads 4` | +| `--worker-heap-size ` | `workerHeapSize` | `--worker-heap-size 1024` | +| `--chunk-size ` | `chunkSize` | `--chunk-size 10` | diff --git a/packages/cli/bin/commands/generate.mjs b/packages/cli/bin/commands/generate.mjs index 95c3cdb67..534701744 100644 --- a/packages/cli/bin/commands/generate.mjs +++ b/packages/cli/bin/commands/generate.mjs @@ -18,6 +18,7 @@ const { runGenerators } = createGenerator(); * @property {string[]} ignore * @property {string} output * @property {number} threads + * @property {number} workerHeapSize * @property {number} chunkSize * @property {string} version * @property {string} changelog @@ -53,6 +54,12 @@ export default new Command('generate') 'Number of threads to use (minimum: 1)' ) ) + .addOption( + new Option( + '--worker-heap-size ', + 'Heap size limit of each worker thread, in MB (minimum: 1)' + ) + ) .addOption( new Option( '--chunk-size ', diff --git a/packages/core/package.json b/packages/core/package.json index 2f2035605..86abb3543 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -57,6 +57,7 @@ "@swc/html-wasm": "^1.16.2", "@swc/wasm": "^1.15.46", "acorn": "^8.17.0", + "birpc": "^4.2.0", "cosmiconfig": "^9.0.2", "dedent": "^1.7.2", "github-slugger": "^2.0.0", diff --git a/packages/core/src/generators.mjs b/packages/core/src/generators.mjs index b73db0967..50bdd5b32 100644 --- a/packages/core/src/generators.mjs +++ b/packages/core/src/generators.mjs @@ -107,7 +107,7 @@ const createGenerator = () => { * @returns {Promise} Results of all requested generators */ const runGenerators = async configuration => { - const { target, threads } = configuration; + const { target, threads, workerHeapSize } = configuration; // Resolve shorthand names and load the full dependency closure up front, // so scheduling below is fully synchronous. @@ -132,7 +132,7 @@ const createGenerator = () => { cache.populateConsumerCounts(targets, specifier => inputOf.get(specifier)); // Create worker pool - pool = createWorkerPool(threads); + pool = createWorkerPool(threads, workerHeapSize); // Schedule all generators for (const specifier of targets) { diff --git a/packages/core/src/generators/metadata/generate.mjs b/packages/core/src/generators/metadata/generate.mjs index b35fa134f..7e7619c60 100644 --- a/packages/core/src/generators/metadata/generate.mjs +++ b/packages/core/src/generators/metadata/generate.mjs @@ -3,8 +3,6 @@ import getConfig from '#utils/configuration/index.mjs'; import { loadFromURL } from '#utils/loaders.mjs'; -import { parseApiDoc } from './utils/parse.mjs'; - /** * Process a chunk of API doc files in a worker thread. * Called by chunk-worker.mjs for parallel processing. @@ -12,6 +10,10 @@ import { parseApiDoc } from './utils/parse.mjs'; * @type {import('./types').Generator['processChunk']} */ export async function processChunk(fullInput, itemIndices, typeMap) { + // Loaded on first use rather than with the generator, so the main thread, + // which never parses, does not load the TypeScript parser (~40MB of WASM) + const { parseApiDoc } = await import('./utils/parse.mjs'); + const results = []; for (const idx of itemIndices) { diff --git a/packages/core/src/plugins/shiki/__tests__/highlighter.test.mjs b/packages/core/src/plugins/shiki/__tests__/highlighter.test.mjs index 91208b8e4..5d9b7416e 100644 --- a/packages/core/src/plugins/shiki/__tests__/highlighter.test.mjs +++ b/packages/core/src/plugins/shiki/__tests__/highlighter.test.mjs @@ -86,6 +86,57 @@ describe('createHighlighter', () => { ); }); + it('registers a bundled language once code in it is highlighted', async () => { + const highlighter = await createHighlighter({ + langAlias: { py: 'python' }, + }); + const loaded = () => highlighter.shiki.getLoadedLanguages(); + + assert.ok(!loaded().includes('python')); + assert.ok(!loaded().includes('javascript')); + + // By its name, an alias of its own, or one of the options + assert.equal(highlighter.resolveLanguage('py'), 'py'); + assert.equal(highlighter.resolveLanguage('mjs'), 'mjs'); + + assert.ok(loaded().includes('python')); + assert.ok(loaded().includes('javascript')); + assert.ok(loaded().includes('cjs')); + + assert.equal(highlighter.resolveLanguage(undefined), 'text'); + assert.equal(highlighter.resolveLanguage('plaintext'), 'plaintext'); + }); + + it('lists the bundled languages without registering them', async () => { + const highlighter = await createHighlighter({ langs: [grammar] }); + + assert.deepStrictEqual( + highlighter.langs.find(({ name }) => name === 'rust'), + { name: 'rust', displayName: 'Rust', aliases: ['rs'] } + ); + assert.equal(highlighter.langs.at(-1), grammar); + assert.ok(!highlighter.shiki.getLoadedLanguages().includes('rust')); + }); + + it('registers the bundled languages a language embeds', async () => { + const highlighter = await createHighlighter({ + langs: [ + { + ...grammar, + name: 'oxcscript', + scopeName: 'source.oxcscript', + embeddedLangs: ['javascript'], + patterns: [{ include: 'source.js' }], + }, + ], + }); + + assert.match( + highlighter.highlightToHtml('const on = 1', 'oxcscript'), + /--shiki-dark/ + ); + }); + it('gives the same highlighter for the same options', async () => { const highlighter = await createHighlighter({ langs: [grammar] }); diff --git a/packages/core/src/plugins/shiki/highlighter.mjs b/packages/core/src/plugins/shiki/highlighter.mjs index 6ecdef8de..687b85b45 100644 --- a/packages/core/src/plugins/shiki/highlighter.mjs +++ b/packages/core/src/plugins/shiki/highlighter.mjs @@ -1,14 +1,24 @@ 'use strict'; +import { createRequire } from 'node:module'; import { endianness } from 'node:os'; -import { LANGS } from '@node-core/rehype-shiki'; import createSyntaxHighlighter from '@node-core/rehype-shiki/highlighter'; +import { isSpecialLang } from 'shiki/core'; +import { bundledLanguagesInfo } from 'shiki/langs'; import { bundledThemes } from 'shiki/themes'; import { importFromURL } from '#utils/loaders.mjs'; import { getMarkdownPlugins } from '#utils/markdown/plugins.mjs'; +const require = createRequire(import.meta.url); + +/** + * A language a highlighter highlights. + * + * @typedef {Pick} Language + */ + /** * A syntax highlighter, creating its Shiki instance on first use. * @@ -17,7 +27,7 @@ import { getMarkdownPlugins } from '#utils/markdown/plugins.mjs'; * @property {(languageId?: string) => string} resolveLanguage - Resolves a language, falling back to plain text for unknown ones * @property {(code: string, lang: string, meta?: Record) => string} highlightToHtml - Highlights code, returning the inner HTML of its `` element * @property {(code: string, lang: string, meta?: Record) => ReturnType} highlightToHast - Highlights code, returning a HAST tree - * @property {Array} langs - The languages it highlights + * @property {Array} langs - The languages it highlights */ /** @@ -53,6 +63,54 @@ let engine; // The highlighters of the options given, by their JSON const highlighters = new Map(); +// What code in an unknown language is highlighted as +const FALLBACK_LANGUAGE = 'text'; + +// The languages Shiki bundles. Registering a grammar takes time and memory, +// and slows down every highlight after it, so each one is only imported and +// registered once code in it is highlighted (see `resolveLanguage`). +const BUNDLED_LANGUAGES = bundledLanguagesInfo.map(({ id, name, aliases }) => ({ + name: id, + displayName: name, + aliases, +})); + +// The bundled language each of their names and aliases stands for, as code +// names its language by either +const BUNDLED_NAMES = new Map( + BUNDLED_LANGUAGES.flatMap(({ name, aliases = [] }) => + [name, ...aliases].map(alias => [alias, name]) + ) +); + +/** + * Imports a bundled language: its grammar, and those of the languages it + * embeds. + * + * @param {string} name - A name or alias of the language + * @returns {Array} + */ +const importBundledLanguage = name => + require(`shiki/langs/${BUNDLED_NAMES.get(name)}.mjs`).default; + +/** + * Adds the bundled languages that languages embed, which Shiki registers + * along with them. + * + * @param {Array} langs - The languages + * @returns {Array} + */ +const withEmbeddedLanguages = langs => { + const names = new Set(langs.map(({ name }) => name)); + + const embedded = langs + .flatMap(({ embeddedLangs = [] }) => embeddedLangs) + .filter(name => !names.has(name) && BUNDLED_NAMES.has(name)) + .flatMap(importBundledLanguage); + + return [...embedded, ...langs]; +}; + /** * Imports a list of options, each given as it is, or as a module (a path * relative to the working directory, or a URL) default-exporting it, or a list @@ -119,13 +177,12 @@ const importHighlighter = async ({ const coreOptions = { engine: regexEngine, - langs: [...LANGS, ...importedLangs], - // A copy, as Shiki adds the aliases of the languages it bundles to it + // The bundled languages are registered as code uses them + langs: withEmbeddedLanguages(importedLangs), + // A copy, as Shiki adds the aliases of the languages it registers to it langAlias: { ...langAlias }, }; - const highlighterOptions = { transformers: importedTransformers }; - // Without themes of its own, the highlighter has a default light and dark one if (themes) { const [light, dark] = await Promise.all([ @@ -134,58 +191,116 @@ const importHighlighter = async ({ ]); coreOptions.themes = [light, dark]; - highlighterOptions.themes = { light: light.name, dark: dark.name }; - highlighterOptions.defaultColor = 'light'; } - let highlighter; + let shiki; + let highlightOptions; /** - * Gives the Shiki highlighter, creating it on first use. + * Gives the Shiki instance, creating it on first use. + * + * @returns {import('shiki').HighlighterCore} */ - const current = () => - (highlighter ??= createSyntaxHighlighter({ - coreOptions, - highlighterOptions, - })); + const current = () => { + if (!shiki) { + ({ shiki } = createSyntaxHighlighter({ coreOptions })); + + // The themes are given by name: Shiki keeps the themes it loaded parsed, + // but parses a theme object it's given again for every highlight + const [light, dark] = shiki.getLoadedThemes(); + + highlightOptions = { + themes: { light, dark }, + defaultColor: 'light', + transformers: importedTransformers, + }; + } + + return shiki; + }; + + /** + * Resolves a language, falling back to plain text for unknown ones. A + * bundled language is registered the first time it's resolved. + * + * @param {string} [languageId] + * @returns {string} + */ + const resolveLanguage = languageId => { + if (!languageId) { + return FALLBACK_LANGUAGE; + } + + const instance = current(); + const name = instance.resolveLangAlias(languageId.toLowerCase()); + + if (isSpecialLang(name) || instance.getLoadedLanguages().includes(name)) { + return languageId; + } + + if (!BUNDLED_NAMES.has(name)) { + return FALLBACK_LANGUAGE; + } + + instance.loadLanguageSync(importBundledLanguage(name)); + + return languageId; + }; + + /** + * The options Shiki highlights code with. + * + * @param {string} lang - The language of the code + * @param {Record} meta - Its metadata + */ + const optionsFor = (lang, meta) => ({ + lang: resolveLanguage(lang), + ...highlightOptions, + meta, + }); return { - langs: coreOptions.langs, + langs: [...BUNDLED_LANGUAGES, ...importedLangs], /** * The Shiki instance. */ get shiki() { - return current().shiki; + return current(); }, - /** - * Resolves a language, falling back to plain text for unknown ones. - * - * @param {string} [languageId] - */ - resolveLanguage: languageId => current().resolveLanguage(languageId), + resolveLanguage, /** * Highlights code, returning the inner HTML of its `` element. * - * @param {...any} args - The code, its language, and its metadata + * @param {string} code - The code + * @param {string} lang - Its language + * @param {Record} [meta] - Its metadata */ - highlightToHtml: (...args) => current().highlightToHtml(...args), + highlightToHtml: (code, lang, meta = {}) => + current() + .codeToHtml(code, optionsFor(lang, meta)) + // Shiki wraps the highlighted code in a
 and a 
+        .match(/(.+?)<\/code>/s)[1],
 
     /**
      * Highlights code, returning a HAST tree.
      *
-     * @param {...any} args - The code, its language, and its metadata
+     * @param {string} code - The code
+     * @param {string} lang - Its language
+     * @param {Record} [meta] - Its metadata
      */
-    highlightToHast: (...args) => current().highlightToHast(...args),
+    highlightToHast: (code, lang, meta = {}) =>
+      current().codeToHast(code, optionsFor(lang, meta)),
   };
 };
 
 /**
  * Creates a highlighter of every language Shiki bundles, with the given
  * options (see `./rehype.mjs`). Its Shiki instance is created on first use,
- * and the same options give the same highlighter.
+ * each bundled language is registered once code in it is highlighted, and the
+ * same options give the same highlighter.
  *
  * @param {import('./rehype.mjs').ShikiOptions} [options] - The options
  * @returns {Promise}
diff --git a/packages/core/src/plugins/type-annotations/highlighter.mjs b/packages/core/src/plugins/type-annotations/highlighter.mjs
index f26f9b25c..9bdd2931c 100644
--- a/packages/core/src/plugins/type-annotations/highlighter.mjs
+++ b/packages/core/src/plugins/type-annotations/highlighter.mjs
@@ -23,11 +23,11 @@ export const createTypeAnnotationHandler = getHighlighter => (state, node) => {
     return typeAnnotationToHast(state, node);
   }
 
-  const { shiki } = getHighlighter();
+  const { shiki, resolveLanguage } = getHighlighter();
   const [lightTheme, darkTheme] = shiki.getLoadedThemes();
 
   const root = shiki.codeToHast(node.value, {
-    lang: node.data?.typescript ? 'typescript' : 'text',
+    lang: resolveLanguage(node.data?.typescript ? 'typescript' : 'text'),
     themes: { light: lightTheme, dark: darkTheme },
     decorations: links.map(({ start, end, href }) => ({
       start,
diff --git a/packages/core/src/threading/__tests__/child-process.test.mjs b/packages/core/src/threading/__tests__/child-process.test.mjs
new file mode 100644
index 000000000..5afcf1b6c
--- /dev/null
+++ b/packages/core/src/threading/__tests__/child-process.test.mjs
@@ -0,0 +1,46 @@
+import { notStrictEqual, rejects, strictEqual } from 'node:assert';
+import { describe, it } from 'node:test';
+
+import createChildProcess from '../child-process.mjs';
+
+const fixture = new URL('./fixtures/child-module.mjs', import.meta.url);
+
+describe('createChildProcess', () => {
+  it("calls the module's exports in a process of its own", async context => {
+    const { rpc, close } = createChildProcess(fixture);
+    context.after(close);
+
+    const pid = await rpc.pid();
+
+    strictEqual(typeof pid, 'number');
+    notStrictEqual(pid, process.pid);
+  });
+
+  it("rejects with the module's error", async context => {
+    const { rpc, close } = createChildProcess(fixture);
+    context.after(close);
+
+    await rejects(rpc.fail(), /Thrown in the child/);
+  });
+
+  it('fails the calls in flight when the process exits', async context => {
+    const { rpc, close } = createChildProcess(fixture);
+    context.after(close);
+
+    await rejects(rpc.exit(), /child-module\.mjs exited \(code 3\)/);
+  });
+
+  it('ends the process on close, failing the calls in flight', async () => {
+    const { rpc, close } = createChildProcess(fixture);
+
+    // Still starting: it cannot have answered yet
+    const call = rpc.pid();
+
+    await close();
+
+    await rejects(call, /exited \(SIGTERM\)/);
+
+    // Once the process is gone, closing again does nothing
+    await close();
+  });
+});
diff --git a/packages/core/src/threading/__tests__/fixtures/child-module.mjs b/packages/core/src/threading/__tests__/fixtures/child-module.mjs
new file mode 100644
index 000000000..d1efbd8de
--- /dev/null
+++ b/packages/core/src/threading/__tests__/fixtures/child-module.mjs
@@ -0,0 +1,12 @@
+// A module for `createChildProcess` to run (see `../child-process.test.mjs`)
+
+/** @returns {number} The ID of the process it runs in */
+export const pid = () => process.pid;
+
+/** Throws, for the error to reach the parent */
+export const fail = () => {
+  throw new Error('Thrown in the child');
+};
+
+/** Ends the process before it can answer */
+export const exit = () => process.exit(3);
diff --git a/packages/core/src/threading/__tests__/fixtures/heap-limit-reporter.mjs b/packages/core/src/threading/__tests__/fixtures/heap-limit-reporter.mjs
new file mode 100644
index 000000000..c7fc00a80
--- /dev/null
+++ b/packages/core/src/threading/__tests__/fixtures/heap-limit-reporter.mjs
@@ -0,0 +1,19 @@
+import { getHeapStatistics } from 'node:v8';
+
+/**
+ * Test generator that reports the heap limit of the worker it runs in, so the
+ * worker pool's resource limits can be asserted.
+ *
+ * @type {GeneratorMetadata}
+ */
+export default {
+  name: 'heap-limit-reporter',
+  version: '1.0.0',
+  description: 'Reports the heap limit of the worker it runs in',
+  dependsOn: 'ast',
+  processChunk: async (_input, itemIndices) =>
+    itemIndices.map(() => getHeapStatistics().heap_size_limit),
+  async generate() {
+    return [getHeapStatistics().heap_size_limit];
+  },
+};
diff --git a/packages/core/src/threading/__tests__/index.test.mjs b/packages/core/src/threading/__tests__/index.test.mjs
index dc6b63a3c..3f6686216 100644
--- a/packages/core/src/threading/__tests__/index.test.mjs
+++ b/packages/core/src/threading/__tests__/index.test.mjs
@@ -1,4 +1,4 @@
-import { deepStrictEqual, strictEqual } from 'node:assert';
+import { deepStrictEqual, ok, strictEqual } from 'node:assert';
 import { describe, it } from 'node:test';
 import { fileURLToPath } from 'node:url';
 
@@ -14,6 +14,10 @@ const pluginsReporterSpecifier = fileURLToPath(
   import.meta.resolve('./fixtures/markdown-plugins-reporter.mjs')
 );
 
+const heapLimitReporter = fileURLToPath(
+  import.meta.resolve('./fixtures/heap-limit-reporter.mjs')
+);
+
 /**
  * Runs a function with the logger temporarily set to the given level.
  *
@@ -114,4 +118,24 @@ describe('createWorkerPool', () => {
       await pool.destroy();
     }
   });
+
+  it("limits each worker's heap to the given size", async () => {
+    const pool = createWorkerPool(1, 512);
+
+    try {
+      const [workerLimit] = await pool.run({
+        generatorSpecifier: heapLimitReporter,
+        input: [null],
+        itemIndices: [0],
+        extra: {},
+        configuration: {},
+      });
+
+      strictEqual(pool.options.resourceLimits.maxOldGenerationSizeMb, 512);
+      // Its old space, plus a young generation of a few dozen MB
+      ok(workerLimit < 1024 ** 3);
+    } finally {
+      await pool.destroy();
+    }
+  });
 });
diff --git a/packages/core/src/threading/child-process-host.mjs b/packages/core/src/threading/child-process-host.mjs
new file mode 100644
index 000000000..34d8d3e85
--- /dev/null
+++ b/packages/core/src/threading/child-process-host.mjs
@@ -0,0 +1,19 @@
+import { createBirpc } from 'birpc';
+
+// What `createChildProcess` runs in the child: it answers the parent's calls to
+// the given module's exports until the parent ends it. Node holds the parent's
+// messages until the listener below is added, so none are lost to the import.
+const functions = await import(process.argv[2]);
+
+createBirpc(
+  { ...functions },
+  {
+    /** @param {unknown} message */
+    post: message => process.send(message),
+    /** @param {(message: unknown) => void} listener */
+    on: listener => {
+      process.on('message', listener);
+    },
+    timeout: -1,
+  }
+);
diff --git a/packages/core/src/threading/child-process.mjs b/packages/core/src/threading/child-process.mjs
new file mode 100644
index 000000000..bec98cb5c
--- /dev/null
+++ b/packages/core/src/threading/child-process.mjs
@@ -0,0 +1,58 @@
+import { fork } from 'node:child_process';
+import { once } from 'node:events';
+
+import { createBirpc } from 'birpc';
+
+const hostScript = new URL('./child-process-host.mjs', import.meta.url);
+
+/**
+ * Runs a module in a child process of its own, and calls its exports over
+ * `birpc`. A process gives all of its memory back when it exits, native
+ * memory included, which a worker thread does not.
+ *
+ * @template T
+ * @param {string | URL} moduleURL - The module to run
+ * @returns {import('./types').ChildProcess}
+ */
+export default function createChildProcess(moduleURL) {
+  const child = fork(hostScript, [String(moduleURL)], {
+    serialization: 'advanced',
+  });
+
+  /** @type {import('birpc').BirpcReturn} */
+  const rpc = createBirpc(
+    {},
+    {
+      /** @param {unknown} message */
+      post: message => child.send(message),
+      // Returns nothing: birpc holds its first call until `on`'s result settles
+      /** @param {(message: unknown) => void} listener */
+      on: listener => {
+        child.on('message', listener);
+      },
+      // A call takes as long as it takes; a child that dies fails its calls
+      timeout: -1,
+    }
+  );
+
+  child.on('error', error => rpc.$close(error));
+  child.on('exit', (code, signal) =>
+    rpc.$close(
+      new Error(
+        `The process running ${moduleURL} exited (${signal ?? `code ${code}`})`
+      )
+    )
+  );
+
+  return {
+    rpc,
+
+    /** Ends the process, and with it everything the module held. */
+    async close() {
+      // `kill` is false once the process is gone, or if it never started
+      if (child.kill()) {
+        await once(child, 'exit');
+      }
+    },
+  };
+}
diff --git a/packages/core/src/threading/index.mjs b/packages/core/src/threading/index.mjs
index 2cd4c9a19..40acc896b 100644
--- a/packages/core/src/threading/index.mjs
+++ b/packages/core/src/threading/index.mjs
@@ -10,11 +10,13 @@ const workerScript = import.meta.resolve('./chunk-worker.mjs');
  * Creates a Piscina worker pool for parallel processing.
  *
  * @param {number} threads - Maximum number of worker threads
+ * @param {number} heapSize - Each worker's heap size limit (old space), in MB
  * @returns {import('piscina').Piscina} Configured Piscina instance
  */
-export default function createWorkerPool(threads) {
+export default function createWorkerPool(threads, heapSize) {
   poolLogger.debug(`WorkerPool initialized`, {
     threads,
+    heapSize,
     workerScript,
   });
 
@@ -22,7 +24,11 @@ export default function createWorkerPool(threads) {
     filename: workerScript,
     minThreads: 0,
     maxThreads: threads,
-    idleTimeout: 1_000,
+    // A worker idle for half a second ends, so its heap is gone during a long
+    // stretch of work on the main thread, such as bundling the site, while the
+    // short gaps between generators leave it running
+    idleTimeout: 500,
+    resourceLimits: { maxOldGenerationSizeMb: heapSize },
     workerData: { logLevel: logger.getLogLevel() },
   });
 }
diff --git a/packages/core/src/threading/types.d.ts b/packages/core/src/threading/types.d.ts
new file mode 100644
index 000000000..78199eea3
--- /dev/null
+++ b/packages/core/src/threading/types.d.ts
@@ -0,0 +1,10 @@
+import type { BirpcReturn } from 'birpc';
+
+// A module running in a child process of its own (see `createChildProcess`).
+// Extend it with the module's methods, calling them through `rpc`.
+export interface ChildProcess {
+  // Calls the module's exports. Calls in flight fail once the process exits.
+  rpc: BirpcReturn;
+  // Ends the process, and with it everything the module held.
+  close(): Promise;
+}
diff --git a/packages/core/src/utils/__tests__/misc.test.mjs b/packages/core/src/utils/__tests__/misc.test.mjs
index 52b739dfd..3809697c0 100644
--- a/packages/core/src/utils/__tests__/misc.test.mjs
+++ b/packages/core/src/utils/__tests__/misc.test.mjs
@@ -8,6 +8,7 @@ import {
   isPlainObject,
   isAsyncIterable,
   omitKeys,
+  flatten,
   deepMerge,
 } from '../misc.mjs';
 
@@ -113,6 +114,22 @@ describe('omitKeys', () => {
   });
 });
 
+describe('flatten', () => {
+  it('should return the same string', () => {
+    let string = '';
+
+    for (let i = 0; i < 1000; i++) {
+      string += `piece ${i} `;
+    }
+
+    assert.strictEqual(flatten(string), string);
+  });
+
+  it('should handle the empty string', () => {
+    assert.strictEqual(flatten(''), '');
+  });
+});
+
 describe('deepMerge', () => {
   it('should merge flat objects with latter-arguments taking precedence', () => {
     const result = deepMerge({ a: 1, b: 2 }, { b: 10, c: 3 });
diff --git a/packages/core/src/utils/configuration/__tests__/index.test.mjs b/packages/core/src/utils/configuration/__tests__/index.test.mjs
index 140c0f350..12ca474f8 100644
--- a/packages/core/src/utils/configuration/__tests__/index.test.mjs
+++ b/packages/core/src/utils/configuration/__tests__/index.test.mjs
@@ -4,8 +4,10 @@ import { tmpdir } from 'node:os';
 import { join } from 'node:path';
 import { describe, it, mock, beforeEach } from 'node:test';
 import { pathToFileURL } from 'node:url';
+import { getHeapStatistics } from 'node:v8';
 
 import logger from '../../../logger/index.mjs';
+import { DEFAULT_MAX_WORKER_HEAP_SIZE } from '../constants.mjs';
 
 // Mock dependencies
 const mockParseChangelog = mock.fn(async changelog => [changelog]);
@@ -211,6 +213,7 @@ describe('config.mjs', () => {
         typeMap: { String: 'string' },
         target: 'json',
         threads: 4,
+        workerHeapSize: 1024,
         chunkSize: 5,
       };
 
@@ -230,6 +233,7 @@ describe('config.mjs', () => {
         metadata: { typeMap: { String: 'string' } },
         target: 'json',
         threads: 4,
+        workerHeapSize: 1024,
         chunkSize: 5,
       });
     });
@@ -330,13 +334,27 @@ describe('config.mjs', () => {
     it('should enforce minimum constraints', async () => {
       const config = await createRunConfiguration({
         threads: -5,
+        workerHeapSize: 0,
         chunkSize: 0,
       });
 
       assert.strictEqual(config.threads, 1);
+      assert.strictEqual(config.workerHeapSize, 1);
       assert.strictEqual(config.chunkSize, 1);
     });
 
+    it("should default the worker heap size to V8's limit, at most 512MB", async () => {
+      const config = await createRunConfiguration({ version: '20.0.0' });
+
+      assert.strictEqual(
+        config.workerHeapSize,
+        Math.min(
+          Math.floor(getHeapStatistics().heap_size_limit / 1024 ** 2),
+          DEFAULT_MAX_WORKER_HEAP_SIZE
+        )
+      );
+    });
+
     it('should work without config file', async () => {
       const config = await createRunConfiguration({
         version: '20.0.0',
diff --git a/packages/core/src/utils/configuration/constants.mjs b/packages/core/src/utils/configuration/constants.mjs
index 1af80ccd0..db14bfcbc 100644
--- a/packages/core/src/utils/configuration/constants.mjs
+++ b/packages/core/src/utils/configuration/constants.mjs
@@ -1,12 +1,30 @@
 'use strict';
 
+import { getHeapStatistics } from 'node:v8';
+
 /**
- * The default `threads` ceiling. Each worker that highlights code holds Shiki's
- * grammars and regex engine (~300MB) on top of the pages it is building, so
- * past a few threads memory, not CPU, is what runs out. `--threads` raises it.
+ * The default `threads` ceiling. Each worker holds a heap of its own, with the
+ * libraries and the pages it is working on, so past a few threads memory, not
+ * CPU, is what runs out. `--threads` raises it.
  */
 export const DEFAULT_MAX_THREADS = 4;
 
+/**
+ * The heap size limit V8 gave this process, in MB.
+ */
+export const HEAP_SIZE_LIMIT = Math.floor(
+  getHeapStatistics().heap_size_limit / 1024 ** 2
+);
+
+/**
+ * The default `workerHeapSize` ceiling, in MB. V8 lets a heap grow to several
+ * times its live data before collecting it, the more the higher its limit (four
+ * times from 2GB up, and on a machine with plenty of memory that limit is 4GB),
+ * so every worker held several times what it was using. The biggest page of the
+ * Node.js docs, `all.html`, takes ~300MB. `--worker-heap-size` raises it.
+ */
+export const DEFAULT_MAX_WORKER_HEAP_SIZE = 512;
+
 /**
  * The default number of items each worker task processes.
  */
diff --git a/packages/core/src/utils/configuration/index.mjs b/packages/core/src/utils/configuration/index.mjs
index 5ab508039..f17eace8c 100644
--- a/packages/core/src/utils/configuration/index.mjs
+++ b/packages/core/src/utils/configuration/index.mjs
@@ -22,7 +22,12 @@ import {
 import { resolveMarkdown } from '#utils/markdown/plugins.mjs';
 import { deepMerge } from '#utils/misc.mjs';
 
-import { DEFAULT_CHUNK_SIZE, DEFAULT_MAX_THREADS } from './constants.mjs';
+import {
+  DEFAULT_CHUNK_SIZE,
+  DEFAULT_MAX_THREADS,
+  DEFAULT_MAX_WORKER_HEAP_SIZE,
+  HEAP_SIZE_LIMIT,
+} from './constants.mjs';
 
 const configExplorer = cosmiconfig('doc-kit');
 
@@ -79,6 +84,8 @@ export const getDefaultConfig = (generators, config) =>
         process.arch === 'riscv64'
           ? 1
           : Math.min(cpus().length, DEFAULT_MAX_THREADS),
+      // No more than V8 gives this process: less on a machine with less memory
+      workerHeapSize: Math.min(HEAP_SIZE_LIMIT, DEFAULT_MAX_WORKER_HEAP_SIZE),
       chunkSize: DEFAULT_CHUNK_SIZE,
     })
   );
@@ -226,6 +233,7 @@ export const createConfigFromCLIOptions = options => ({
   },
   target: options.target,
   threads: options.threads,
+  workerHeapSize: options.workerHeapSize,
   chunkSize: options.chunkSize,
 });
 
@@ -250,7 +258,7 @@ export const assertRunnableOptions = config => {
 /**
  * Creates a complete run configuration by merging config file, user options, and defaults.
  * Processes and validates configuration values including version coercion, changelog parsing,
- * and constraint enforcement for threads and chunk size.
+ * and constraint enforcement for threads, worker heap size and chunk size.
  *
  * @param {import('../../../bin/commands/generate.mjs').CLIOptions} options - User-provided configuration options
  * @returns {Promise} The configuration
@@ -275,6 +283,7 @@ export const createRunConfiguration = async options => {
 
   // These need to be coerced
   merged.threads = Math.max(merged.threads, 1);
+  merged.workerHeapSize = Math.max(merged.workerHeapSize, 1);
   merged.chunkSize = Math.max(merged.chunkSize, 1);
 
   if (process.arch === 'riscv64' && merged.threads > 1) {
diff --git a/packages/core/src/utils/configuration/types.d.ts b/packages/core/src/utils/configuration/types.d.ts
index 2437302bc..13242f6a4 100644
--- a/packages/core/src/utils/configuration/types.d.ts
+++ b/packages/core/src/utils/configuration/types.d.ts
@@ -14,6 +14,9 @@ export type Configuration = {
   // The number of threads the process is allowed to use
   threads: number;
 
+  // The heap size limit of each worker thread (V8's old space), in MB
+  workerHeapSize: number;
+
   // Number of items to process per worker thread
   chunkSize: number;
 } & {
diff --git a/packages/core/src/utils/misc.mjs b/packages/core/src/utils/misc.mjs
index 92d91e01a..2b7af50bd 100644
--- a/packages/core/src/utils/misc.mjs
+++ b/packages/core/src/utils/misc.mjs
@@ -46,6 +46,24 @@ export const omitKeys = (obj, keys = []) =>
     Object.entries(obj).filter(([key]) => !keys.includes(key))
   );
 
+/**
+ * Flattens a string built up by concatenation, in place.
+ *
+ * V8 represents `a + b` as a rope that points at both halves instead of
+ * copying them, so a long string assembled from many small pieces (generated
+ * code, rendered HTML) keeps every one of those pieces alive: around ten times
+ * the size of its text. Reading one of its characters makes V8 flatten it into
+ * a single contiguous string, after which the pieces can be garbage collected.
+ *
+ * @param {string} string
+ * @returns {string} The same string, flattened
+ */
+export const flatten = string => {
+  string.charCodeAt(0);
+
+  return string;
+};
+
 /**
  * Recursively merges plain objects from left to right.
  * @template T
diff --git a/packages/node-legacy/src/legacy-html/plugins/shiki.mjs b/packages/node-legacy/src/legacy-html/plugins/shiki.mjs
index 188915fdd..090939b6d 100644
--- a/packages/node-legacy/src/legacy-html/plugins/shiki.mjs
+++ b/packages/node-legacy/src/legacy-html/plugins/shiki.mjs
@@ -41,6 +41,12 @@ function isCodeBlock(node) {
  * Creates a HAST transformer for Shiki which is used for transforming our codeboxes
  */
 export default function rehypeShikiji() {
+  const [lightTheme, darkTheme] = shikiConfig.themes;
+
+  // Loaded so they can be given by name, as Shiki parses a theme object it's
+  // given again for every code block
+  highlighter.shiki.loadThemeSync(lightTheme, darkTheme);
+
   /**
    * @param {import('hast').Root} tree - The HAST tree to be transformed.
    */
@@ -90,9 +96,9 @@ export default function rehypeShikiji() {
       const { children } = highlighter.shiki.codeToHast(
         preElement.children[0].value,
         {
-          lang: languageId,
+          lang: highlighter.resolveLanguage(languageId),
           // Allows support for dual themes (light, dark) for Shiki
-          themes: { light: shikiConfig.themes[0], dark: shikiConfig.themes[1] },
+          themes: { light: lightTheme.name, dark: darkTheme.name },
         }
       );
 
diff --git a/packages/react/package.json b/packages/react/package.json
index e9199f728..0fb86b6d8 100644
--- a/packages/react/package.json
+++ b/packages/react/package.json
@@ -41,6 +41,7 @@
     "@orama/ui": "^1.5.4",
     "estree-util-to-js": "^2.0.0",
     "github-slugger": "^2.0.0",
+    "hast-util-to-jsx-runtime": "^2.3.6",
     "hast-util-to-string": "^3.0.1",
     "hastscript": "^9.0.1",
     "mdast-util-slice-markdown": "^2.0.1",
diff --git a/packages/react/src/html/README.md b/packages/react/src/html/README.md
index caa8610f6..2fe25ec00 100644
--- a/packages/react/src/html/README.md
+++ b/packages/react/src/html/README.md
@@ -49,11 +49,13 @@ from the module pages' compiled content rather than built again from scratch.
   [`navigation`](#navigation). **Default:** `{}`.
 - `generateAllPage` {boolean} When `true`, writes `all.html`: every module
   page's content on one page, in sidebar order, assembled from the module pages
-  rather than built again. Chunk pages and the index are left out.
+  rather than built again. Chunk pages and the index are left out. It is never
+  minified: the minifier would need about twelve times its size in memory.
   **Default:** `true`.
 - `bundler` {WebBundler} Adapter that bundles the component library and the
   client assets, and compiles page programs. See
-  [Bundler adapters](#bundler-adapters). **Default:** `createViteBundler()`.
+  [Bundler adapters](#bundler-adapters). **Default:** the Vite adapter, run
+  in a child process (see [Vite adapter](#vite-adapter)).
 
 ### `head`
 
@@ -194,6 +196,8 @@ omitted rather than rendered empty.
   JavaScript Node can import.
 - `buildClient` {Function} Bundle the client `entry` into `config.output` and
   return the assets every page loads.
+- `close` {Function} Optional. Release what the adapter holds. Called once
+  every page program is compiled, before the pages are rendered.
 
 The `bundler` option accepts a small Doc Kit adapter rather than configuration
 for a particular build system.
@@ -266,9 +270,11 @@ export default {
 
 ### Vite adapter
 
-When `bundler` is omitted, the generator imports and uses
-`createViteBundler()` automatically. To customize Vite, import the adapter
-directly and pass Vite's `UserConfig` to it:
+When `bundler` is omitted, the generator runs the Vite adapter in a child
+process of its own, and ends it once every page program is compiled. Vite
+bundles with Rolldown, whose native memory a process only gets back when it
+exits, so it is returned before the pages are rendered. To customize Vite,
+import the adapter directly and pass Vite's `UserConfig` to it:
 
 ```js
 // doc-kit.config.mjs
@@ -316,9 +322,9 @@ hashed names of the fonts to preload. A manifest is written either way; pass
 `build: { manifest: true }` (or a file name) to `createViteBundler` to keep it
 in the output for another tool.
 
-The adapter is only ever used on the main thread, so function-valued plugins
-and hooks are supported. Worker threads receive the `html` configuration with
-its function values removed.
+An adapter passed as `bundler` runs in the generator's own process, on the
+main thread, so function-valued plugins and hooks are supported. Worker threads
+receive the `html` configuration with its function values removed.
 
 ### Default `imports`
 
@@ -509,9 +515,9 @@ Since the template supports arbitrary JS expressions, you can use conditionals a
 ${title} ${assets}
 ```
 
-The populated page is the final HTML: it is minified when `minify` is set and
-written as is. Put `${assets}` in the ``, or the page loads no script and
-no stylesheet.
+The populated page is the final HTML: it is minified when `minify` is set
+(except `all.html`) and written as is. Put `${assets}` in the ``, or the
+page loads no script and no stylesheet.
 
 ## Client-side navigation
 
diff --git a/packages/react/src/html/__tests__/generate.test.mjs b/packages/react/src/html/__tests__/generate.test.mjs
index 4b453124e..ee571c2aa 100644
--- a/packages/react/src/html/__tests__/generate.test.mjs
+++ b/packages/react/src/html/__tests__/generate.test.mjs
@@ -150,8 +150,14 @@ describe('web generate', () => {
     assert.match(html, /File system body[\s\S]*Zlib body/);
     assert.doesNotMatch(html, /Index body/);
     // Their tables of contents, concatenated
-    assert.match(html, /href=#fs[\s\S]*href=#zlib/);
+    assert.match(html, /href="#fs"[\s\S]*href="#zlib"/);
     assert.doesNotMatch(html, /View As/);
+    // Unlike the module pages, it is left unminified
+    assert.match(html, //);
+    assert.match(
+      await readFile(join(output, 'fs.html'), 'utf8'),
+      //
+    );
   });
 
   it('renders chunk pages with navigation back to their module', async context => {
@@ -297,6 +303,24 @@ describe('web generate', () => {
     assert.match(code, /__DOC_KIT_PLUGIN__/);
   });
 
+  it('fails with what the default bundler throws, from its child process', async context => {
+    await createTestConfiguration(context);
+
+    const fs = createEntry('fs', 'File system');
+    const content = await buildContent([fs], fs);
+    const page = toPage(content);
+
+    // A page program that cannot compile
+    page.content = '

never closed'; + + await assert.rejects(generate([page]), error => { + assert.ok(error instanceof Error); + assert.doesNotMatch(error.message, /exit code/); + + return true; + }); + }); + it('uses a custom bundler adapter for server and client output', async context => { const { config, output } = await createTestConfiguration(context); const calls = []; diff --git a/packages/react/src/html/bundlers/__tests__/index.test.mjs b/packages/react/src/html/bundlers/__tests__/index.test.mjs new file mode 100644 index 000000000..3d9bde98d --- /dev/null +++ b/packages/react/src/html/bundlers/__tests__/index.test.mjs @@ -0,0 +1,74 @@ +import assert from 'node:assert/strict'; +import { mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { describe, it } from 'node:test'; + +import { + default as getConfig, + setConfig, +} from '@doc-kit/core/utils/configuration/index.mjs'; + +import { resolveBundler } from '../index.mjs'; +import { compile } from '../vite.mjs'; + +await setConfig({ + target: ['html'], + output: join(tmpdir(), 'doc-kit-bundler-test-output'), + version: 'v22.0.0', + changelog: [], + generators: { + html: {}, + }, +}); + +const program = [ + 'import { h as _jsx, Fragment as _Fragment } from "file:///library.mjs";', + 'export const content = () => <>

Hi

;', +].join('\n'); + +describe('resolveBundler', () => { + it('returns a configured bundler as is', async () => { + const bundler = { buildServer() {}, compile() {}, buildClient() {} }; + + const resolved = await resolveBundler(bundler); + + assert.equal(resolved, bundler); + }); + + it('compiles with the Vite adapter, in a child process', async context => { + const bundler = await resolveBundler(); + context.after(() => bundler.close()); + + assert.equal( + await bundler.compile(program, 'fs.jsx'), + await compile(program, 'fs.jsx') + ); + }); + + it('builds an importable library module from a virtual entry', async context => { + const outDir = await mkdtemp(join(tmpdir(), 'doc-kit-bundler-test-')); + context.after(() => rm(outDir, { recursive: true, force: true })); + + const bundler = await resolveBundler(); + context.after(() => bundler.close()); + + const url = await bundler.buildServer({ + entry: 'export { answer } from "virtual:answer";', + virtualImports: { 'virtual:answer': 'export const answer = 42;' }, + outDir, + config: getConfig('html'), + }); + + const library = await import(url); + + assert.equal(library.answer, 42); + }); + + it("rejects with the adapter's error", async context => { + const bundler = await resolveBundler(); + context.after(() => bundler.close()); + + await assert.rejects(bundler.compile('export const = ;', 'broken.jsx')); + }); +}); diff --git a/packages/react/src/html/bundlers/child.mjs b/packages/react/src/html/bundlers/child.mjs new file mode 100644 index 000000000..33983ff11 --- /dev/null +++ b/packages/react/src/html/bundlers/child.mjs @@ -0,0 +1,36 @@ +import createChildProcess from '@doc-kit/core/threading/child-process.mjs'; + +/** + * Runs the Vite adapter in a child process of its own, behind the same + * contract (see `WebBundler`). + * + * Vite bundles with Rolldown, whose native memory a process only gets back + * when it exits: building the Node.js docs leaves ~250MB of it behind. In a + * child, it is all returned once the page programs are compiled (`close`), + * before the pages are rendered, and the generator's own process never loads + * Vite at all. + * + * @returns {import('../types').WebBundler} + */ +export const createChildBundler = () => { + /** @type {import('@doc-kit/core/threading/types').ChildProcess} */ + const { rpc: vite, ...child } = createChildProcess( + new URL('./vite.mjs', import.meta.url) + ); + + return { + ...child, + + /** @param {import('../types').ServerBundleOptions} options */ + buildServer: options => vite.buildServer(options), + + /** + * @param {string} code + * @param {string} fileName + */ + compile: (code, fileName) => vite.compile(code, fileName), + + /** @param {import('../types').ClientBundleOptions} options */ + buildClient: options => vite.buildClient(options), + }; +}; diff --git a/packages/react/src/html/bundlers/index.mjs b/packages/react/src/html/bundlers/index.mjs index 7b426ea1c..58b82beba 100644 --- a/packages/react/src/html/bundlers/index.mjs +++ b/packages/react/src/html/bundlers/index.mjs @@ -1,5 +1,6 @@ /** - * Returns the configured bundler or lazily creates the default Vite adapter. + * Returns the configured bundler, or the default Vite adapter, which runs in a + * child process of its own (see `child.mjs`). * * @param {import('../types').WebBundler|undefined} bundler * @returns {Promise} @@ -9,6 +10,6 @@ export const resolveBundler = async bundler => { return bundler; } - const { createViteBundler } = await import('./vite.mjs'); - return createViteBundler(); + const { createChildBundler } = await import('./child.mjs'); + return createChildBundler(); }; diff --git a/packages/react/src/html/generate.mjs b/packages/react/src/html/generate.mjs index d48f6a598..03a8f3a5c 100644 --- a/packages/react/src/html/generate.mjs +++ b/packages/react/src/html/generate.mjs @@ -12,48 +12,39 @@ import { resolveBundler } from './bundlers/index.mjs'; import { buildAllPage } from './utils/all.mjs'; import { copyStaticAssets } from './utils/copying.mjs'; import createProgramBuilder, { moduleFileName } from './utils/generate.mjs'; -import { createVirtualImports } from './utils/processing.mjs'; import { createPageWriter } from './utils/render.mjs'; const htmlLogger = logger.child('html'); /** - * Main generation function: turns the pages' JSX into the static site. + * Bundles the component library and the client assets, and compiles every + * page's program into `outDir`, for the worker pool to render. * - * Receives `jsx-ast`'s output as `{ data, headings, readingTime, content }` - * items, `content` being each page's JSX code. The site is then built in - * pieces that are each as small as they can be: + * Nothing is bundled or compiled past this point, so the bundler is closed + * here: whatever it holds is released before the pages are rendered. * - * 1. The component library is bundled once, for the server. - * 2. The client assets are bundled once; every page loads the same ones. - * 3. Each page's program is compiled (JSX to a plain module) and written to a - * temporary directory, one at a time, so no page is held longer than that. - * 4. The worker pool imports, renders, templates, minifies and writes the - * pages, one page in memory per worker. - * 5. `all.html`, when enabled, is a program that imports the module pages' - * content, so it is written last from what was already compiled. - * - * @type {import('./types').Generator['generate']} + * @param {object} options + * @param {Array} options.pages - Every page, in render order + * @param {import('./types').ComposedPage} [options.all] - `all.html`, when enabled + * @param {string} options.outDir - Where the library and the programs are written + * @param {import('./types').ResolvedWebConfiguration} options.config + * @returns {Promise<{ assets: import('./types').ClientAssets, tasks: Array }>} */ -export async function generate(input, worker) { - const config = getConfig('html'); - - const template = await readFile(config.templatePath, 'utf-8'); - - const pages = [...input]; - const all = config.generateAllPage ? buildAllPage(pages) : undefined; - +const buildPrograms = async ({ pages, all, outDir, config }) => { // Every page's metadata, in render order — the sidebar, the index and the // cross links need the whole set. const datas = [...pages, ...(all ? [all] : [])].map(({ data }) => data); - const bundler = await resolveBundler(config.bundler); + // Loaded here rather than with the generator, so the threads rendering + // pages, which load this module too, never load what only this needs + const { createVirtualImports } = await import('./utils/config.mjs'); + const { buildLibraryProgram, buildPageProgram, clientProgram } = createProgramBuilder(); - // The built library and the compiled page programs live here until every - // page is written; the directory is removed afterwards - const outDir = await mkdtemp(join(tmpdir(), 'doc-kit-html-')); + // Resolved last: the default bundler runs in a child process, which only + // ends once it is closed + const bundler = await resolveBundler(config.bundler); try { const libraryURL = await bundler.buildServer({ @@ -85,14 +76,13 @@ export async function generate(input, worker) { const compile = async page => { const file = join(modulesDir, moduleFileName(page.data.api)); - await writeFile( - file, - await bundler.compile( - buildPageProgram(page, libraryURL), - `${page.data.api}.jsx` - ) + const code = await bundler.compile( + buildPageProgram(page, libraryURL), + `${page.data.api}.jsx` ); + await writeFile(file, code); + const { data, headings, readingTime } = page; return { @@ -106,21 +96,75 @@ export async function generate(input, worker) { const tasks = []; for (const page of pages) { - tasks.push(await compile(page)); + const task = await compile(page); + + tasks.push(task); + } + + // The composed page imports the other pages' compiled programs, which + // exist from here on, so it is rendered by the worker pool alongside them. + // Rendering it on this thread instead would hold the whole site in it and + // block the pool from shutting down its idle workers until it is done. + // It goes first: it takes by far the longest, so the other pages are + // rendered while it is. + // + // It is not minified. The minifier's memory grows to about twelve times + // the page it is given and is never returned, and this page is the whole + // site: minifying the Node.js docs' ~35MB `all.html` takes ~400MB and over + // a second, for a page 2% smaller once compressed. + if (all) { + const allTask = await compile(all); + + tasks.unshift({ ...allTask, minify: false }); } htmlLogger.debug(`Compiled ${tasks.length} page programs`); - const writePages = createPageWriter(worker); - const extra = { template, assets }; + return { assets, tasks }; + } finally { + await bundler.close?.(); + } +}; + +/** + * Main generation function: turns the pages' JSX into the static site. + * + * Receives `jsx-ast`'s output as `{ data, headings, readingTime, content }` + * items, `content` being each page's JSX code. The site is then built in + * pieces that are each as small as they can be: + * + * 1. The component library is bundled once, for the server. + * 2. The client assets are bundled once; every page loads the same ones. + * 3. Each page's program is compiled (JSX to a plain module) and written to a + * temporary directory, one at a time, so no page is held longer than that. + * 4. `all.html`, when enabled, is a program that imports the module pages' + * content, so it is compiled from what was already compiled. + * 5. The worker pool imports, renders, templates, minifies and writes the + * pages, one page in memory per worker. + * + * @type {import('./types').Generator['generate']} + */ +export async function generate(input, worker) { + const config = getConfig('html'); - await writePages(tasks, extra); + const template = await readFile(config.templatePath, 'utf-8'); - // The composed page imports the others' compiled programs, so it can only - // be rendered once those exist — which they now do. - if (all) { - await writePages([await compile(all)], extra); - } + const pages = [...input]; + const all = config.generateAllPage ? buildAllPage(pages) : undefined; + + // The built library and the compiled page programs live here until every + // page is written; the directory is removed afterwards + const outDir = await mkdtemp(join(tmpdir(), 'doc-kit-html-')); + + try { + const { assets, tasks } = await buildPrograms({ + pages, + all, + outDir, + config, + }); + + await createPageWriter(worker)(tasks, { template, assets }); } finally { await rm(outDir, { recursive: true, force: true }); } diff --git a/packages/react/src/html/types.d.ts b/packages/react/src/html/types.d.ts index 5bfb72510..1fb4b60e3 100644 --- a/packages/react/src/html/types.d.ts +++ b/packages/react/src/html/types.d.ts @@ -45,6 +45,8 @@ export type Page = PageCode | ComposedPage; export type PageTask = Pick & { // `file:` URL of the compiled module. moduleURL: string; + // `false` leaves the page unminified even when `minify` is set. + minify?: boolean; }; // The client assets every page loads, as paths relative to the output root. @@ -90,6 +92,9 @@ export type WebBundler = { // Bundles the client entry into `config.output` and returns the assets every // page must load. buildClient(options: ClientBundleOptions): Promise; + // Releases what the bundler holds. Called once every page program is + // compiled, before the pages are rendered, and not called again after. + close?(): Promise; }; export type Configuration = { diff --git a/packages/react/src/html/utils/__tests__/config.test.mjs b/packages/react/src/html/utils/__tests__/config.test.mjs index 94e2e0c38..886d14de8 100644 --- a/packages/react/src/html/utils/__tests__/config.test.mjs +++ b/packages/react/src/html/utils/__tests__/config.test.mjs @@ -1,18 +1,24 @@ import assert from 'node:assert/strict'; +import { createRequire } from 'node:module'; import { describe, it, mock } from 'node:test'; +import { fileURLToPath } from 'node:url'; import { setConfig } from '@doc-kit/core/utils/configuration/index.mjs'; import { loadMarkdownPlugins } from '@doc-kit/core/utils/markdown/plugins.mjs'; import { SemVer } from 'semver'; -mock.module('@node-core/rehype-shiki', { +// The languages Shiki bundles, as `@doc-kit/core` imports them +const core = createRequire( + fileURLToPath(import.meta.resolve('@doc-kit/core/package.json')) +); + +mock.module(core.resolve('shiki/langs'), { exports: { - LANGS: [ - { name: 'javascript', aliases: ['js'], displayName: 'JavaScript' }, - { name: 'typescript', aliases: ['ts'], displayName: 'TypeScript' }, - { name: 'python', displayName: 'Python' }, + bundledLanguagesInfo: [ + { id: 'javascript', name: 'JavaScript', aliases: ['js'] }, + { id: 'typescript', name: 'TypeScript', aliases: ['ts'] }, + { id: 'python', name: 'Python' }, ], - default: async () => ({}), }, }); @@ -28,6 +34,7 @@ await loadMarkdownPlugins({ const { default: createConfigSource, + createVirtualImports, buildVersionEntries, buildPageList, buildChunkGroups, @@ -44,7 +51,7 @@ const config = await setConfig({ }); // Loading the real `html` generator would pull in the full rendering stack -// (which the `rehype-shiki` mock above cannot satisfy), so its resolved +// (which the `shiki/langs` mock above cannot satisfy), so its resolved // configuration is stubbed in directly. config.html = { ...config.global, @@ -312,6 +319,20 @@ describe('createConfigSource', () => { }); }); +describe('createVirtualImports', () => { + it('adds the `#theme/config` module to the configured ones', () => { + const datas = [makeEntry('fs', 'File System', '/fs')]; + const imports = createVirtualImports( + datas, + { 'virtual:extra': 'export default 1;' }, + true + ); + + assert.equal(imports['virtual:extra'], 'export default 1;'); + assert.equal(imports['#theme/config'], createConfigSource(datas, true)); + }); +}); + describe('buildLanguageDisplayNameMap', () => { it('returns entries suitable for constructing a Map', () => { const result = buildLanguageDisplayNameMap(); diff --git a/packages/react/src/html/utils/config.mjs b/packages/react/src/html/utils/config.mjs index 512cbdbf2..594b73e7f 100644 --- a/packages/react/src/html/utils/config.mjs +++ b/packages/react/src/html/utils/config.mjs @@ -223,3 +223,20 @@ export default function createConfigSource(input, server = false) { return lines.join('\n'); } + +/** + * Creates the virtual imports for one bundle target. + * + * Only the thread bundling the site needs this module: it loads Shiki (see + * `buildLanguageDisplayNameMap`) and a Markdown processor of its own, which + * the threads rendering pages have no use for. + * + * @param {Array} datas - Per-page metadata + * @param {Record} virtualImports + * @param {boolean} server + * @returns {Record} + */ +export const createVirtualImports = (datas, virtualImports, server) => ({ + ...virtualImports, + '#theme/config': createConfigSource(datas, server), +}); diff --git a/packages/react/src/html/utils/processing.mjs b/packages/react/src/html/utils/processing.mjs index b18a1d61c..4dc60378e 100644 --- a/packages/react/src/html/utils/processing.mjs +++ b/packages/react/src/html/utils/processing.mjs @@ -3,22 +3,8 @@ import { populate } from '@doc-kit/core/utils/configuration/templates.mjs'; import { ROUTER_DATA_ATTRIBUTE } from '../ui/constants.mjs'; import { THEME_SCRIPT } from '../ui/theme-script.mjs'; -import createConfigSource from './config.mjs'; import { relativeOrAbsolute } from './relativeOrAbsolute.mjs'; -/** - * Creates the virtual imports for one bundle target. - * - * @param {Array} datas - Per-page metadata - * @param {Record} virtualImports - * @param {boolean} server - * @returns {Record} - */ -export const createVirtualImports = (datas, virtualImports, server) => ({ - ...virtualImports, - '#theme/config': createConfigSource(datas, server), -}); - /** * Populates a template string by evaluating it as a JavaScript template literal, * allowing full JS expression syntax (e.g., ${if ...}, ${JSON.stringify(...)}). diff --git a/packages/react/src/html/utils/render.mjs b/packages/react/src/html/utils/render.mjs index 75312c627..b45b7e5d9 100644 --- a/packages/react/src/html/utils/render.mjs +++ b/packages/react/src/html/utils/render.mjs @@ -5,8 +5,7 @@ import { dirname, join } from 'node:path'; import logger from '@doc-kit/core/logger/index.mjs'; import getConfig from '@doc-kit/core/utils/configuration/index.mjs'; -import { minifyHTML } from '@doc-kit/core/utils/html-minifier.mjs'; -import { omitKeys } from '@doc-kit/core/utils/misc.mjs'; +import { flatten, omitKeys } from '@doc-kit/core/utils/misc.mjs'; import { pageFileName, populatePage } from './processing.mjs'; @@ -35,7 +34,7 @@ export const processChunk = async (tasks, indices, { template, assets }) => { const written = []; for (const index of indices) { - const { moduleURL, data, headings, readingTime } = tasks[index]; + const { moduleURL, data, headings, readingTime, minify } = tasks[index]; const { default: render } = await import(moduleURL); @@ -47,14 +46,23 @@ export const processChunk = async (tasks, indices, { template, assets }) => { 'changes', ]); + const dehydrated = await render({ metadata, headings, readingTime }); + let html = populatePage({ template, data, - dehydrated: await render({ metadata, headings, readingTime }), + // Rendered a tag at a time, so flattened before it is templated and + // minified (see `flatten`): as rendered, `all.html` alone is ~400MB + dehydrated: flatten(dehydrated), assets, }); - if (config.minify) { + if (config.minify && minify !== false) { + // Loaded on first use, so that only the threads rendering pages load + // the minifier (~30MB of WASM) + const { minifyHTML } = + await import('@doc-kit/core/utils/html-minifier.mjs'); + html = await minifyHTML(html); } diff --git a/packages/react/src/jsx-ast/generate.mjs b/packages/react/src/jsx-ast/generate.mjs index e04474120..b3adc8628 100644 --- a/packages/react/src/jsx-ast/generate.mjs +++ b/packages/react/src/jsx-ast/generate.mjs @@ -1,8 +1,8 @@ import getConfig from '@doc-kit/core/utils/configuration/index.mjs'; import { groupNodesByModule } from '@doc-kit/core/utils/generators.mjs'; +import { flatten } from '@doc-kit/core/utils/misc.mjs'; import { jsx, toJs } from 'estree-util-to-js'; -import buildContent from './utils/buildContent.mjs'; import { getSortedHeadNodes } from './utils/getSortedHeadNodes.mjs'; import { buildNotFoundPage } from './utils/synthetic/404.mjs'; @@ -15,9 +15,16 @@ import { buildNotFoundPage } from './utils/synthetic/404.mjs'; * crosses back to or accumulates on the main thread. Only the code string, the * table of contents and the page metadata are returned. * + * The code is generated a few characters at a time, so it is flattened before + * the next page is built (see `flatten`). + * * @type {import('./types').Generator['processChunk']} */ export async function processChunk(slicedInput, itemIndices) { + // Loaded on first use rather than with the generator, so the main thread, + // which never builds a page, does not load what building one takes + const { default: buildContent } = await import('./utils/buildContent.mjs'); + const results = []; for (const idx of itemIndices) { @@ -25,7 +32,10 @@ export async function processChunk(slicedInput, itemIndices) { const { content, ...page } = await buildContent(entries, head); - results.push({ ...page, content: toJs(content, { handlers: jsx }).value }); + results.push({ + ...page, + content: flatten(toJs(content, { handlers: jsx }).value), + }); } return results; diff --git a/packages/react/src/jsx-ast/index.mjs b/packages/react/src/jsx-ast/index.mjs index b1dd1aeb5..59a9ff298 100644 --- a/packages/react/src/jsx-ast/index.mjs +++ b/packages/react/src/jsx-ast/index.mjs @@ -1,10 +1,10 @@ 'use strict'; import { getHighlighter } from '@doc-kit/core/plugins/shiki/highlighter.mjs'; -import { createTypeAnnotationHandler } from '@doc-kit/core/plugins/type-annotations/highlighter.mjs'; import { AST_NODE_TYPES } from './constants.mjs'; import { generate, processChunk } from './generate.mjs'; +import { embedHighlightedTypes } from './plugins/static-markup.mjs'; /** * Generator for converting MDAST to JSX AST. @@ -40,9 +40,10 @@ export default { // within our Markdown and we trust the sources of the Markdown files allowDangerousHtml: true, passThrough: ['element', ...Object.values(AST_NODE_TYPES.MDX)], - // Types are highlighted, with their links embedded + // Types are highlighted, with their links embedded, as static + // markup (see `./plugins/static-markup.mjs`) handlers: { - typeAnnotation: createTypeAnnotationHandler(() => + typeAnnotation: embedHighlightedTypes(() => getHighlighter('jsx-ast') ), }, @@ -52,6 +53,8 @@ export default { // The configured rehype plugins run before code blocks are highlighted '...', '@doc-kit/core/plugins/shiki/rehype.mjs', + // Highlighted code reaches the pages as the markup it renders to + './plugins/static-markup.mjs', './plugins/transformer.mjs', ], recmaPlugins: ['rehype-recma', 'recma-jsx', '...', 'recma-stringify'], diff --git a/packages/react/src/jsx-ast/plugins/__tests__/static-markup.test.mjs b/packages/react/src/jsx-ast/plugins/__tests__/static-markup.test.mjs new file mode 100644 index 000000000..c31e6fc5d --- /dev/null +++ b/packages/react/src/jsx-ast/plugins/__tests__/static-markup.test.mjs @@ -0,0 +1,187 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; + +import { createHighlighter } from '@doc-kit/core/plugins/shiki/highlighter.mjs'; +import { toString } from 'hast-util-to-string'; +import rehypeRaw from 'rehype-raw'; +import { unified } from 'unified'; + +import rehypeStaticMarkup, { + embedHighlightedTypes, +} from '../static-markup.mjs'; + +const highlighter = await createHighlighter(); + +const embedTypes = embedHighlightedTypes(() => highlighter); + +// A minimal mdast-util-to-hast state: the handlers only use patch/applyData +const state = { patch: () => {}, applyData: (_, result) => result }; + +/** + * The text an HTML fragment reads as, parsed as a browser would parse it. + * + * @param {string} html + */ +const textContent = html => + toString( + unified() + .use(rehypeRaw) + .runSync({ type: 'root', children: [{ type: 'raw', value: html }] }) + ); + +/** + * Reads an attribute of a JSX element: its value, or the value of the object + * it's given (as `dangerouslySetInnerHTML` is). + * + * @param {{ attributes: Array<{ name: string, value: unknown }> }} element + * @param {string} name + */ +const attribute = (element, name) => { + const { value } = element.attributes.find(entry => entry.name === name); + + if (typeof value === 'string') { + return value; + } + + const [{ expression }] = value.data.estree.body; + + return Object.fromEntries( + expression.properties.map(({ key, value: entry }) => [ + key.name, + entry.value, + ]) + ); +}; + +/** + * The markup a JSX `` holds through `dangerouslySetInnerHTML`. + * + * @param {{ attributes: Array<{ name: string, value: unknown }> }} element + */ +const innerHTML = element => + attribute(element, 'dangerouslySetInnerHTML').__html; + +describe('rehypeStaticMarkup', () => { + const embed = rehypeStaticMarkup(); + + it("embeds a highlighted block's code as the markup it renders to", () => { + const tree = highlighter.highlightToHast('const a = 1;', 'js'); + const [pre] = tree.children; + + embed(tree); + + // The
 stays as it is, for the code box to render
+    assert.equal(pre.tagName, 'pre');
+    assert.match(pre.properties.class, /^shiki /);
+
+    const [code] = pre.children;
+
+    assert.equal(code.type, 'mdxJsxFlowElement');
+    assert.equal(code.name, 'code');
+    assert.equal(code.children.length, 0);
+    // Rendered by Preact, as the page itself would render it
+    assert.match(
+      innerHTML(code),
+      /^const<\/span>/i
+    );
+    assert.equal(textContent(innerHTML(code)), 'const a = 1;');
+  });
+
+  it('leaves code that was not highlighted as it is', () => {
+    const code = {
+      type: 'element',
+      tagName: 'code',
+      properties: { className: ['language-js'] },
+      children: [{ type: 'text', value: 'a();' }],
+    };
+
+    const tree = {
+      type: 'root',
+      children: [
+        { type: 'element', tagName: 'pre', properties: {}, children: [code] },
+      ],
+    };
+
+    embed(tree);
+
+    assert.equal(tree.children[0].children[0], code);
+  });
+});
+
+describe('embedHighlightedTypes', () => {
+  const makeNode = (value, data) => ({ type: 'typeAnnotation', value, data });
+
+  it('embeds a highlighted type as one inline , its links included', () => {
+    const element = embedTypes(
+      state,
+      makeNode('Promise', {
+        typescript: true,
+        links: [{ start: 0, end: 7, text: 'Promise', href: 'mdn.io/promise' }],
+      })
+    );
+
+    assert.equal(element.type, 'mdxJsxTextElement');
+    assert.equal(element.name, 'code');
+    assert.match(attribute(element, 'className'), /^shiki .* type$/);
+
+    const html = innerHTML(element);
+
+    assert.match(html, //);
+    assert.doesNotMatch(html, /');
+  });
+
+  it('highlights the same type once for each highlighter', async () => {
+    let calls = 0;
+
+    // The highlighter, counting what its Shiki instance highlights
+    const counting = {
+      resolveLanguage: highlighter.resolveLanguage,
+      get shiki() {
+        const { shiki } = highlighter;
+
+        return {
+          ...shiki,
+          codeToHast: (...args) => {
+            calls++;
+
+            return shiki.codeToHast(...args);
+          },
+        };
+      },
+    };
+
+    const themed = await createHighlighter({
+      themes: { light: 'github-light', dark: 'github-dark' },
+    });
+
+    let current = counting;
+
+    const embed = embedHighlightedTypes(() => current);
+
+    const promise = () =>
+      makeNode('Promise', {
+        typescript: true,
+        links: [{ start: 0, end: 7, text: 'Promise', href: 'mdn.io/promise' }],
+      });
+
+    const first = embed(state, promise());
+    const second = embed(state, promise());
+
+    assert.equal(calls, 1);
+    // Each type is a node of its own, with the same markup
+    assert.notEqual(second, first);
+    assert.equal(innerHTML(second), innerHTML(first));
+
+    current = themed;
+
+    assert.notEqual(innerHTML(embed(state, promise())), innerHTML(first));
+  });
+
+  it('leaves a type that was not highlighted as it is', () => {
+    const element = embedTypes(state, makeNode('Whatever', { links: [] }));
+
+    assert.equal(element.type, 'element');
+    assert.deepStrictEqual(element.properties, { className: ['type'] });
+  });
+});
diff --git a/packages/react/src/jsx-ast/plugins/static-markup.mjs b/packages/react/src/jsx-ast/plugins/static-markup.mjs
new file mode 100644
index 000000000..80ea4156a
--- /dev/null
+++ b/packages/react/src/jsx-ast/plugins/static-markup.mjs
@@ -0,0 +1,144 @@
+'use strict';
+
+import { createTypeAnnotationHandler } from '@doc-kit/core/plugins/type-annotations/highlighter.mjs';
+import { toJsxRuntime } from 'hast-util-to-jsx-runtime';
+import { renderToString } from 'preact-render-to-string';
+import { Fragment, jsx, jsxs } from 'preact/jsx-runtime';
+import { SKIP, visit } from 'unist-util-visit';
+
+import { createJSXElement } from '../utils/ast.mjs';
+
+// Highlighted code is static: an island adopts it without rendering it again.
+// So it reaches the page as the markup it renders to, rather than each of its
+// tokens becoming a hast element, then a JSX element, then generated code,
+// only to be rendered back into that same markup.
+
+/**
+ * Whether an element is code Shiki highlighted: the `
` of a block, or
+ * the `` of a type.
+ *
+ * @param {import('hast').Element} element
+ */
+const isHighlighted = ({ properties = {} }) =>
+  [properties.class ?? properties.className]
+    .flat()
+    .join(' ')
+    .split(' ')
+    .includes('shiki');
+
+/**
+ * Renders hast with the JSX runtime and the renderer of the pages, so the
+ * markup is the one a page would render from that hast itself.
+ *
+ * @param {Array} children
+ * @returns {string}
+ */
+const render = children =>
+  renderToString(
+    toJsxRuntime({ type: 'root', children }, { Fragment, jsx, jsxs })
+  );
+
+/**
+ * The attributes and the markup of the `` of highlighted code.
+ *
+ * @param {import('hast').Element} code - The ``
+ * @returns {{ attributes: Record, html: string }}
+ */
+const toMarkup = ({
+  properties: { class: className, ...properties },
+  children,
+}) => ({
+  attributes: {
+    className: [className].flat().join(' ') || undefined,
+    ...properties,
+  },
+  html: render(children),
+});
+
+/**
+ * A JSX `` holding highlighted markup as it is.
+ *
+ * @param {ReturnType} markup
+ * @param {boolean} [inline] - Whether it's a type's, rather than a block's
+ */
+const createCodeElement = ({ attributes, html }, inline = false) =>
+  createJSXElement('code', {
+    inline,
+    ...attributes,
+    dangerouslySetInnerHTML: { __html: html },
+  });
+
+/**
+ * Embeds the code blocks of a tree that Shiki highlighted.
+ *
+ * @template {import('hast').Root} T
+ * @param {T} tree
+ * @returns {T}
+ */
+export const embedHighlightedBlocks = tree => {
+  visit(tree, 'element', node => {
+    const [code] = node.children;
+
+    if (node.tagName === 'pre' && code?.tagName === 'code') {
+      if (isHighlighted(node)) {
+        node.children[0] = createCodeElement(toMarkup(code));
+      }
+
+      return SKIP;
+    }
+  });
+
+  return tree;
+};
+
+/**
+ * Creates the `typeAnnotation` handler of `remark-rehype` highlighting types
+ * and embedding them.
+ *
+ * Every highlighted type is kept, by highlighter: the same few hundred types
+ * are highlighted thousands of times, `{string}` alone on most pages.
+ *
+ * @param {() => import('@doc-kit/core/plugins/shiki/highlighter.mjs').SyntaxHighlighter} getHighlighter - Gives the highlighter, once a type is highlighted
+ * @returns {(state: import('mdast-util-to-hast').State, node: import('mdast').Node) => import('hast').ElementContent}
+ */
+export const embedHighlightedTypes = getHighlighter => {
+  const highlight = createTypeAnnotationHandler(getHighlighter);
+  const highlighted = new WeakMap();
+
+  return (state, node) => {
+    const highlighter = getHighlighter();
+
+    if (!highlighted.has(highlighter)) {
+      highlighted.set(highlighter, new Map());
+    }
+
+    const types = highlighted.get(highlighter);
+    const key = JSON.stringify([node.value, node.data]);
+
+    if (!types.has(key)) {
+      const result = highlight(state, node);
+
+      // A type that didn't parse, or links nowhere, isn't highlighted
+      if (!isHighlighted(result)) {
+        return result;
+      }
+
+      types.set(key, toMarkup(result));
+    }
+
+    const code = createCodeElement(types.get(key), true);
+
+    state.patch(node, code);
+
+    return code;
+  };
+};
+
+/**
+ * Embeds the code blocks that Shiki highlighted.
+ *
+ * @type {import('unified').Plugin<[], import('hast').Root>}
+ */
+export default function rehypeStaticMarkup() {
+  return embedHighlightedBlocks;
+}
diff --git a/packages/react/src/jsx-ast/utils/__tests__/getFullName.test.mjs b/packages/react/src/jsx-ast/utils/__tests__/getFullName.test.mjs
new file mode 100644
index 000000000..009102bc6
--- /dev/null
+++ b/packages/react/src/jsx-ast/utils/__tests__/getFullName.test.mjs
@@ -0,0 +1,199 @@
+import assert from 'node:assert/strict';
+import { describe, it } from 'node:test';
+
+import { getFullName } from '../getFullName.mjs';
+
+describe('getFullName', () => {
+  it('returns fallback when name equals text', () => {
+    const result = getFullName({ name: 'test', text: 'test' }, 'fallback');
+    assert.strictEqual(result, 'fallback');
+  });
+
+  it('returns name as fallback when name equals text and no fallback provided', () => {
+    const result = getFullName({ name: 'test', text: 'test' });
+    assert.strictEqual(result, 'test');
+  });
+
+  it('extracts inline code that includes the name', () => {
+    const result = getFullName({
+      name: 'myFunc',
+      text: 'This is `myFunc(param1, param2)` function',
+    });
+    assert.strictEqual(result, 'myFunc');
+  });
+
+  it('handles inline code with extra content after name', () => {
+    const result = getFullName({
+      name: 'authenticate',
+      text: 'The `authenticate(user, password)` method',
+    });
+    assert.strictEqual(result, 'authenticate');
+  });
+
+  it('strips quotes from the beginning', () => {
+    const result = getFullName({
+      name: 'func',
+      text: 'The `"func"()` method',
+    });
+    assert.strictEqual(result, 'func');
+  });
+
+  it('strips single quotes from the beginning', () => {
+    const result = getFullName({
+      name: 'func',
+      text: "The `'func'()` method",
+    });
+    assert.strictEqual(result, 'func');
+  });
+
+  it('strips "new" keyword from the beginning', () => {
+    const result = getFullName({
+      name: 'Constructor',
+      text: 'The `new Constructor()` call',
+    });
+    assert.strictEqual(result, 'Constructor');
+  });
+
+  it('strips "new " with space from the beginning', () => {
+    const result = getFullName({
+      name: 'MyClass',
+      text: 'The `new MyClass(param)` constructor',
+    });
+    assert.strictEqual(result, 'MyClass');
+  });
+
+  it('returns fallback when no inline code found', () => {
+    const result = getFullName(
+      {
+        name: 'func',
+        text: 'This is a function without code blocks',
+      },
+      'fallback'
+    );
+    assert.strictEqual(result, 'fallback');
+  });
+
+  it('returns fallback when inline code does not include name', () => {
+    const result = getFullName(
+      {
+        name: 'myFunc',
+        text: 'This is `otherFunc()` function',
+      },
+      'fallback'
+    );
+    assert.strictEqual(result, 'fallback');
+  });
+
+  it('handles empty inline code', () => {
+    const result = getFullName(
+      {
+        name: 'func',
+        text: 'This has `` empty code',
+      },
+      'fallback'
+    );
+    assert.strictEqual(result, 'fallback');
+  });
+
+  it('handles multiple inline code blocks, uses first match', () => {
+    const result = getFullName({
+      name: 'func',
+      text: 'This has `func()` and `other()` code',
+    });
+    assert.strictEqual(result, 'func');
+  });
+
+  it('handles complex inline code with parameters', () => {
+    const result = getFullName({
+      name: 'processData',
+      text: 'The `processData(input, options = {})` method processes data',
+    });
+    assert.strictEqual(result, 'processData');
+  });
+
+  it('strips both quotes and new keyword', () => {
+    const result = getFullName({
+      name: 'MyClass',
+      text: '`"new MyClass"()`',
+    });
+    assert.strictEqual(result, 'MyClass');
+  });
+
+  it('handles text with no backticks', () => {
+    const result = getFullName(
+      {
+        name: 'func',
+        text: 'This function does something',
+      },
+      'fallbackValue'
+    );
+    assert.strictEqual(result, 'fallbackValue');
+  });
+
+  it('skips occurrences of the name within the receiver', () => {
+    const result = getFullName({
+      name: 'channel',
+      text: '`diagnostics_channel.channel(name)`',
+    });
+    assert.strictEqual(result, 'diagnostics_channel.channel');
+  });
+
+  it('skips occurrences of the name that are a prefix of the receiver', () => {
+    const result = getFullName({
+      name: 'read',
+      text: '`readable.read([size])`',
+    });
+    assert.strictEqual(result, 'readable.read');
+  });
+
+  it('ignores parameters repeating the name', () => {
+    const result = getFullName({
+      name: 'percentile',
+      text: '`histogram.percentile(percentile)`',
+    });
+    assert.strictEqual(result, 'histogram.percentile');
+  });
+
+  it('handles symbol-keyed methods', () => {
+    const result = getFullName({
+      name: "[Symbol.for('nodejs.rejection')]",
+      text: "`emitter[Symbol.for('nodejs.rejection')](err, eventName[, ...args])`",
+    });
+    assert.strictEqual(result, "emitter[Symbol.for('nodejs.rejection')]");
+  });
+
+  it('keeps quoted names intact', () => {
+    const result = getFullName({
+      name: 'console.log',
+      text: "Event: `'console.log'`",
+    });
+    assert.strictEqual(result, 'console.log');
+  });
+
+  it('does not strip "new" from within a name', () => {
+    const result = getFullName({
+      name: 'newListener',
+      text: "Event: `'newListener'`",
+    });
+    assert.strictEqual(result, 'newListener');
+  });
+
+  it('does not strip "new" from within a dotted name', () => {
+    const result = getFullName({
+      name: 'onnewtoken',
+      text: '`session.onnewtoken`',
+    });
+    assert.strictEqual(result, 'session.onnewtoken');
+  });
+
+  it('returns fallback when no occurrence terminates the name', () => {
+    const result = getFullName(
+      {
+        name: 'read',
+        text: '`readable`',
+      },
+      'fallback'
+    );
+    assert.strictEqual(result, 'fallback');
+  });
+});
diff --git a/packages/react/src/jsx-ast/utils/__tests__/signature.test.mjs b/packages/react/src/jsx-ast/utils/__tests__/signature.test.mjs
index 0ae5670ee..9469ebfc9 100644
--- a/packages/react/src/jsx-ast/utils/__tests__/signature.test.mjs
+++ b/packages/react/src/jsx-ast/utils/__tests__/signature.test.mjs
@@ -1,7 +1,9 @@
 import assert from 'node:assert/strict';
 import { describe, it } from 'node:test';
 
-import { generateSignature, getFullName } from '../signature.mjs';
+import { loadMarkdownPlugins } from '@doc-kit/core/utils/markdown/plugins.mjs';
+
+import { createSignatureCodeBlock, generateSignature } from '../signature.mjs';
 
 describe('generateSignature', () => {
   describe('function signatures', () => {
@@ -246,197 +248,32 @@ describe('generateSignature', () => {
   });
 });
 
-describe('getFullName', () => {
-  it('returns fallback when name equals text', () => {
-    const result = getFullName({ name: 'test', text: 'test' }, 'fallback');
-    assert.strictEqual(result, 'fallback');
-  });
-
-  it('returns name as fallback when name equals text and no fallback provided', () => {
-    const result = getFullName({ name: 'test', text: 'test' });
-    assert.strictEqual(result, 'test');
-  });
-
-  it('extracts inline code that includes the name', () => {
-    const result = getFullName({
-      name: 'myFunc',
-      text: 'This is `myFunc(param1, param2)` function',
-    });
-    assert.strictEqual(result, 'myFunc');
-  });
-
-  it('handles inline code with extra content after name', () => {
-    const result = getFullName({
-      name: 'authenticate',
-      text: 'The `authenticate(user, password)` method',
-    });
-    assert.strictEqual(result, 'authenticate');
-  });
-
-  it('strips quotes from the beginning', () => {
-    const result = getFullName({
-      name: 'func',
-      text: 'The `"func"()` method',
-    });
-    assert.strictEqual(result, 'func');
-  });
-
-  it('strips single quotes from the beginning', () => {
-    const result = getFullName({
-      name: 'func',
-      text: "The `'func'()` method",
-    });
-    assert.strictEqual(result, 'func');
-  });
-
-  it('strips "new" keyword from the beginning', () => {
-    const result = getFullName({
-      name: 'Constructor',
-      text: 'The `new Constructor()` call',
-    });
-    assert.strictEqual(result, 'Constructor');
-  });
-
-  it('strips "new " with space from the beginning', () => {
-    const result = getFullName({
-      name: 'MyClass',
-      text: 'The `new MyClass(param)` constructor',
-    });
-    assert.strictEqual(result, 'MyClass');
-  });
-
-  it('returns fallback when no inline code found', () => {
-    const result = getFullName(
-      {
-        name: 'func',
-        text: 'This is a function without code blocks',
-      },
-      'fallback'
-    );
-    assert.strictEqual(result, 'fallback');
-  });
-
-  it('returns fallback when inline code does not include name', () => {
-    const result = getFullName(
-      {
-        name: 'myFunc',
-        text: 'This is `otherFunc()` function',
-      },
-      'fallback'
-    );
-    assert.strictEqual(result, 'fallback');
-  });
-
-  it('handles empty inline code', () => {
-    const result = getFullName(
-      {
-        name: 'func',
-        text: 'This has `` empty code',
-      },
-      'fallback'
-    );
-    assert.strictEqual(result, 'fallback');
-  });
-
-  it('handles multiple inline code blocks, uses first match', () => {
-    const result = getFullName({
-      name: 'func',
-      text: 'This has `func()` and `other()` code',
-    });
-    assert.strictEqual(result, 'func');
-  });
-
-  it('handles complex inline code with parameters', () => {
-    const result = getFullName({
-      name: 'processData',
-      text: 'The `processData(input, options = {})` method processes data',
-    });
-    assert.strictEqual(result, 'processData');
-  });
-
-  it('strips both quotes and new keyword', () => {
-    const result = getFullName({
-      name: 'MyClass',
-      text: '`"new MyClass"()`',
-    });
-    assert.strictEqual(result, 'MyClass');
-  });
-
-  it('handles text with no backticks', () => {
-    const result = getFullName(
-      {
-        name: 'func',
-        text: 'This function does something',
+describe('createSignatureCodeBlock', () => {
+  it('highlights the signature, its code embedded as markup', async () => {
+    // Signatures are highlighted by the Shiki plugin of `jsx-ast`
+    await loadMarkdownPlugins({
+      name: 'jsx-ast',
+      markdown: {
+        rehypePlugins: [
+          import.meta.resolve('@doc-kit/core/plugins/shiki/rehype.mjs'),
+        ],
       },
-      'fallbackValue'
-    );
-    assert.strictEqual(result, 'fallbackValue');
-  });
-
-  it('skips occurrences of the name within the receiver', () => {
-    const result = getFullName({
-      name: 'channel',
-      text: '`diagnostics_channel.channel(name)`',
     });
-    assert.strictEqual(result, 'diagnostics_channel.channel');
-  });
-
-  it('skips occurrences of the name that are a prefix of the receiver', () => {
-    const result = getFullName({
-      name: 'read',
-      text: '`readable.read([size])`',
-    });
-    assert.strictEqual(result, 'readable.read');
-  });
-
-  it('ignores parameters repeating the name', () => {
-    const result = getFullName({
-      name: 'percentile',
-      text: '`histogram.percentile(percentile)`',
-    });
-    assert.strictEqual(result, 'histogram.percentile');
-  });
-
-  it('handles symbol-keyed methods', () => {
-    const result = getFullName({
-      name: "[Symbol.for('nodejs.rejection')]",
-      text: "`emitter[Symbol.for('nodejs.rejection')](err, eventName[, ...args])`",
-    });
-    assert.strictEqual(result, "emitter[Symbol.for('nodejs.rejection')]");
-  });
 
-  it('keeps quoted names intact', () => {
-    const result = getFullName({
-      name: 'console.log',
-      text: "Event: `'console.log'`",
+    const block = createSignatureCodeBlock('fn', {
+      params: [{ name: 'a' }],
+      return: { type: 'string' },
     });
-    assert.strictEqual(result, 'console.log');
-  });
 
-  it('does not strip "new" from within a name', () => {
-    const result = getFullName({
-      name: 'newListener',
-      text: "Event: `'newListener'`",
-    });
-    assert.strictEqual(result, 'newListener');
-  });
+    assert.deepStrictEqual(block.properties, { className: ['signature'] });
 
-  it('does not strip "new" from within a dotted name', () => {
-    const result = getFullName({
-      name: 'onnewtoken',
-      text: '`session.onnewtoken`',
-    });
-    assert.strictEqual(result, 'session.onnewtoken');
-  });
+    const [pre] = block.children;
+    const [code] = pre.children;
 
-  it('returns fallback when no occurrence terminates the name', () => {
-    const result = getFullName(
-      {
-        name: 'read',
-        text: '`readable`',
-      },
-      'fallback'
+    assert.match(pre.properties.class, /^shiki /);
+    assert.equal(code.type, 'mdxJsxFlowElement');
+    assert.ok(
+      code.attributes.some(({ name }) => name === 'dangerouslySetInnerHTML')
     );
-    assert.strictEqual(result, 'fallback');
   });
 });
diff --git a/packages/react/src/jsx-ast/utils/buildBarProps.mjs b/packages/react/src/jsx-ast/utils/buildBarProps.mjs
index 5bb3823e8..bd2362c72 100644
--- a/packages/react/src/jsx-ast/utils/buildBarProps.mjs
+++ b/packages/react/src/jsx-ast/utils/buildBarProps.mjs
@@ -3,7 +3,7 @@
 import { visit } from 'unist-util-visit';
 
 import { TOC_MAX_HEADING_DEPTH } from '../constants.mjs';
-import { getFullName } from './signature.mjs';
+import { getFullName } from './getFullName.mjs';
 
 // Callable heading types whose ToC label should be the bare function name
 // rather than the full signature.
diff --git a/packages/react/src/jsx-ast/utils/buildContent.mjs b/packages/react/src/jsx-ast/utils/buildContent.mjs
index 82688fa26..a01ed1058 100644
--- a/packages/react/src/jsx-ast/utils/buildContent.mjs
+++ b/packages/react/src/jsx-ast/utils/buildContent.mjs
@@ -29,11 +29,11 @@ import {
 } from '../constants.mjs';
 import { createJSXElement } from './ast.mjs';
 import { extractHeadings, extractTextContent } from './buildBarProps.mjs';
+import { getFullName } from './getFullName.mjs';
 import { renderAsJSX } from './render.mjs';
 import {
   insertSignatureCodeBlock,
   createSignatureTable,
-  getFullName,
 } from './signature.mjs';
 
 /**
diff --git a/packages/react/src/jsx-ast/utils/getFullName.mjs b/packages/react/src/jsx-ast/utils/getFullName.mjs
new file mode 100644
index 000000000..5b59295f0
--- /dev/null
+++ b/packages/react/src/jsx-ast/utils/getFullName.mjs
@@ -0,0 +1,52 @@
+'use strict';
+
+/**
+ * Infers the "real" function name from a heading node.
+ * Useful when auto-generated headings differ from code tokens.
+ *
+ * @param {import('@doc-kit/core/generators/metadata/types').HeadingData} heading - Metadata with name and text fields.
+ * @param {any} fallback - Fallback value if inference fails.
+ */
+export const getFullName = ({ name, text }, fallback = name) => {
+  // If the name and text are identical, just use fallback
+  if (name === text) {
+    return fallback;
+  }
+
+  // Attempt to extract inline code from heading text
+  const code = text.trim().match(/`([^`]+)`/)?.[1];
+
+  if (!code?.includes(name)) {
+    return fallback;
+  }
+
+  // Find the occurrence of `name` that denotes the documented entry: the one
+  // immediately followed by its parameter list, a closing quote, or the end
+  // of the code. Earlier occurrences are mere substrings of the receiver
+  // (e.g. `channel` within `diagnostics_channel.channel`, `read` within
+  // `readable.read`), and later ones can be parameters repeating the name.
+  let end = -1;
+  let index = code.indexOf(name);
+
+  while (index !== -1) {
+    const next = code[index + name.length];
+
+    if (next === undefined || next === '(' || next === "'" || next === '"') {
+      end = index + name.length;
+      break;
+    }
+
+    index = code.indexOf(name, index + 1);
+  }
+
+  // If inline code includes the name, return a clean version of it
+  return end === -1
+    ? fallback
+    : code
+        .slice(0, end) // Truncate everything after the name.
+        // Strip a leading quote and/or the "new" keyword. The latter requires
+        // following whitespace so names containing "new" (e.g. `newListener`)
+        // stay intact.
+        .replace(/^["']/, '')
+        .replace(/^new\s+/, '');
+};
diff --git a/packages/react/src/jsx-ast/utils/signature.mjs b/packages/react/src/jsx-ast/utils/signature.mjs
index 3edf90156..eb9d51d60 100644
--- a/packages/react/src/jsx-ast/utils/signature.mjs
+++ b/packages/react/src/jsx-ast/utils/signature.mjs
@@ -5,7 +5,9 @@ import parseSignature from '@doc-kit/core/utils/signature/parseSignature.mjs';
 import { h as createElement } from 'hastscript';
 
 import { JSX_IMPORTS } from '../../html/constants.mjs';
+import { embedHighlightedBlocks } from '../plugins/static-markup.mjs';
 import { createJSXElement } from './ast.mjs';
+import { getFullName } from './getFullName.mjs';
 import { parseListIntoProperties } from './types.mjs';
 
 /**
@@ -68,58 +70,9 @@ export const createSignatureCodeBlock = (functionName, signature, heading) => {
   const highlighter = getHighlighter('jsx-ast');
   const highlighted = highlighter.highlightToHast(sig, 'typescript');
 
-  return createElement('div', { class: 'signature' }, [highlighted]);
-};
-
-/**
- * Infers the "real" function name from a heading node.
- * Useful when auto-generated headings differ from code tokens.
- *
- * @param {import('@doc-kit/core/generators/metadata/types').HeadingData} heading - Metadata with name and text fields.
- * @param {any} fallback - Fallback value if inference fails.
- */
-export const getFullName = ({ name, text }, fallback = name) => {
-  // If the name and text are identical, just use fallback
-  if (name === text) {
-    return fallback;
-  }
-
-  // Attempt to extract inline code from heading text
-  const code = text.trim().match(/`([^`]+)`/)?.[1];
-
-  if (!code?.includes(name)) {
-    return fallback;
-  }
-
-  // Find the occurrence of `name` that denotes the documented entry: the one
-  // immediately followed by its parameter list, a closing quote, or the end
-  // of the code. Earlier occurrences are mere substrings of the receiver
-  // (e.g. `channel` within `diagnostics_channel.channel`, `read` within
-  // `readable.read`), and later ones can be parameters repeating the name.
-  let end = -1;
-  let index = code.indexOf(name);
-
-  while (index !== -1) {
-    const next = code[index + name.length];
-
-    if (next === undefined || next === '(' || next === "'" || next === '"') {
-      end = index + name.length;
-      break;
-    }
-
-    index = code.indexOf(name, index + 1);
-  }
-
-  // If inline code includes the name, return a clean version of it
-  return end === -1
-    ? fallback
-    : code
-        .slice(0, end) // Truncate everything after the name.
-        // Strip a leading quote and/or the "new" keyword. The latter requires
-        // following whitespace so names containing "new" (e.g. `newListener`)
-        // stay intact.
-        .replace(/^["']/, '')
-        .replace(/^new\s+/, '');
+  return createElement('div', { class: 'signature' }, [
+    embedHighlightedBlocks(highlighted),
+  ]);
 };
 
 /**
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index e4274c3d6..42e14b572 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -95,6 +95,9 @@ importers:
       acorn:
         specifier: ^8.17.0
         version: 8.18.0
+      birpc:
+        specifier: ^4.2.0
+        version: 4.2.0
       cosmiconfig:
         specifier: ^9.0.2
         version: 9.0.2(typescript@5.9.3)
@@ -259,6 +262,9 @@ importers:
       github-slugger:
         specifier: ^2.0.0
         version: 2.0.0
+      hast-util-to-jsx-runtime:
+        specifier: ^2.3.6
+        version: 2.3.6(supports-color@10.2.2)
       hast-util-to-string:
         specifier: ^3.0.1
         version: 3.0.1
@@ -2115,6 +2121,9 @@ packages:
     resolution: {integrity: sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==}
     engines: {node: 18 || 20 || >=22}
 
+  birpc@4.2.0:
+    resolution: {integrity: sha512-KxgKcZPfrtzJDDALHPguGpGJUrzdgpymyiQQgzFjWreHMOpWrnFNVREr5J48x2DBh8ZVioscrV1SBkDipGiX+Q==}
+
   blake3-wasm@2.1.5:
     resolution: {integrity: sha512-F1+K8EbfOZE49dtoPtmxUQrpXaBIl3ICvasLh+nJta0xkz+9kF/7uet9fLnwKqhDrmj6g+6K3Tw9yQPUg2ka5g==}
 
@@ -5115,6 +5124,8 @@ snapshots:
 
   balanced-match@4.0.4: {}
 
+  birpc@4.2.0: {}
+
   blake3-wasm@2.1.5: {}
 
   boolbase@1.0.0: {}