perf: reduce build memory and time

.changeset/build-memory.md

@@ -0,0 +1,8 @@
+---
+'@doc-kit/cli': minor
+'@doc-kit/core': minor
+'@doc-kit/generator-react': minor
+'@node-core/doc-kit-legacy': patch
+---
+
+Build with 30–50% less memory, in less than half the time: Shiki registers each language once code in it is highlighted, highlighted code reaches pages as static markup rather than a tree per token, the default Vite bundler runs in a child process (`createChildProcess`), `all.html` is rendered by the worker pool and no longer minified, idle workers end after 500ms, and each worker's heap is limited to 512MB by default (the new `workerHeapSize` option, or `--worker-heap-size`)

docs/cli.md

@@ -39,4 +39,6 @@ Runs the generators and writes their output. Requires a `target` and an
 - `--type-map <url>` {string} Type map URL or path (custom type-name → URL
   links).
 - `-p, --threads <n>` {number} Worker threads to use (minimum 1).
+- `--worker-heap-size <mb>` {number} Heap size limit of each worker thread, in
+  MB (minimum 1).
 - `--chunk-size <n>` {number} Items per worker thread (minimum 1).

docs/configuration.md

@@ -135,8 +135,23 @@ Top-level, alongside `target` and `global`:
 
 - `threads` {number} Worker threads used for generation. Defaults to your
   CPU count.
+- `workerHeapSize` {number} Heap size limit of each worker thread (V8's old
+  space), in MB. Defaults to V8's own limit, at most `512`: V8 lets a heap grow
+  to several times its live data before collecting it, the more the higher its
+  limit. A worker running out of memory fails the build; raise it then. When
+  Node's `--max-old-space-size` is set (`NODE_OPTIONS` takes it too), it limits
+  the workers instead, and the main thread, which runs the generators with
+  `threads: 1`.
 - `chunkSize` {number} Items processed per worker thread. **Default:** `10`.
 
+> [!NOTE]
+> On Node.js 26, a worker thread that ends leaves its memory for the threads
+> started after it to reuse, rather than giving it back. Building the Node.js
+> docs, that's ~800MB held while the site is bundled. Passing
+> `--no-memory-pool-share-memory-on-teardown` to the `node` running doc-kit
+> gives it back. Node.js 24 gives it back by itself after a few seconds, and
+> doesn't have the flag.
+
 ## Generator options
 
 Each generator documents its own options on its reference page — see the
@@ -196,6 +211,7 @@ own.
 - `rehypePlugins` {Array} Run on the HTML of the generators rendering
   Markdown, such as `jsx-ast`, before code is highlighted.
 - `recmaPlugins` {Array} Run on the JavaScript `jsx-ast` compiles the pages to.
+  Highlighted code is in it as the markup it renders to, rather than as JSX.
 
 A generator only takes the plugins its pipeline has a place for: `jsx-ast`
 takes all three kinds, `ast`, `metadata`, and `json` take remark plugins, and
@@ -261,17 +277,18 @@ precedence):
 
 CLI options map to configuration properties:
 
-| CLI Option             | Config Property    | Example                   |
-| ---------------------- | ------------------ | ------------------------- |
-| `--input <path>`       | `global.input`     | `--input src/`            |
-| `--output <path>`      | `global.output`    | `--output dist/`          |
-| `--ignore <pattern>`   | `global.ignore[]`  | `--ignore test/`          |
-| `--minify`             | `global.minify`    | `--minify`                |
-| `--git-ref <ref>`      | `global.ref`       | `--git-ref v20.0.0`       |
-| `--version <version>`  | `global.version`   | `--version 20.0.0`        |
-| `--changelog <url>`    | `global.changelog` | `--changelog https://...` |
-| `--index <url>`        | `global.index`     | `--index file://...`      |
-| `--type-map <map>`     | `metadata.typeMap` | `--type-map file://...`   |
-| `--target <generator>` | `target`           | `--target json`           |
-| `--threads <n>`        | `threads`          | `--threads 4`             |
-| `--chunk-size <n>`     | `chunkSize`        | `--chunk-size 10`         |
+| CLI Option                | Config Property    | Example                   |
+| ------------------------- | ------------------ | ------------------------- |
+| `--input <path>`          | `global.input`     | `--input src/`            |
+| `--output <path>`         | `global.output`    | `--output dist/`          |
+| `--ignore <pattern>`      | `global.ignore[]`  | `--ignore test/`          |
+| `--minify`                | `global.minify`    | `--minify`                |
+| `--git-ref <ref>`         | `global.ref`       | `--git-ref v20.0.0`       |
+| `--version <version>`     | `global.version`   | `--version 20.0.0`        |
+| `--changelog <url>`       | `global.changelog` | `--changelog https://...` |
+| `--index <url>`           | `global.index`     | `--index file://...`      |
+| `--type-map <map>`        | `metadata.typeMap` | `--type-map file://...`   |
+| `--target <generator>`    | `target`           | `--target json`           |
+| `--threads <n>`           | `threads`          | `--threads 4`             |
+| `--worker-heap-size <mb>` | `workerHeapSize`   | `--worker-heap-size 1024` |
+| `--chunk-size <n>`        | `chunkSize`        | `--chunk-size 10`         |

packages/cli/bin/commands/generate.mjs

@@ -18,6 +18,7 @@ const { runGenerators } = createGenerator();
  * @property {string[]} ignore
  * @property {string} output
  * @property {number} threads
+ * @property {number} workerHeapSize
  * @property {number} chunkSize
  * @property {string} version
  * @property {string} changelog
@@ -53,6 +54,12 @@ export default new Command('generate')
       'Number of threads to use (minimum: 1)'
     )
   )
+  .addOption(
+    new Option(
+      '--worker-heap-size <mb>',
+      'Heap size limit of each worker thread, in MB (minimum: 1)'
+    )
+  )
   .addOption(
     new Option(
       '--chunk-size <number>',

packages/core/package.json

@@ -57,6 +57,7 @@
     "@swc/html-wasm": "^1.16.2",
     "@swc/wasm": "^1.15.46",
     "acorn": "^8.17.0",
+    "birpc": "^4.2.0",
     "cosmiconfig": "^9.0.2",
     "dedent": "^1.7.2",
     "github-slugger": "^2.0.0",

packages/core/src/generators.mjs

@@ -107,7 +107,7 @@ const createGenerator = () => {
    * @returns {Promise<unknown[]>} Results of all requested generators
    */
   const runGenerators = async configuration => {
-    const { target, threads } = configuration;
+    const { target, threads, workerHeapSize } = configuration;
 
     // Resolve shorthand names and load the full dependency closure up front,
     // so scheduling below is fully synchronous.
@@ -132,7 +132,7 @@ const createGenerator = () => {
     cache.populateConsumerCounts(targets, specifier => inputOf.get(specifier));
 
     // Create worker pool
-    pool = createWorkerPool(threads);
+    pool = createWorkerPool(threads, workerHeapSize);
 
     // Schedule all generators
     for (const specifier of targets) {

packages/core/src/generators/metadata/generate.mjs

@@ -3,15 +3,17 @@
 import getConfig from '#utils/configuration/index.mjs';
 import { loadFromURL } from '#utils/loaders.mjs';
 
-import { parseApiDoc } from './utils/parse.mjs';
-
 /**
  * Process a chunk of API doc files in a worker thread.
  * Called by chunk-worker.mjs for parallel processing.
  *
  * @type {import('./types').Generator['processChunk']}
  */
 export async function processChunk(fullInput, itemIndices, typeMap) {
+  // Loaded on first use rather than with the generator, so the main thread,
+  // which never parses, does not load the TypeScript parser (~40MB of WASM)
+  const { parseApiDoc } = await import('./utils/parse.mjs');
+
   const results = [];
 
   for (const idx of itemIndices) {

packages/core/src/plugins/shiki/__tests__/highlighter.test.mjs

@@ -86,6 +86,57 @@ describe('createHighlighter', () => {
     );
   });
 
+  it('registers a bundled language once code in it is highlighted', async () => {
+    const highlighter = await createHighlighter({
+      langAlias: { py: 'python' },
+    });
+    const loaded = () => highlighter.shiki.getLoadedLanguages();
+
+    assert.ok(!loaded().includes('python'));
+    assert.ok(!loaded().includes('javascript'));
+
+    // By its name, an alias of its own, or one of the options
+    assert.equal(highlighter.resolveLanguage('py'), 'py');
+    assert.equal(highlighter.resolveLanguage('mjs'), 'mjs');
+
+    assert.ok(loaded().includes('python'));
+    assert.ok(loaded().includes('javascript'));
+    assert.ok(loaded().includes('cjs'));
+
+    assert.equal(highlighter.resolveLanguage(undefined), 'text');
+    assert.equal(highlighter.resolveLanguage('plaintext'), 'plaintext');
+  });
+
+  it('lists the bundled languages without registering them', async () => {
+    const highlighter = await createHighlighter({ langs: [grammar] });
+
+    assert.deepStrictEqual(
+      highlighter.langs.find(({ name }) => name === 'rust'),
+      { name: 'rust', displayName: 'Rust', aliases: ['rs'] }
+    );
+    assert.equal(highlighter.langs.at(-1), grammar);
+    assert.ok(!highlighter.shiki.getLoadedLanguages().includes('rust'));
+  });
+
+  it('registers the bundled languages a language embeds', async () => {
+    const highlighter = await createHighlighter({
+      langs: [
+        {
+          ...grammar,
+          name: 'oxcscript',
+          scopeName: 'source.oxcscript',
+          embeddedLangs: ['javascript'],
+          patterns: [{ include: 'source.js' }],
+        },
+      ],
+    });
+
+    assert.match(
+      highlighter.highlightToHtml('const on = 1', 'oxcscript'),
+      /--shiki-dark/
+    );
+  });
+
   it('gives the same highlighter for the same options', async () => {
     const highlighter = await createHighlighter({ langs: [grammar] });