From 108ddaf1607c8d83165e8ff0b5cc6fccc5646dd5 Mon Sep 17 00:00:00 2001 From: Claudio Wunder Date: Thu, 8 Oct 2026 12:57:06 +0200 Subject: [PATCH 01/10] refactor(core): move the type annotations plugin to `src/plugins` It is a remark plugin, with its mdast and hast handlers, rather than a utility. `#plugins/*` imports it. Assisted-by: Claude Opus 5.5 --- packages/core/package.json | 1 + .../type-annotations/__tests__/hast.test.mjs | 2 +- .../type-annotations/__tests__/remark.test.mjs | 0 .../core/src/{utils => plugins}/type-annotations/hast.mjs | 0 .../type-annotations/highlighter.mjs} | 0 .../core/src/{utils => plugins}/type-annotations/mdast.mjs | 0 .../core/src/{utils => plugins}/type-annotations/remark.mjs | 0 .../core/src/{utils => plugins}/type-annotations/syntax.mjs | 0 packages/core/src/utils/remark.mjs | 5 +++-- packages/react/src/jsx-ast/utils/remark.mjs | 2 +- 10 files changed, 6 insertions(+), 4 deletions(-) rename packages/core/src/{utils => plugins}/type-annotations/__tests__/hast.test.mjs (98%) rename packages/core/src/{utils => plugins}/type-annotations/__tests__/remark.test.mjs (100%) rename packages/core/src/{utils => plugins}/type-annotations/hast.mjs (100%) rename packages/core/src/{utils/type-annotations/highlighted.mjs => plugins/type-annotations/highlighter.mjs} (100%) rename packages/core/src/{utils => plugins}/type-annotations/mdast.mjs (100%) rename packages/core/src/{utils => plugins}/type-annotations/remark.mjs (100%) rename packages/core/src/{utils => plugins}/type-annotations/syntax.mjs (100%) diff --git a/packages/core/package.json b/packages/core/package.json index d1f8f3460..4bbf018ae 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -39,6 +39,7 @@ "./src/parsers/*", "./src/parsers/*.d.ts" ], + "#plugins/*": "./src/plugins/*", "#utils/*": "./src/utils/*" }, "files": [ diff --git a/packages/core/src/utils/type-annotations/__tests__/hast.test.mjs b/packages/core/src/plugins/type-annotations/__tests__/hast.test.mjs similarity index 98% rename from packages/core/src/utils/type-annotations/__tests__/hast.test.mjs rename to packages/core/src/plugins/type-annotations/__tests__/hast.test.mjs index 34124ab9d..f2ce3971e 100644 --- a/packages/core/src/utils/type-annotations/__tests__/hast.test.mjs +++ b/packages/core/src/plugins/type-annotations/__tests__/hast.test.mjs @@ -5,7 +5,7 @@ import { toHtml } from 'hast-util-to-html'; import { toString } from 'hast-util-to-string'; import { typeAnnotationToHast } from '../hast.mjs'; -import { typeAnnotationToHighlightedHast } from '../highlighted.mjs'; +import { typeAnnotationToHighlightedHast } from '../highlighter.mjs'; // A minimal mdast-util-to-hast state — the handlers only use patch/applyData const state = { patch: () => {}, applyData: (_, result) => result }; diff --git a/packages/core/src/utils/type-annotations/__tests__/remark.test.mjs b/packages/core/src/plugins/type-annotations/__tests__/remark.test.mjs similarity index 100% rename from packages/core/src/utils/type-annotations/__tests__/remark.test.mjs rename to packages/core/src/plugins/type-annotations/__tests__/remark.test.mjs diff --git a/packages/core/src/utils/type-annotations/hast.mjs b/packages/core/src/plugins/type-annotations/hast.mjs similarity index 100% rename from packages/core/src/utils/type-annotations/hast.mjs rename to packages/core/src/plugins/type-annotations/hast.mjs diff --git a/packages/core/src/utils/type-annotations/highlighted.mjs b/packages/core/src/plugins/type-annotations/highlighter.mjs similarity index 100% rename from packages/core/src/utils/type-annotations/highlighted.mjs rename to packages/core/src/plugins/type-annotations/highlighter.mjs diff --git a/packages/core/src/utils/type-annotations/mdast.mjs b/packages/core/src/plugins/type-annotations/mdast.mjs similarity index 100% rename from packages/core/src/utils/type-annotations/mdast.mjs rename to packages/core/src/plugins/type-annotations/mdast.mjs diff --git a/packages/core/src/utils/type-annotations/remark.mjs b/packages/core/src/plugins/type-annotations/remark.mjs similarity index 100% rename from packages/core/src/utils/type-annotations/remark.mjs rename to packages/core/src/plugins/type-annotations/remark.mjs diff --git a/packages/core/src/utils/type-annotations/syntax.mjs b/packages/core/src/plugins/type-annotations/syntax.mjs similarity index 100% rename from packages/core/src/utils/type-annotations/syntax.mjs rename to packages/core/src/plugins/type-annotations/syntax.mjs diff --git a/packages/core/src/utils/remark.mjs b/packages/core/src/utils/remark.mjs index 022e55b20..afbbe7f5e 100644 --- a/packages/core/src/utils/remark.mjs +++ b/packages/core/src/utils/remark.mjs @@ -8,9 +8,10 @@ import remarkRehype from 'remark-rehype'; import remarkStringify from 'remark-stringify'; import { unified } from 'unified'; +import { typeAnnotationToHast } from '#plugins/type-annotations/hast.mjs'; +import remarkTypeAnnotations from '#plugins/type-annotations/remark.mjs'; + import { lazy } from './misc.mjs'; -import { typeAnnotationToHast } from './type-annotations/hast.mjs'; -import remarkTypeAnnotations from './type-annotations/remark.mjs'; // Nothing in this module loads Shiki: the `ast` and `metadata` stages (and // every worker that runs them) import it, and none of them highlight code. diff --git a/packages/react/src/jsx-ast/utils/remark.mjs b/packages/react/src/jsx-ast/utils/remark.mjs index 59bafd01c..7cc9fb4a3 100644 --- a/packages/react/src/jsx-ast/utils/remark.mjs +++ b/packages/react/src/jsx-ast/utils/remark.mjs @@ -1,8 +1,8 @@ 'use strict'; +import { typeAnnotationToHighlightedHast } from '@doc-kit/core/plugins/type-annotations/highlighter.mjs'; import { highlighter } from '@doc-kit/core/utils/highlighter.mjs'; import { lazy } from '@doc-kit/core/utils/misc.mjs'; -import { typeAnnotationToHighlightedHast } from '@doc-kit/core/utils/type-annotations/highlighted.mjs'; import rehypeShikiji from '@node-core/rehype-shiki/plugin'; import recmaJsx from 'recma-jsx'; import recmaStringify from 'recma-stringify'; From 83a5f223ea836bf6295439fe3ba3ce1916570f87 Mon Sep 17 00:00:00 2001 From: Claudio Wunder Date: Thu, 8 Oct 2026 12:59:09 +0200 Subject: [PATCH 02/10] refactor(react): move the `jsx-ast` plugins out of `utils` They are the remark and rehype plugins of `jsx-ast`, rather than utilities. Assisted-by: Claude Opus 5.5 --- .../jsx-ast/{utils => }/plugins/__tests__/alerts.test.mjs | 0 .../{utils => }/plugins/__tests__/transformer.test.mjs | 0 packages/react/src/jsx-ast/{utils => }/plugins/alerts.mjs | 6 +++--- .../react/src/jsx-ast/{utils => }/plugins/transformer.mjs | 2 +- packages/react/src/jsx-ast/utils/remark.mjs | 4 ++-- 5 files changed, 6 insertions(+), 6 deletions(-) rename packages/react/src/jsx-ast/{utils => }/plugins/__tests__/alerts.test.mjs (100%) rename packages/react/src/jsx-ast/{utils => }/plugins/__tests__/transformer.test.mjs (100%) rename packages/react/src/jsx-ast/{utils => }/plugins/alerts.mjs (91%) rename packages/react/src/jsx-ast/{utils => }/plugins/transformer.mjs (97%) diff --git a/packages/react/src/jsx-ast/utils/plugins/__tests__/alerts.test.mjs b/packages/react/src/jsx-ast/plugins/__tests__/alerts.test.mjs similarity index 100% rename from packages/react/src/jsx-ast/utils/plugins/__tests__/alerts.test.mjs rename to packages/react/src/jsx-ast/plugins/__tests__/alerts.test.mjs diff --git a/packages/react/src/jsx-ast/utils/plugins/__tests__/transformer.test.mjs b/packages/react/src/jsx-ast/plugins/__tests__/transformer.test.mjs similarity index 100% rename from packages/react/src/jsx-ast/utils/plugins/__tests__/transformer.test.mjs rename to packages/react/src/jsx-ast/plugins/__tests__/transformer.test.mjs diff --git a/packages/react/src/jsx-ast/utils/plugins/alerts.mjs b/packages/react/src/jsx-ast/plugins/alerts.mjs similarity index 91% rename from packages/react/src/jsx-ast/utils/plugins/alerts.mjs rename to packages/react/src/jsx-ast/plugins/alerts.mjs index 4ede03e56..3823ad46c 100644 --- a/packages/react/src/jsx-ast/utils/plugins/alerts.mjs +++ b/packages/react/src/jsx-ast/plugins/alerts.mjs @@ -2,9 +2,9 @@ import { SKIP, visit } from 'unist-util-visit'; -import { JSX_IMPORTS } from '../../../html/constants.mjs'; -import { ALERT_MARKER, GITHUB_ALERT_TYPES } from '../../constants.mjs'; -import { createJSXElement } from '../ast.mjs'; +import { JSX_IMPORTS } from '../../html/constants.mjs'; +import { ALERT_MARKER, GITHUB_ALERT_TYPES } from '../constants.mjs'; +import { createJSXElement } from '../utils/ast.mjs'; /** * Converts a marker keyword into a human-readable title (e.g. `NOTE` -> `Note`). diff --git a/packages/react/src/jsx-ast/utils/plugins/transformer.mjs b/packages/react/src/jsx-ast/plugins/transformer.mjs similarity index 97% rename from packages/react/src/jsx-ast/utils/plugins/transformer.mjs rename to packages/react/src/jsx-ast/plugins/transformer.mjs index 819b16307..cbb768357 100644 --- a/packages/react/src/jsx-ast/utils/plugins/transformer.mjs +++ b/packages/react/src/jsx-ast/plugins/transformer.mjs @@ -1,7 +1,7 @@ import { toString } from 'hast-util-to-string'; import { visit } from 'unist-util-visit'; -import { TAG_TRANSFORMS } from '../../constants.mjs'; +import { TAG_TRANSFORMS } from '../constants.mjs'; /** * Checks whether a HAST node is the generated GFM footnotes section. diff --git a/packages/react/src/jsx-ast/utils/remark.mjs b/packages/react/src/jsx-ast/utils/remark.mjs index 7cc9fb4a3..6e6db0b5d 100644 --- a/packages/react/src/jsx-ast/utils/remark.mjs +++ b/packages/react/src/jsx-ast/utils/remark.mjs @@ -14,8 +14,8 @@ import { unified } from 'unified'; import { visit } from 'unist-util-visit'; import { AST_NODE_TYPES } from '../constants.mjs'; -import transformAlerts from './plugins/alerts.mjs'; -import transformElements from './plugins/transformer.mjs'; +import transformAlerts from '../plugins/alerts.mjs'; +import transformElements from '../plugins/transformer.mjs'; const passThrough = ['element', ...Object.values(AST_NODE_TYPES.MDX)]; const codeMetaProperty = 'codeMeta'; From 8f3a30a66e37591e6deea599fcafeac63b8db9e9 Mon Sep 17 00:00:00 2001 From: Claudio Wunder Date: Thu, 8 Oct 2026 12:59:37 +0200 Subject: [PATCH 03/10] feat(core): add Markdown pipelines and the `markdown` option Generators declare the unified plugins they process Markdown with as `markdown`, taking the configured plugins in place of `'...'`. The `markdown` option adds plugins globally and per generator, resolved from the file declaring them so that worker threads can import them. Each thread loads the pipeline of the generators it runs, and `getProcessor(name)` gives their processor. Assisted-by: Claude Opus 5.5 --- .../core/src/__tests__/generators.test.mjs | 30 +++ packages/core/src/generators.mjs | 4 + packages/core/src/generators/loader.mjs | 14 ++ packages/core/src/generators/types.d.ts | 8 + .../src/plugins/type-annotations/remark.mjs | 8 +- .../fixtures/markdown-plugins-reporter.mjs | 24 ++ .../__tests__/fixtures/rehype-plugin.mjs | 4 + .../src/threading/__tests__/index.test.mjs | 36 ++- packages/core/src/threading/chunk-worker.mjs | 4 + .../configuration/__tests__/index.test.mjs | 115 ++++++++- .../core/src/utils/configuration/index.mjs | 109 +++++++-- .../core/src/utils/configuration/types.d.ts | 26 +++ packages/core/src/utils/loaders.mjs | 24 +- .../utils/markdown/__tests__/plugins.test.mjs | 197 ++++++++++++++++ .../markdown/__tests__/processor.test.mjs | 105 +++++++++ .../core/src/utils/markdown/constants.mjs | 7 + packages/core/src/utils/markdown/plugins.mjs | 219 ++++++++++++++++++ .../core/src/utils/markdown/processor.mjs | 60 +++++ 18 files changed, 969 insertions(+), 25 deletions(-) create mode 100644 packages/core/src/threading/__tests__/fixtures/markdown-plugins-reporter.mjs create mode 100644 packages/core/src/threading/__tests__/fixtures/rehype-plugin.mjs create mode 100644 packages/core/src/utils/markdown/__tests__/plugins.test.mjs create mode 100644 packages/core/src/utils/markdown/__tests__/processor.test.mjs create mode 100644 packages/core/src/utils/markdown/constants.mjs create mode 100644 packages/core/src/utils/markdown/plugins.mjs create mode 100644 packages/core/src/utils/markdown/processor.mjs diff --git a/packages/core/src/__tests__/generators.test.mjs b/packages/core/src/__tests__/generators.test.mjs index 85fc1e039..d3be80da3 100644 --- a/packages/core/src/__tests__/generators.test.mjs +++ b/packages/core/src/__tests__/generators.test.mjs @@ -150,6 +150,17 @@ mock.module('../threading/parallel.mjs', { }, }); +// The Markdown pipelines loaded, with the generators that had run by then +const loadedPipelines = []; + +mock.module('../utils/markdown/plugins.mjs', { + exports: { + loadMarkdownPlugins: async ({ name }, markdown) => { + loadedPipelines.push({ name, markdown, ran: Object.keys(runs) }); + }, + }, +}); + const createGenerator = (await import('../generators.mjs')).default; describe('createGenerator orchestration', () => { @@ -215,4 +226,23 @@ describe('createGenerator orchestration', () => { { all: { d: [{ meta: 1 }, { spliced: true }] } }, ]); }); + + it('loads the Markdown pipeline of each generator as it starts', async () => { + const { runGenerators } = createGenerator(); + const markdown = { remarkPlugins: ['file:///plugin.mjs'] }; + + loadedPipelines.length = 0; + + await runGenerators({ + target: ['gen-a'], + threads: 1, + 'gen-a': { markdown }, + }); + + assert.deepStrictEqual(loadedPipelines, [ + { name: 'ast', markdown: undefined, ran: [] }, + { name: 'metadata', markdown: undefined, ran: ['ast'] }, + { name: 'gen-a', markdown, ran: ['ast', 'metadata'] }, + ]); + }); }); diff --git a/packages/core/src/generators.mjs b/packages/core/src/generators.mjs index 79a61235e..b73db0967 100644 --- a/packages/core/src/generators.mjs +++ b/packages/core/src/generators.mjs @@ -9,6 +9,7 @@ import { resolvePipeline } from './generators/pipeline.mjs'; import logger from './logger/index.mjs'; import createWorkerPool from './threading/index.mjs'; import createParallelWorker from './threading/parallel.mjs'; +import { loadMarkdownPlugins } from './utils/markdown/plugins.mjs'; import { isAsyncIterable } from './utils/misc.mjs'; const generatorsLogger = logger.child('generators'); @@ -78,6 +79,9 @@ const createGenerator = () => { generatorsLogger.debug(`Starting "${name}"`); + // Load its Markdown pipeline, for what it processes on this thread + await loadMarkdownPlugins(generator, configuration[name]?.markdown); + // Create parallel worker for streaming generators const worker = hasParallelProcessor ? createParallelWorker(specifier, generator, pool, configuration) diff --git a/packages/core/src/generators/loader.mjs b/packages/core/src/generators/loader.mjs index 49ba85b53..f2c0279f0 100644 --- a/packages/core/src/generators/loader.mjs +++ b/packages/core/src/generators/loader.mjs @@ -7,6 +7,9 @@ import { enforceArray } from '#utils/array.mjs'; import { allGenerators } from './index.mjs'; +// The module each loaded generator comes from +const generatorModules = new WeakMap(); + /** * Resolves a CLI/configuration target into an import specifier. Shorthand * names map through the alias table; filesystem paths become file URLs; @@ -64,9 +67,20 @@ export const loadGenerator = async specifier => { ); } + generatorModules.set(generator, import.meta.resolve(resolved)); + return generator; }; +/** + * The URL of the module a generator was loaded from, which the specifiers of + * its Markdown pipeline resolve against. + * + * @param {GeneratorMetadata} generator - A generator loaded by `loadGenerator` + * @returns {string | undefined} + */ +export const getGeneratorModule = generator => generatorModules.get(generator); + /** * Loads the given generators plus the transitive closure of their * dependencies (via `dependsOn`) and dependents (via `dependent`). diff --git a/packages/core/src/generators/types.d.ts b/packages/core/src/generators/types.d.ts index 0db82b9f6..8bee0399e 100644 --- a/packages/core/src/generators/types.d.ts +++ b/packages/core/src/generators/types.d.ts @@ -70,6 +70,14 @@ declare global { hasParallelProcessor?: boolean; + /** + * The unified plugins this generator processes Markdown with, listed as + * in the `markdown` option, with paths relative to its module. Each list + * takes the configured plugins in place of its `'...'`. + * `getProcessor(name)` gives its processor. + */ + markdown?: import('../utils/configuration/types').MarkdownPipeline; + /** * The immediate generator that this generator depends on. * For example, the `html` generator depends on the `react` generator. diff --git a/packages/core/src/plugins/type-annotations/remark.mjs b/packages/core/src/plugins/type-annotations/remark.mjs index 82279efeb..1ef8c351a 100644 --- a/packages/core/src/plugins/type-annotations/remark.mjs +++ b/packages/core/src/plugins/type-annotations/remark.mjs @@ -10,12 +10,16 @@ import { typeAnnotationSyntax } from './syntax.mjs'; * Remark plugin that teaches the parser to treat any balanced `{...}` span in * text as a `typeAnnotation` node whose value is a TypeScript type expression. * - * Only registered on the non-MDX pipeline — in MDX, `{...}` is a real - * expression and is handled by remark-mdx instead. + * It does nothing in MDX, where `{...}` is a real expression, handled by + * remark-mdx instead. * * @this {import('unified').Processor} */ export default function remarkTypeAnnotations() { + if (this.data('mdx')) { + return; + } + const data = this.data(); (data.micromarkExtensions ??= []).push(typeAnnotationSyntax()); diff --git a/packages/core/src/threading/__tests__/fixtures/markdown-plugins-reporter.mjs b/packages/core/src/threading/__tests__/fixtures/markdown-plugins-reporter.mjs new file mode 100644 index 000000000..19eb4cadc --- /dev/null +++ b/packages/core/src/threading/__tests__/fixtures/markdown-plugins-reporter.mjs @@ -0,0 +1,24 @@ +import { getMarkdownPlugins } from '#utils/markdown/plugins.mjs'; + +/** + * Test generator that reports the rehype plugins of its Markdown pipeline as + * loaded inside the worker, so their loading there can be asserted. + * + * @type {GeneratorMetadata} + */ +export default { + name: 'markdown-plugins-reporter', + version: '1.0.0', + description: 'Reports the rehype plugins loaded inside the worker', + dependsOn: 'ast', + markdown: { rehypePlugins: ['./rehype-plugin.mjs', '...'] }, + processChunk: async (_input, itemIndices) => + itemIndices.map(() => + getMarkdownPlugins('markdown-plugins-reporter').rehypePlugins.map( + ([plugin, options]) => `${plugin.name} ${JSON.stringify(options)}` + ) + ), + async generate() { + return []; + }, +}; diff --git a/packages/core/src/threading/__tests__/fixtures/rehype-plugin.mjs b/packages/core/src/threading/__tests__/fixtures/rehype-plugin.mjs new file mode 100644 index 000000000..047c6b46e --- /dev/null +++ b/packages/core/src/threading/__tests__/fixtures/rehype-plugin.mjs @@ -0,0 +1,4 @@ +/** + * A rehype plugin doing nothing, for workers to load. + */ +export default function rehypeFixture() {} diff --git a/packages/core/src/threading/__tests__/index.test.mjs b/packages/core/src/threading/__tests__/index.test.mjs index b5f6b63fa..dc6b63a3c 100644 --- a/packages/core/src/threading/__tests__/index.test.mjs +++ b/packages/core/src/threading/__tests__/index.test.mjs @@ -1,4 +1,4 @@ -import { strictEqual } from 'node:assert'; +import { deepStrictEqual, strictEqual } from 'node:assert'; import { describe, it } from 'node:test'; import { fileURLToPath } from 'node:url'; @@ -10,6 +10,10 @@ const reporterSpecifier = fileURLToPath( import.meta.resolve('./fixtures/log-level-reporter.mjs') ); +const pluginsReporterSpecifier = fileURLToPath( + import.meta.resolve('./fixtures/markdown-plugins-reporter.mjs') +); + /** * Runs a function with the logger temporarily set to the given level. * @@ -80,4 +84,34 @@ describe('createWorkerPool', () => { await pool.destroy(); } }); + + it('should load the Markdown pipeline of the generator in the worker', async () => { + const pool = createWorkerPool(1); + + try { + const [plugins] = await pool.run({ + generatorSpecifier: pluginsReporterSpecifier, + input: [null], + itemIndices: [0], + extra: {}, + configuration: { + 'markdown-plugins-reporter': { + markdown: { + rehypePlugins: [ + [ + import.meta.resolve('./fixtures/rehype-plugin.mjs'), + { configured: true }, + ], + ], + }, + }, + }, + }); + + // The configured plugin is the one its pipeline has, which it configures + deepStrictEqual(plugins, ['rehypeFixture {"configured":true}']); + } finally { + await pool.destroy(); + } + }); }); diff --git a/packages/core/src/threading/chunk-worker.mjs b/packages/core/src/threading/chunk-worker.mjs index 7bfbae112..a0ff75015 100644 --- a/packages/core/src/threading/chunk-worker.mjs +++ b/packages/core/src/threading/chunk-worker.mjs @@ -3,6 +3,7 @@ import { workerData } from 'node:worker_threads'; import { loadGenerator } from '#generators/loader.mjs'; import logger from '#logger/index.mjs'; import { setConfig } from '#utils/configuration/index.mjs'; +import { loadMarkdownPlugins } from '#utils/markdown/plugins.mjs'; if (workerData?.logLevel !== undefined) { logger.setLogLevel(workerData.logLevel); @@ -26,5 +27,8 @@ export default async ({ const generator = await loadGenerator(generatorSpecifier); + // Plugins can't be sent to workers, which import the pipeline themselves + await loadMarkdownPlugins(generator, configuration[generator.name]?.markdown); + return generator.processChunk(input, itemIndices, extra); }; diff --git a/packages/core/src/utils/configuration/__tests__/index.test.mjs b/packages/core/src/utils/configuration/__tests__/index.test.mjs index c08023cf9..140c0f350 100644 --- a/packages/core/src/utils/configuration/__tests__/index.test.mjs +++ b/packages/core/src/utils/configuration/__tests__/index.test.mjs @@ -3,6 +3,9 @@ import { mkdtempSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { describe, it, mock, beforeEach } from 'node:test'; +import { pathToFileURL } from 'node:url'; + +import logger from '../../../logger/index.mjs'; // Mock dependencies const mockParseChangelog = mock.fn(async changelog => [changelog]); @@ -22,7 +25,11 @@ const createMockConfig = (overrides = {}) => ({ // Synthetic generators keyed by specifier; the identity resolver below means // shorthand names and specifiers are the same thing in these tests. const mockGenerators = { - json: { name: 'json', defaultConfiguration: { format: 'json' } }, + json: { + name: 'json', + defaultConfiguration: { format: 'json' }, + markdown: { remarkPlugins: ['file:///syntax.mjs', '...'] }, + }, html: { name: 'html', defaultConfiguration: { format: 'html' } }, markdown: { name: 'markdown' }, web: { @@ -31,6 +38,11 @@ const mockGenerators = { showSearchBox: Array.isArray(config.target) && config.target.includes('orama-db'), }), + markdown: { + remarkPlugins: ['...'], + rehypePlugins: ['...'], + recmaPlugins: ['...'], + }, }, }; @@ -39,6 +51,7 @@ mock.module('../../../generators/loader.mjs', { exports: { resolveGeneratorSpecifier: specifier => specifier, loadGenerator: async specifier => mockGenerators[specifier], + getGeneratorModule: () => undefined, // Defaults are computed from the loaded generators; returning the full // set regardless of targets keeps the assertions below simple. loadGenerators: async () => new Map(Object.entries(mockGenerators)), @@ -140,6 +153,48 @@ describe('config.mjs', () => { assert.strictEqual(result.global.project, 'Node.js'); assert.strictEqual(result.global.repository, 'nodejs/node'); }); + + it('should resolve Markdown plugins from the file declaring them', async () => { + const dir = mkdtempSync(join(tmpdir(), 'doc-kit-config-')); + + writeFileSync( + join(dir, 'preset.mjs'), + 'export default { "jsx-ast": { markdown: { rehypePlugins: ["./rehype.mjs"] } } };' + ); + + mockConfigLoad.mock.mockImplementationOnce(async () => ({ + config: { + extends: '../preset.mjs', + global: { + markdown: { remarkPlugins: [['../remark.mjs', { a: 1 }]] }, + }, + }, + filepath: join(dir, 'config', 'doc-kit.config.mjs'), + })); + + const result = await loadConfigFile('any'); + + assert.deepStrictEqual(result.global.markdown.remarkPlugins, [ + [pathToFileURL(join(dir, 'remark.mjs')).href, { a: 1 }], + ]); + // A preset's plugins resolve from the preset + assert.deepStrictEqual(result['jsx-ast'].markdown.rehypePlugins, [ + pathToFileURL(join(dir, 'rehype.mjs')).href, + ]); + }); + + it('should reject Markdown plugins that are not module specifiers', async () => { + mockConfigLoad.mock.mockImplementationOnce(async () => ({ + config: { global: { markdown: { rehypePlugins: [() => {}] } } }, + filepath: '/doc-kit.config.mjs', + })); + + await assert.rejects(loadConfigFile('any'), { + name: 'TypeError', + message: + /^global\.markdown\.rehypePlugins\[0\] in \/doc-kit\.config\.mjs must be a module specifier/, + }); + }); }); describe('createConfigFromCLIOptions', () => { @@ -322,6 +377,64 @@ describe('config.mjs', () => { assert.ok(config.html); assert.ok(config.markdown); }); + + /** + * Creates a run configuration from a configuration file's contents. + * + * @param {object} contents - The configuration file's contents + */ + const configureWith = contents => { + mockConfigLoad.mock.mockImplementationOnce(async () => ({ + config: createMockConfig(contents), + filepath: '/doc-kit.config.mjs', + })); + + return createRunConfiguration({ configFile: '/doc-kit.config.mjs' }); + }; + + const url = name => pathToFileURL(`/plugins/${name}.mjs`).href; + + it('should give each generator the Markdown plugins its pipeline takes', async () => { + const config = await configureWith({ + global: { + markdown: { + remarkPlugins: ['/plugins/math.mjs'], + rehypePlugins: ['/plugins/katex.mjs'], + }, + }, + web: { markdown: { remarkPlugins: ['/plugins/web.mjs'] } }, + }); + + assert.deepStrictEqual(config.json.markdown, { + remarkPlugins: [url('math')], + }); + // The global remark plugins ran when `ast` parsed what `web` renders + assert.deepStrictEqual(config.web.markdown, { + remarkPlugins: [url('web')], + rehypePlugins: [url('katex')], + recmaPlugins: [], + }); + assert.strictEqual(config.html.markdown, undefined); + }); + + it('should ignore the Markdown plugins a generator does not take', async t => { + const warn = t.mock.method(logger, 'warn', () => {}); + + const config = await configureWith({ + json: { markdown: { rehypePlugins: ['/plugins/rehype.mjs'] } }, + html: { markdown: { remarkPlugins: ['/plugins/remark.mjs'] } }, + }); + + assert.deepStrictEqual(config.json.markdown, { remarkPlugins: [] }); + assert.strictEqual(config.html.markdown, undefined); + assert.deepStrictEqual( + warn.mock.calls.map(({ arguments: [message] }) => message), + [ + 'Ignoring `json.markdown.rehypePlugins`: `json` does not take them.', + 'Ignoring `html.markdown.remarkPlugins`: `html` does not take them.', + ] + ); + }); }); describe('setConfig and getConfig', () => { diff --git a/packages/core/src/utils/configuration/index.mjs b/packages/core/src/utils/configuration/index.mjs index af7d2b4a6..ac61fff35 100644 --- a/packages/core/src/utils/configuration/index.mjs +++ b/packages/core/src/utils/configuration/index.mjs @@ -1,8 +1,6 @@ import { readFileSync } from 'node:fs'; -import { createRequire } from 'node:module'; import { cpus } from 'node:os'; -import { dirname, isAbsolute, resolve } from 'node:path'; -import { pathToFileURL } from 'node:url'; +import { fileURLToPath } from 'node:url'; import { isMainThread } from 'node:worker_threads'; import { cosmiconfig } from 'cosmiconfig'; @@ -16,6 +14,15 @@ import logger from '#logger/index.mjs'; import { parseChangelog, parseIndex } from '#parsers/markdown.mjs'; import { enforceArray } from '#utils/array.mjs'; import { leftHandAssign } from '#utils/generators.mjs'; +import { resolveSpecifier } from '#utils/loaders.mjs'; +import { + CONFIGURED_PLUGINS, + PLUGIN_LISTS, +} from '#utils/markdown/constants.mjs'; +import { + resolveMarkdown, + resolveMarkdownPipeline, +} from '#utils/markdown/plugins.mjs'; import { deepMerge } from '#utils/misc.mjs'; import { DEFAULT_CHUNK_SIZE, DEFAULT_MAX_THREADS } from './constants.mjs'; @@ -64,6 +71,7 @@ export const getDefaultConfig = (generators, config) => // from, so generators render single-version output. changelog: [], pathsToCopy: ['assets', 'public', 'static'], + markdown: { remarkPlugins: [], rehypePlugins: [], recmaPlugins: [] }, }, // The number of wasm memory instances is severely limited on @@ -79,21 +87,29 @@ export const getDefaultConfig = (generators, config) => ); /** - * Resolves an `extends` entry of a configuration file into an importable - * URL: relative paths resolve against the configuration file, anything else - * resolves as a package import specifier (e.g. `@node-core/doc-kit/config`). + * Resolves the Markdown plugins of a configuration (in `global`, and each + * generator's section) from the file declaring them. * - * @param {string} specifier - The `extends` entry - * @param {string} configFilePath - The configuration file it appears in - * @returns {string} A `file:` URL to import + * @param {Partial} config - The configuration + * @param {string} filePath - The file declaring it + * @returns {Partial} */ -const resolveConfigExtends = (specifier, configFilePath) => { - if (specifier.startsWith('.') || isAbsolute(specifier)) { - return pathToFileURL(resolve(dirname(configFilePath), specifier)).href; - } - - return pathToFileURL(createRequire(configFilePath).resolve(specifier)).href; -}; +const resolveMarkdownPlugins = (config, filePath) => + Object.fromEntries( + Object.entries(config).map(([name, value]) => [ + name, + value?.markdown + ? { + ...value, + markdown: resolveMarkdown( + value.markdown, + `${name}.markdown`, + filePath + ), + } + : value, + ]) + ); /** * Loads an explicit configuration file or searches for one using cosmiconfig. @@ -112,15 +128,60 @@ export const loadConfigFile = async filePath => { let { extends: presets, ...config } = result.config ?? {}; + config = resolveMarkdownPlugins(config, result.filepath); + for (const preset of enforceArray(presets ?? []).toReversed()) { - const module = await import(resolveConfigExtends(preset, result.filepath)); + const url = resolveSpecifier(preset, result.filepath); + const module = await import(url); - config = deepMerge(module.default ?? module, config); + // A preset's Markdown plugins resolve from the preset + config = deepMerge( + resolveMarkdownPlugins(module.default ?? module, fileURLToPath(url)), + config + ); } return config; }; +/** + * Returns the Markdown plugins a generator takes: for each list of its + * pipeline with a `'...'`, the global ones, then its own. Generators rendering + * Markdown (with rehype or recma plugins) skip the global remark plugins, + * which already ran in `ast`. Its own plugins it doesn't take are ignored with + * a warning. + * + * @param {GeneratorMetadata} generator - The generator + * @param {Partial} [markdown] - Its own plugins + * @param {import('./types').GlobalConfiguration} global - The global configuration + * @returns {Partial | undefined} + */ +const configureMarkdown = (generator, markdown = {}, global) => { + const pipeline = resolveMarkdownPipeline(generator); + const renders = Boolean(pipeline.rehypePlugins || pipeline.recmaPlugins); + const configured = {}; + + for (const list of PLUGIN_LISTS) { + if (!pipeline[list]?.includes(CONFIGURED_PLUGINS)) { + if (markdown[list]?.length > 0) { + logger.warn( + `Ignoring \`${generator.name}.markdown.${list}\`: ` + + `\`${generator.name}\` does not take them.` + ); + } + + continue; + } + + configured[list] = [ + ...(list === 'remarkPlugins' && renders ? [] : global.markdown[list]), + ...(markdown[list] ?? []), + ]; + } + + return generator.markdown && configured; +}; + /** * Transforms configuration values that need async processing or coercion. * Only processes values that haven't been transformed yet (strings for changelog/index, non-coerced versions). @@ -232,12 +293,20 @@ export const createRunConfiguration = async options => { // Now assign to each generator config (they inherit from global) await Promise.all( - [...generators.values()].map(async ({ name }) => { - const value = merged[name]; + [...generators.values()].map(async generator => { + const value = merged[generator.name]; // Transform generator-specific overrides await transformConfig(value); + // A generator's own Markdown plugins add to the global ones. Set even + // when undefined, so the assignment below doesn't copy the global ones + value.markdown = configureMarkdown( + generator, + value.markdown, + merged.global + ); + // Assign from global (this populates missing values from global) leftHandAssign(value, merged.global); }) diff --git a/packages/core/src/utils/configuration/types.d.ts b/packages/core/src/utils/configuration/types.d.ts index b9b4b6299..2437302bc 100644 --- a/packages/core/src/utils/configuration/types.d.ts +++ b/packages/core/src/utils/configuration/types.d.ts @@ -62,4 +62,30 @@ export type GlobalConfiguration = { // Git ref (i.e. HEAD) ref: string; + + // Markdown plugins added to the pipeline of each generator processing + // Markdown. A generator's own `markdown` adds to them, rather than + // replacing them. + markdown: MarkdownConfiguration; +}; + +// A plugin's module specifier (a package name, or a path relative to the file +// declaring it), alone or with its options +export type PluginEntry = string | [specifier: string, options?: unknown]; + +export type MarkdownConfiguration = { + // remark plugins, run on Markdown syntax trees + remarkPlugins: Array; + + // rehype plugins, run on HTML syntax trees + rehypePlugins: Array; + + // recma plugins, run on JavaScript syntax trees + recmaPlugins: Array; +}; + +// A generator's Markdown pipeline, where each list takes the configured +// plugins in place of its `'...'` +export type MarkdownPipeline = { + [List in keyof MarkdownConfiguration]?: Array; }; diff --git a/packages/core/src/utils/loaders.mjs b/packages/core/src/utils/loaders.mjs index 0d350c94b..b2a7fe78a 100644 --- a/packages/core/src/utils/loaders.mjs +++ b/packages/core/src/utils/loaders.mjs @@ -1,5 +1,6 @@ import { readFile } from 'node:fs/promises'; -import { extname, isAbsolute, join } from 'node:path'; +import { createRequire } from 'node:module'; +import { dirname, extname, isAbsolute, join, resolve } from 'node:path'; import { pathToFileURL } from 'node:url'; /** @@ -28,6 +29,27 @@ export const loadFromURL = async url => { } }; +/** + * Resolves an import specifier into a `file:` URL: relative paths resolve + * against the file it appears in, anything else as a package import + * specifier (e.g. `@node-core/doc-kit/config`). + * + * @param {string} specifier - A path, a package specifier, or a `file:` URL + * @param {string} filePath - The file it appears in + * @returns {string} A `file:` URL to import + */ +export const resolveSpecifier = (specifier, filePath) => { + if (specifier.startsWith('file:')) { + return specifier; + } + + if (specifier.startsWith('.') || isAbsolute(specifier)) { + return pathToFileURL(resolve(dirname(filePath), specifier)).href; + } + + return pathToFileURL(createRequire(filePath).resolve(specifier)).href; +}; + /** * Dynamically imports a module from a URL, using JSON import assertion if applicable. * diff --git a/packages/core/src/utils/markdown/__tests__/plugins.test.mjs b/packages/core/src/utils/markdown/__tests__/plugins.test.mjs new file mode 100644 index 000000000..3cd4517ff --- /dev/null +++ b/packages/core/src/utils/markdown/__tests__/plugins.test.mjs @@ -0,0 +1,197 @@ +import assert from 'node:assert/strict'; +import { mkdtempSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { describe, it } from 'node:test'; +import { pathToFileURL } from 'node:url'; + +import { getMarkdownPlugins, loadMarkdownPlugins } from '../plugins.mjs'; + +const dir = mkdtempSync(join(tmpdir(), 'doc-kit-markdown-plugins-')); + +/** + * Writes a module, returning its URL, as configuration resolves it. + * + * @param {string} name + * @param {string} source + */ +const writeModule = (name, source) => { + writeFileSync(join(dir, name), source); + + return pathToFileURL(join(dir, name)).href; +}; + +const plugin = writeModule('plugin.mjs', 'export default function plugin() {}'); +const other = writeModule('other.mjs', 'export default function other() {}'); +const list = writeModule( + 'list.mjs', + `import plugin from './plugin.mjs'; + export default [[plugin, { from: 'list' }]];` +); +const preset = writeModule( + 'preset.mjs', + `import plugin from './plugin.mjs'; + export default { plugins: [plugin] };` +); +const setUp = writeModule( + 'set-up.mjs', + `export async function load(options) { + return Object.assign(function setUp() {}, { options }); + }` +); + +const { default: pluginFunction } = await import(plugin); +const { default: otherFunction } = await import(other); + +/** + * A generator whose Markdown pipeline only has the configured plugins. + * + * @param {string} name + */ +const configurable = name => ({ + name, + markdown: { + remarkPlugins: ['...'], + rehypePlugins: ['...'], + recmaPlugins: ['...'], + }, +}); + +describe('loadMarkdownPlugins', () => { + it('loads plugins as `processor.use()` takes them', async () => { + await loadMarkdownPlugins(configurable('formats'), { + remarkPlugins: [plugin, [other, { a: 1 }]], + rehypePlugins: [list], + recmaPlugins: [preset], + }); + + assert.deepStrictEqual(getMarkdownPlugins('formats'), { + remarkPlugins: [[pluginFunction], [otherFunction, { a: 1 }]], + // A list is used as a preset + rehypePlugins: [{ plugins: [[pluginFunction, { from: 'list' }]] }], + recmaPlugins: [{ plugins: [pluginFunction] }], + }); + }); + + it('loads a plugin through its `load`, given the options', async () => { + await loadMarkdownPlugins(configurable('setting-up'), { + rehypePlugins: [[setUp, { ready: true }]], + }); + + const [[setUpPlugin, ...options]] = + getMarkdownPlugins('setting-up').rehypePlugins; + + assert.equal(setUpPlugin.name, 'setUp'); + assert.deepStrictEqual(setUpPlugin.options, { ready: true }); + assert.deepStrictEqual(options, []); + }); + + it('puts the configured plugins in place of `...`', async () => { + await loadMarkdownPlugins( + { + name: 'pipeline', + markdown: { + remarkPlugins: [other, '...', [other, { last: true }]], + // Without `...`, a list takes no configured plugins + rehypePlugins: [other], + }, + }, + { remarkPlugins: [plugin], rehypePlugins: [plugin] } + ); + + assert.deepStrictEqual(getMarkdownPlugins('pipeline'), { + remarkPlugins: [ + [otherFunction], + [pluginFunction], + [otherFunction, { last: true }], + ], + rehypePlugins: [[otherFunction]], + recmaPlugins: [], + }); + + // The generator's own plugins, without the configured ones + assert.deepStrictEqual(getMarkdownPlugins('pipeline', false), { + remarkPlugins: [[otherFunction], [otherFunction, { last: true }]], + rehypePlugins: [[otherFunction]], + recmaPlugins: [], + }); + }); + + it('configures the plugins it has, rather than running them twice', async () => { + await loadMarkdownPlugins( + { + name: 'configuring', + markdown: { + rehypePlugins: [[plugin, { list: [1], nested: { a: 1 } }], '...'], + }, + }, + { + rehypePlugins: [ + [plugin, { list: [2], nested: { b: 2 } }], + // As a generator's own plugins add to the global ones + [setUp, { list: [1] }], + other, + [setUp, { list: [2] }], + ], + } + ); + + const [own, [setUpPlugin], ...rest] = + getMarkdownPlugins('configuring').rehypePlugins; + + // Options merge: lists add up, and objects merge + assert.deepStrictEqual(own, [ + pluginFunction, + { list: [1, 2], nested: { a: 1, b: 2 } }, + ]); + assert.deepStrictEqual(setUpPlugin.options, { list: [1, 2] }); + assert.deepStrictEqual(rest, [[otherFunction]]); + + // Its own plugins are configured without the configured ones too + assert.deepStrictEqual(getMarkdownPlugins('configuring', false), { + remarkPlugins: [], + rehypePlugins: [own], + recmaPlugins: [], + }); + }); + + it('rejects the plugins it cannot import', async () => { + await assert.rejects( + loadMarkdownPlugins(configurable('missing'), { + remarkPlugins: [`${plugin}-missing`], + }), + { code: 'ERR_MODULE_NOT_FOUND' } + ); + }); + + it('loads each generator its own plugins, once', async () => { + const one = configurable('one'); + const markdown = { rehypePlugins: [[plugin, { for: 'one' }]] }; + + await loadMarkdownPlugins(one, markdown); + await loadMarkdownPlugins(configurable('two'), { + rehypePlugins: [[plugin, { for: 'two' }]], + }); + + const plugins = getMarkdownPlugins('one'); + + assert.deepStrictEqual(plugins.rehypePlugins, [ + [pluginFunction, { for: 'one' }], + ]); + assert.deepStrictEqual(getMarkdownPlugins('two').rehypePlugins, [ + [pluginFunction, { for: 'two' }], + ]); + + // As workers do for each task, from a copy of the configuration + await loadMarkdownPlugins(one, structuredClone(markdown)); + + assert.equal(getMarkdownPlugins('one'), plugins); + + // A generator without a pipeline has none to load + await loadMarkdownPlugins({ name: 'none' }, markdown); + + assert.throws(() => getMarkdownPlugins('none'), { + message: 'The Markdown pipeline of "none" is not loaded on this thread', + }); + }); +}); diff --git a/packages/core/src/utils/markdown/__tests__/processor.test.mjs b/packages/core/src/utils/markdown/__tests__/processor.test.mjs new file mode 100644 index 000000000..4d6fb8f57 --- /dev/null +++ b/packages/core/src/utils/markdown/__tests__/processor.test.mjs @@ -0,0 +1,105 @@ +import assert from 'node:assert/strict'; +import { mkdtempSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { beforeEach, describe, it } from 'node:test'; +import { pathToFileURL } from 'node:url'; + +import { loadMarkdownPlugins } from '../plugins.mjs'; +import { getProcessor } from '../processor.mjs'; + +const dir = mkdtempSync(join(tmpdir(), 'doc-kit-processor-')); + +// What the plugins below saw, in the order they ran +const seen = []; + +globalThis.seenByPlugins = seen; + +/** + * Writes a plugin recording the type of the first node of its tree, and + * whether it processes MDX, returning its URL. + * + * @param {string} name + */ +const recorder = name => { + writeFileSync( + join(dir, `${name}.mjs`), + `export default function ${name}() { + const mdx = this.data('mdx') ? ' (MDX)' : ''; + + return tree => { + globalThis.seenByPlugins.push('${name}: ' + tree.children[0].type + mdx); + }; + }` + ); + + return pathToFileURL(join(dir, `${name}.mjs`)).href; +}; + +// A pipeline turning Markdown into HTML, with doc-kit's Markdown syntax +await loadMarkdownPlugins( + { + name: 'to-html', + markdown: { + remarkPlugins: [ + import.meta.resolve('remark-parse'), + import.meta.resolve('#plugins/type-annotations/remark.mjs'), + import.meta.resolve('remark-gfm'), + '...', + ], + rehypePlugins: [ + import.meta.resolve('remark-rehype'), + '...', + import.meta.resolve('rehype-stringify'), + ], + }, + }, + { remarkPlugins: [recorder('remark')], rehypePlugins: [recorder('rehype')] } +); + +describe('getProcessor', () => { + beforeEach(() => { + seen.length = 0; + }); + + it('runs the plugins of each syntax tree in turn', async () => { + const html = await getProcessor('to-html').process('# Hi ~~there~~'); + + assert.equal(String(html), '

Hi there

'); + assert.deepStrictEqual(seen, ['remark: heading', 'rehype: element']); + }); + + it('processes MDX, telling the plugins so', async () => { + const processor = getProcessor('to-html', { mdx: true }); + + // `{...}` is an expression in MDX, and a type annotation in Markdown + const [paragraph] = processor.parse('A {string}').children; + + assert.equal(paragraph.children[1].type, 'mdxTextExpression'); + assert.equal( + getProcessor('to-html').parse('A {string}').children[0].children[1].type, + 'typeAnnotation' + ); + + await processor.process(''); + + assert.deepStrictEqual(seen, [ + 'remark: mdxJsxFlowElement (MDX)', + 'rehype: element (MDX)', + ]); + }); + + it('runs without the configured plugins, if asked', async () => { + await getProcessor('to-html', { configured: false }).process('# Hi'); + + assert.deepStrictEqual(seen, []); + }); + + it('gives the same processor for the same pipeline', () => { + assert.equal(getProcessor('to-html'), getProcessor('to-html')); + assert.notEqual( + getProcessor('to-html'), + getProcessor('to-html', { mdx: true }) + ); + }); +}); diff --git a/packages/core/src/utils/markdown/constants.mjs b/packages/core/src/utils/markdown/constants.mjs new file mode 100644 index 000000000..dc25c88c7 --- /dev/null +++ b/packages/core/src/utils/markdown/constants.mjs @@ -0,0 +1,7 @@ +'use strict'; + +// The plugin lists of a Markdown pipeline, one for each syntax tree +export const PLUGIN_LISTS = ['remarkPlugins', 'rehypePlugins', 'recmaPlugins']; + +// Marks where a list of a generator's pipeline takes the configured plugins +export const CONFIGURED_PLUGINS = '...'; diff --git a/packages/core/src/utils/markdown/plugins.mjs b/packages/core/src/utils/markdown/plugins.mjs new file mode 100644 index 000000000..033251ad3 --- /dev/null +++ b/packages/core/src/utils/markdown/plugins.mjs @@ -0,0 +1,219 @@ +'use strict'; + +import { fileURLToPath } from 'node:url'; + +import { getGeneratorModule } from '#generators/loader.mjs'; +import { enforceArray } from '#utils/array.mjs'; +import { resolveSpecifier } from '#utils/loaders.mjs'; +import { isPlainObject } from '#utils/misc.mjs'; + +import { CONFIGURED_PLUGINS, PLUGIN_LISTS } from './constants.mjs'; + +/** + * The plugins of a Markdown pipeline, as `processor.use()` takes them. + * + * @typedef {Record<'remarkPlugins' | 'rehypePlugins' | 'recmaPlugins', import('unified').PluggableList>} MarkdownPlugins + */ + +// The pipeline each generator runs on this thread, by its name +const loadedPipelines = new Map(); + +/** + * Resolves the module specifiers of Markdown plugins (in the `markdown` + * option, or a generator's pipeline) from the file declaring them into + * `file:` URLs, which any thread can import. + * + * @param {import('../configuration/types').MarkdownPipeline} markdown - The plugins + * @param {string} label - Where they are declared, for errors (e.g. `global.markdown`) + * @param {string} filePath - The file declaring them + * @returns {import('../configuration/types').MarkdownPipeline} + */ +export const resolveMarkdown = (markdown, label, filePath) => { + const resolved = { ...markdown }; + + for (const list of PLUGIN_LISTS) { + resolved[list] &&= enforceArray(markdown[list]).map((entry, index) => { + if (entry === CONFIGURED_PLUGINS) { + return entry; + } + + const [specifier, ...options] = enforceArray(entry); + + if (typeof specifier !== 'string') { + throw new TypeError( + `${label}.${list}[${index}] in ${filePath} must be a module ` + + 'specifier (a package name or a path), as Markdown is processed ' + + 'in worker threads, which import the plugins themselves.' + ); + } + + const url = resolveSpecifier(specifier, filePath); + + return options.length === 0 ? url : [url, ...options]; + }); + } + + return resolved; +}; + +/** + * Resolves the module specifiers of a generator's Markdown pipeline from the + * module it was loaded from. + * + * @param {GeneratorMetadata} generator - The generator + * @returns {import('../configuration/types').MarkdownPipeline} + */ +export const resolveMarkdownPipeline = generator => { + const module = getGeneratorModule(generator); + + return resolveMarkdown( + generator.markdown, + `${generator.name}.markdown`, + module && fileURLToPath(module) + ); +}; + +/** + * Merges options into others: arrays add up, plain objects merge, and other + * values replace the ones they are merged into. + * + * @param {unknown} options - The options merged into + * @param {unknown} added - The options merged + * @returns {unknown} + */ +const mergeOptions = (options, added) => { + if (Array.isArray(options) && Array.isArray(added)) { + return [...options, ...added]; + } + + if (!isPlainObject(options) || !isPlainObject(added)) { + return added; + } + + const merged = { ...options }; + + for (const [key, value] of Object.entries(added)) { + merged[key] = mergeOptions(options[key], value); + } + + return merged; +}; + +/** + * Builds a list of a pipeline, with the configured plugins in place of its + * `'...'`. A configured plugin the list already has, or added earlier, isn't + * added again: its options merge into the ones it has. + * + * @param {Array} [own] - The list + * @param {Array} [configured] - The configured plugins + * @returns {Array<{ entry: import('../configuration/types').PluginEntry, added: boolean }>} + */ +const configureList = (own = [], configured = []) => { + const plugins = own.map(entry => ({ entry, added: false })); + const added = []; + + for (const entry of configured) { + const [specifier, options] = enforceArray(entry); + + const listed = [...plugins, ...added].find( + plugin => enforceArray(plugin.entry)[0] === specifier + ); + + if (!listed) { + added.push({ entry, added: true }); + } else if (options !== undefined) { + const [, listedOptions] = enforceArray(listed.entry); + + listed.entry = [specifier, mergeOptions(listedOptions, options)]; + } + } + + return plugins.flatMap(plugin => + plugin.entry === CONFIGURED_PLUGINS ? added : [plugin] + ); +}; + +/** + * Imports a plugin as `processor.use()` takes it. Its module default-exports + * the plugin, a list of plugins, or a preset, or exports an async `load`, + * taking the plugin's options and returning it, for setup unified plugins + * can't do themselves. + * + * @param {import('../configuration/types').PluginEntry} entry - The plugin + * @returns {Promise} + */ +const importPlugin = async entry => { + const [specifier, ...options] = enforceArray(entry); + const { default: plugin, load } = await import(specifier); + + if (typeof load === 'function') { + return [await load(...options)]; + } + + if (typeof plugin === 'function') { + return [plugin, ...options]; + } + + // A list is used as a preset, not to be taken for a [plugin, options] pair + return Array.isArray(plugin) ? { plugins: plugin } : plugin; +}; + +/** + * Loads a generator's Markdown pipeline on the current thread, with the + * plugins configured for it. Loading the same configuration again does + * nothing. + * + * @param {GeneratorMetadata} generator - The generator + * @param {Partial} [markdown] - The plugins configured for it + * @returns {Promise} + */ +export const loadMarkdownPlugins = async (generator, markdown = {}) => { + if (!generator.markdown) { + return; + } + + const key = JSON.stringify(markdown); + const loaded = loadedPipelines.get(generator.name); + + if (loaded?.generator === generator && loaded.key === key) { + return; + } + + const pipeline = resolveMarkdownPipeline(generator); + const configured = {}; + const own = {}; + + for (const list of PLUGIN_LISTS) { + const plugins = configureList(pipeline[list], markdown[list]); + + const imported = await Promise.all( + plugins.map(({ entry }) => importPlugin(entry)) + ); + + configured[list] = imported; + own[list] = imported.filter((_, index) => !plugins[index].added); + } + + loadedPipelines.set(generator.name, { generator, key, configured, own }); +}; + +/** + * Gets the plugins of a generator's Markdown pipeline, as loaded on the + * current thread: with the configured plugins, or only its own (with their + * configured options). + * + * @param {string} generator - The name of the generator + * @param {boolean} [configured] - Whether with the configured plugins + * @returns {MarkdownPlugins} + */ +export const getMarkdownPlugins = (generator, configured = true) => { + const loaded = loadedPipelines.get(generator); + + if (!loaded) { + throw new Error( + `The Markdown pipeline of "${generator}" is not loaded on this thread` + ); + } + + return configured ? loaded.configured : loaded.own; +}; diff --git a/packages/core/src/utils/markdown/processor.mjs b/packages/core/src/utils/markdown/processor.mjs new file mode 100644 index 000000000..5fbaf01c5 --- /dev/null +++ b/packages/core/src/utils/markdown/processor.mjs @@ -0,0 +1,60 @@ +'use strict'; + +import remarkMdx from 'remark-mdx'; +import { unified } from 'unified'; + +import { PLUGIN_LISTS } from './constants.mjs'; +import { getMarkdownPlugins } from './plugins.mjs'; + +// The processors of each loaded pipeline, of Markdown and of MDX +const processors = new WeakMap(); + +/** + * Creates the processor of a pipeline: its remark plugins, then its rehype + * ones, then its recma ones. A processor of MDX knows its syntax too, and + * tells plugins it processes MDX through `this.data('mdx')`. + * + * @param {import('./plugins.mjs').MarkdownPlugins} plugins - The pipeline + * @param {boolean} mdx - Whether it processes MDX + * @returns {import('unified').Processor} + */ +const createProcessor = (plugins, mdx) => { + const processor = unified().data('mdx', mdx); + + if (mdx) { + processor.use(remarkMdx); + } + + for (const list of PLUGIN_LISTS) { + processor.use(plugins[list]); + } + + return processor; +}; + +/** + * Gets the processor of a generator's Markdown pipeline, as loaded on the + * current thread (see `loadMarkdownPlugins`). + * + * @param {string} generator - The name of the generator + * @param {Object} [options] + * @param {boolean} [options.mdx] - Whether it processes MDX + * @param {boolean} [options.configured] - Whether it runs the configured + * plugins, rather than only the generator's own + * @returns {import('unified').Processor} + */ +export const getProcessor = ( + generator, + { mdx = false, configured = true } = {} +) => { + const plugins = getMarkdownPlugins(generator, configured); + + if (!processors.has(plugins)) { + processors.set(plugins, {}); + } + + const created = processors.get(plugins); + const syntax = mdx ? 'mdx' : 'markdown'; + + return (created[syntax] ??= createProcessor(plugins, mdx)); +}; From 74c9eb00785e535910d7229bad5c8b2af2d9f227 Mon Sep 17 00:00:00 2001 From: Claudio Wunder Date: Thu, 8 Oct 2026 12:59:57 +0200 Subject: [PATCH 04/10] feat(core): add the Shiki rehype plugin Highlights code with Shiki as a rehype plugin, configurable with more languages, aliases, themes, and transformers. It loads through `load`, as creating its highlighter is asynchronous, and creates the Shiki instance on first use, so threads that never highlight code skip it. Assisted-by: Claude Opus 5.5 --- .../shiki/__tests__/highlighter.test.mjs | 122 ++++++++++ .../plugins/shiki/__tests__/rehype.test.mjs | 40 ++++ .../core/src/plugins/shiki/highlighter.mjs | 218 ++++++++++++++++++ packages/core/src/plugins/shiki/rehype.mjs | 38 +++ 4 files changed, 418 insertions(+) create mode 100644 packages/core/src/plugins/shiki/__tests__/highlighter.test.mjs create mode 100644 packages/core/src/plugins/shiki/__tests__/rehype.test.mjs create mode 100644 packages/core/src/plugins/shiki/highlighter.mjs create mode 100644 packages/core/src/plugins/shiki/rehype.mjs diff --git a/packages/core/src/plugins/shiki/__tests__/highlighter.test.mjs b/packages/core/src/plugins/shiki/__tests__/highlighter.test.mjs new file mode 100644 index 000000000..91208b8e4 --- /dev/null +++ b/packages/core/src/plugins/shiki/__tests__/highlighter.test.mjs @@ -0,0 +1,122 @@ +import assert from 'node:assert/strict'; +import { mkdtempSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { describe, it } from 'node:test'; + +import { loadMarkdownPlugins } from '#utils/markdown/plugins.mjs'; + +import { createHighlighter, getHighlighter } from '../highlighter.mjs'; + +const dir = mkdtempSync(join(tmpdir(), 'doc-kit-shiki-')); + +/** + * Writes a module, returning its path. + * + * @param {string} name + * @param {string} source + */ +const writeModule = (name, source) => { + writeFileSync(join(dir, name), source); + + return join(dir, name); +}; + +const grammar = { + name: 'oxcconf', + displayName: 'Oxc Config', + scopeName: 'source.oxcconf', + patterns: [{ match: '\\bon\\b', name: 'keyword.control.oxcconf' }], +}; + +describe('createHighlighter', () => { + it('highlights every language Shiki bundles, in the default themes', async () => { + const highlighter = await createHighlighter(); + + assert.equal(highlighter.resolveLanguage('rust'), 'rust'); + assert.equal(highlighter.resolveLanguage('oxcconf'), 'text'); + assert.deepStrictEqual(highlighter.shiki.getLoadedThemes(), [ + 'github-light-default', + 'nord', + ]); + }); + + it('takes more languages, aliases, themes, and transformers', async () => { + const highlighter = await createHighlighter({ + langs: [ + grammar, + writeModule( + 'grammar.json', + JSON.stringify({ ...grammar, name: 'json-conf', scopeName: 'json' }) + ), + ], + langAlias: { conf: 'ini' }, + themes: { + light: 'github-light', + dark: writeModule('theme.json', JSON.stringify({ type: 'dark' })), + }, + transformers: [ + writeModule( + 'transformers.mjs', + `export default [{ + line(node) { + this.addClassToHast(node, 'tagged'); + }, + }];` + ), + ], + }); + + assert.equal(highlighter.resolveLanguage('oxcconf'), 'oxcconf'); + assert.equal(highlighter.resolveLanguage('json-conf'), 'json-conf'); + assert.equal( + highlighter.highlightToHtml('[section]', 'conf'), + highlighter.highlightToHtml('[section]', 'ini') + ); + + // A theme without a name is named after its scheme + assert.deepStrictEqual(highlighter.shiki.getLoadedThemes(), [ + 'github-light', + 'custom-dark', + ]); + + assert.match( + highlighter.highlightToHtml('minify on', 'oxcconf'), + /class="line tagged"/ + ); + }); + + it('gives the same highlighter for the same options', async () => { + const highlighter = await createHighlighter({ langs: [grammar] }); + + assert.equal(await createHighlighter({ langs: [grammar] }), highlighter); + assert.notEqual(await createHighlighter(), highlighter); + }); +}); + +describe('getHighlighter', () => { + const shiki = import.meta.resolve('../rehype.mjs'); + + it('gives the highlighter of the Shiki plugin of a pipeline', async () => { + await loadMarkdownPlugins( + { name: 'highlighting', markdown: { rehypePlugins: [shiki] } }, + { rehypePlugins: [[shiki, { langAlias: { conf: 'ini' } }]] } + ); + + assert.equal( + getHighlighter('highlighting'), + await createHighlighter({ langAlias: { conf: 'ini' } }) + ); + }); + + it('throws for a pipeline not highlighting code', async () => { + await loadMarkdownPlugins({ + name: 'plain', + markdown: { rehypePlugins: ['...'] }, + }); + + assert.throws(() => getHighlighter('plain'), { + message: 'The Markdown pipeline of "plain" does not highlight code', + }); + }); +}); diff --git a/packages/core/src/plugins/shiki/__tests__/rehype.test.mjs b/packages/core/src/plugins/shiki/__tests__/rehype.test.mjs new file mode 100644 index 000000000..2b596a68b --- /dev/null +++ b/packages/core/src/plugins/shiki/__tests__/rehype.test.mjs @@ -0,0 +1,40 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; + +import { toHtml } from 'hast-util-to-html'; + +import { createHighlighter } from '../highlighter.mjs'; +import { load } from '../rehype.mjs'; + +describe('load', () => { + it('gives the plugin highlighting code blocks, with its highlighter', async () => { + const shiki = await load({ langAlias: { conf: 'ini' } }); + + const tree = { + type: 'root', + children: [ + { + type: 'element', + tagName: 'pre', + properties: {}, + children: [ + { + type: 'element', + tagName: 'code', + properties: { className: ['language-conf'] }, + children: [{ type: 'text', value: '[section]' }], + }, + ], + }, + ], + }; + + shiki()(tree); + + assert.match(toHtml(tree), /class="shiki/); + assert.equal( + shiki.highlighter, + await createHighlighter({ langAlias: { conf: 'ini' } }) + ); + }); +}); diff --git a/packages/core/src/plugins/shiki/highlighter.mjs b/packages/core/src/plugins/shiki/highlighter.mjs new file mode 100644 index 000000000..dbfc8112b --- /dev/null +++ b/packages/core/src/plugins/shiki/highlighter.mjs @@ -0,0 +1,218 @@ +'use strict'; + +import { endianness } from 'node:os'; + +import { LANGS } from '@node-core/rehype-shiki'; +import createSyntaxHighlighter from '@node-core/rehype-shiki/highlighter'; +import { bundledThemes } from 'shiki/themes'; + +import { importFromURL } from '#utils/loaders.mjs'; +import { getMarkdownPlugins } from '#utils/markdown/plugins.mjs'; + +/** + * A syntax highlighter, creating its Shiki instance on first use. + * + * @typedef {Object} SyntaxHighlighter + * @property {import('shiki').HighlighterCore} shiki - The Shiki instance + * @property {(languageId?: string) => string} resolveLanguage - Resolves a language, falling back to plain text for unknown ones + * @property {(code: string, lang: string, meta?: Record) => string} highlightToHtml - Highlights code, returning the inner HTML of its `` element + * @property {(code: string, lang: string, meta?: Record) => ReturnType} highlightToHast - Highlights code, returning a HAST tree + * @property {Array} langs - The languages it highlights + */ + +/** + * Creates Shiki's regular expression engine: the wasm (Oniguruma) one where + * it can, the JavaScript one otherwise. + * + * @returns {Promise} + */ +const createEngine = async () => { + // riscv64 with sv39 has limited virtual memory space, where creating + // too many (>20) wasm memory instances fails. + // https://github.com/nodejs/node/pull/60591 + // + // The wasm highlighter is currently not compatible with big endian. + // https://github.com/nodejs/node/pull/62512#issuecomment-4243469950 + if (process.arch !== 'riscv64' && endianness() === 'LE') { + const { createOnigurumaEngine } = await import('shiki/engine/oniguruma'); + + return createOnigurumaEngine(import('shiki/wasm')); + } + + const { createJavaScriptRegexEngine } = + await import('shiki/engine/javascript'); + + // Not every bundled grammar compiles under the JavaScript engine, + // so skip the patterns it cannot handle instead of throwing. + return createJavaScriptRegexEngine({ forgiving: true }); +}; + +// The regular expression engine of this thread, for all its highlighters +let engine; + +// The highlighters of the options given, by their JSON +const highlighters = new Map(); + +/** + * Imports a list of options, each given as it is, or as a module (a path + * relative to the working directory, or a URL) default-exporting it, or a list + * of them. + * + * @template T + * @param {Array} options - The options + * @returns {Promise>} + */ +const importList = async options => { + const imported = []; + + for (const option of options) { + if (typeof option === 'string') { + imported.push(await importFromURL(option)); + } else { + imported.push(option); + } + } + + return imported.flat(); +}; + +/** + * Imports a theme: the name of one Shiki bundles, a module default-exporting + * one, or the theme itself. Shiki tells themes apart by their name, so a theme + * without one is named after its color scheme. + * + * @param {string | import('shiki').ThemeRegistration} theme - The theme + * @param {'light' | 'dark'} scheme - Its color scheme + * @returns {Promise} + */ +const importTheme = async (theme, scheme) => { + let imported = theme; + + if (typeof theme === 'string') { + if (theme in bundledThemes) { + imported = (await bundledThemes[theme]()).default; + } else { + imported = await importFromURL(theme); + } + } + + return { name: `custom-${scheme}`, ...imported }; +}; + +/** + * Imports the options of a highlighter, and creates it. + * + * @param {import('./rehype.mjs').ShikiOptions} options - The options + * @returns {Promise} + */ +const importHighlighter = async ({ + langs = [], + langAlias = {}, + themes, + transformers = [], +}) => { + engine ??= createEngine(); + + const coreOptions = { + engine: await engine, + langs: [...LANGS, ...(await importList(langs))], + // A copy, as Shiki adds the aliases of the languages it bundles to it + langAlias: { ...langAlias }, + }; + + const highlighterOptions = { + transformers: await importList(transformers), + }; + + // Without themes of its own, the highlighter has a default light and dark one + if (themes) { + const light = await importTheme(themes.light, 'light'); + const dark = await importTheme(themes.dark, 'dark'); + + coreOptions.themes = [light, dark]; + highlighterOptions.themes = { light: light.name, dark: dark.name }; + highlighterOptions.defaultColor = 'light'; + } + + let highlighter; + + /** + * Gives the Shiki highlighter, creating it on first use. + */ + const current = () => + (highlighter ??= createSyntaxHighlighter({ + coreOptions, + highlighterOptions, + })); + + return { + langs: coreOptions.langs, + + /** + * The Shiki instance. + */ + get shiki() { + return current().shiki; + }, + + /** + * Resolves a language, falling back to plain text for unknown ones. + * + * @param {string} [languageId] + */ + resolveLanguage: languageId => current().resolveLanguage(languageId), + + /** + * Highlights code, returning the inner HTML of its `` element. + * + * @param {...any} args - The code, its language, and its metadata + */ + highlightToHtml: (...args) => current().highlightToHtml(...args), + + /** + * Highlights code, returning a HAST tree. + * + * @param {...any} args - The code, its language, and its metadata + */ + highlightToHast: (...args) => current().highlightToHast(...args), + }; +}; + +/** + * Creates a highlighter of every language Shiki bundles, with the given + * options (see `./rehype.mjs`). Its Shiki instance is created on first use, + * and the same options give the same highlighter. + * + * @param {import('./rehype.mjs').ShikiOptions} [options] - The options + * @returns {Promise} + */ +export const createHighlighter = (options = {}) => { + const key = JSON.stringify(options); + + if (!highlighters.has(key)) { + highlighters.set(key, importHighlighter(options)); + } + + return highlighters.get(key); +}; + +/** + * Gets the highlighter of the Shiki plugin of a generator's Markdown pipeline, + * as loaded on the current thread, to highlight code as the pipeline does. + * + * @param {string} generator - The name of the generator + * @returns {SyntaxHighlighter} + */ +export const getHighlighter = generator => { + const shiki = getMarkdownPlugins(generator).rehypePlugins.find( + plugin => Array.isArray(plugin) && plugin[0].highlighter + ); + + if (!shiki) { + throw new Error( + `The Markdown pipeline of "${generator}" does not highlight code` + ); + } + + return shiki[0].highlighter; +}; diff --git a/packages/core/src/plugins/shiki/rehype.mjs b/packages/core/src/plugins/shiki/rehype.mjs new file mode 100644 index 000000000..284a77cb1 --- /dev/null +++ b/packages/core/src/plugins/shiki/rehype.mjs @@ -0,0 +1,38 @@ +'use strict'; + +import rehypeShikiji from '@node-core/rehype-shiki/plugin'; + +import { createHighlighter } from './highlighter.mjs'; + +/** + * The options of the Shiki plugin. Its modules are paths relative to the + * working directory, or URLs. + * + * @typedef {Object} ShikiOptions + * @property {Array} [langs] - More languages: grammars, or modules default-exporting grammars + * @property {Record} [langAlias] - Aliases of languages, e.g. `{ conf: 'ini' }` + * @property {{ light: string | import('shiki').ThemeRegistration, dark: string | import('shiki').ThemeRegistration }} [themes] - The light and dark themes: names of themes Shiki bundles, modules default-exporting themes, or themes + * @property {Array} [transformers] - Modules default-exporting Shiki transformers, or lists of them + */ + +/** + * Loads the rehype plugin highlighting code blocks with Shiki, in every + * language it bundles. The plugin exposes its highlighter as `highlighter` + * (see `getHighlighter`). + * + * @param {ShikiOptions} [options] - The options + * @returns {Promise} + */ +export async function load(options) { + const highlighter = await createHighlighter(options); + const transformer = await rehypeShikiji({ highlighter }); + + /** + * Highlights the code blocks of a tree. + */ + const shiki = () => transformer; + + shiki.highlighter = highlighter; + + return shiki; +} From 46a293811688ffc9dc1a9f77f469e75338b27064 Mon Sep 17 00:00:00 2001 From: Claudio Wunder Date: Thu, 8 Oct 2026 13:00:15 +0200 Subject: [PATCH 05/10] feat(core): use Markdown pipelines in `ast`, `metadata`, and `json` The configured remark plugins run on each whole document once `ast` parses it, and `metadata` and `json` parse and serialise Markdown with their syntax. Assisted-by: Claude Opus 5.5 --- .../ast/__tests__/generate.test.mjs | 45 +++++++++++++++++++ packages/core/src/generators/ast/generate.mjs | 10 +++-- packages/core/src/generators/ast/index.mjs | 9 ++++ .../json/__tests__/generate.test.mjs | 16 ++++++- packages/core/src/generators/json/index.mjs | 9 ++++ .../json/utils/__tests__/entry.test.mjs | 8 ++++ .../json/utils/__tests__/lifecycle.test.mjs | 8 ++++ .../json/utils/__tests__/markdown.test.mjs | 8 ++++ .../json/utils/__tests__/signature.test.mjs | 8 ++++ .../src/generators/json/utils/markdown.mjs | 4 +- .../core/src/generators/metadata/index.mjs | 9 ++++ .../src/generators/metadata/utils/parse.mjs | 6 +-- .../generators/metadata/utils/visitors.mjs | 4 +- 13 files changed, 131 insertions(+), 13 deletions(-) diff --git a/packages/core/src/generators/ast/__tests__/generate.test.mjs b/packages/core/src/generators/ast/__tests__/generate.test.mjs index 00e88cd50..7b85c20e8 100644 --- a/packages/core/src/generators/ast/__tests__/generate.test.mjs +++ b/packages/core/src/generators/ast/__tests__/generate.test.mjs @@ -3,10 +3,19 @@ import { mkdtemp, writeFile, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join, sep } from 'node:path'; import { after, before, describe, it } from 'node:test'; +import { pathToFileURL } from 'node:url'; + +import { loadGenerator } from '#generators/loader.mjs'; +import { loadMarkdownPlugins } from '#utils/markdown/plugins.mjs'; import { STABILITY_INDEX_URL } from '../constants.mjs'; import { processChunk } from '../generate.mjs'; +const ast = await loadGenerator(import.meta.resolve('../index.mjs')); + +// Files are parsed with the pipeline of `ast` +await loadMarkdownPlugins(ast); + let dir; const toPosixPath = value => value.split(sep).join('/'); @@ -157,6 +166,42 @@ describe('processChunk', () => { }); }); + describe('remark plugins', () => { + before(async () => { + // Records the file it runs on, and the headings of its document + await writeFile( + join(dir, 'recorder.mjs'), + `export default () => (tree, file) => { + const headings = tree.children.filter(node => node.type === 'heading'); + + tree.data = { recorded: file.path + ':' + headings.length }; + };` + ); + + await loadMarkdownPlugins(ast, { + remarkPlugins: [pathToFileURL(join(dir, 'recorder.mjs')).href], + }); + }); + + after(() => loadMarkdownPlugins(ast)); + + it('run on each whole document, knowing its file', async () => { + const tuple = await file('plugged.md', '# A\n\n## B\n\n## C\n'); + + const { tree } = await process(tuple); + + assert.strictEqual(tree.data.recorded, `${tuple[0]}:3`); + }); + + it('run on MDX documents too', async () => { + const tuple = await file('plugged.mdx', '# A\n\n\n'); + + const { tree } = await process(tuple); + + assert.strictEqual(tree.data.recorded, `${tuple[0]}:1`); + }); + }); + describe('chunking', () => { it('only processes the requested indices', async () => { const a = await file('a.md', '# A\n'); diff --git a/packages/core/src/generators/ast/generate.mjs b/packages/core/src/generators/ast/generate.mjs index 81ef3f4ca..96812e503 100644 --- a/packages/core/src/generators/ast/generate.mjs +++ b/packages/core/src/generators/ast/generate.mjs @@ -9,8 +9,8 @@ import { parse as parseYaml } from 'yaml'; import getConfig from '#utils/configuration/index.mjs'; import { withExt } from '#utils/file.mjs'; +import { getProcessor } from '#utils/markdown/processor.mjs'; import { QUERIES } from '#utils/queries/index.mjs'; -import { getRemark as remark, getRemarkMdx } from '#utils/remark.mjs'; import { STABILITY_INDEX_URL } from './constants.mjs'; @@ -67,6 +67,8 @@ export async function processChunk(inputSlice, itemIndices) { // The path is the relative path minus the extension const relativePath = sep + withExt(relative(parent, path)); + const processor = getProcessor('ast', { mdx }); + let tree; if (mdx) { @@ -79,7 +81,7 @@ export async function processChunk(inputSlice, itemIndices) { '' ); - tree = getRemarkMdx().parse(source); + tree = processor.parse(source); if (frontmatter) { tree.children.unshift({ @@ -93,9 +95,11 @@ export async function processChunk(inputSlice, itemIndices) { (_, yaml) => `` ); - tree = remark().parse(value); + tree = processor.parse(value); } + tree = await processor.run(tree, { path }); + results.push({ tree, path: relativePath, mdx }); } diff --git a/packages/core/src/generators/ast/index.mjs b/packages/core/src/generators/ast/index.mjs index b9c51974c..7ba9eafb8 100644 --- a/packages/core/src/generators/ast/index.mjs +++ b/packages/core/src/generators/ast/index.mjs @@ -15,6 +15,15 @@ export default { hasParallelProcessor: true, + markdown: { + remarkPlugins: [ + 'remark-parse', + '#plugins/type-annotations/remark.mjs', + 'remark-gfm', + '...', + ], + }, + generate, processChunk, }; diff --git a/packages/core/src/generators/json/__tests__/generate.test.mjs b/packages/core/src/generators/json/__tests__/generate.test.mjs index e3f7cfd82..73a5a6b7c 100644 --- a/packages/core/src/generators/json/__tests__/generate.test.mjs +++ b/packages/core/src/generators/json/__tests__/generate.test.mjs @@ -7,13 +7,25 @@ import { fileURLToPath } from 'node:url'; import Ajv from 'ajv'; import { globSync } from 'tinyglobby'; +import { loadGenerator } from '#generators/loader.mjs'; import { parseApiDoc } from '#generators/metadata/utils/parse.mjs'; +import { loadMarkdownPlugins } from '#utils/markdown/plugins.mjs'; +import { getProcessor } from '#utils/markdown/processor.mjs'; import { QUERIES } from '#utils/queries/index.mjs'; -import { getRemark } from '#utils/remark.mjs'; import schema from '../schema.json' with { type: 'json' }; import { buildDocument } from '../utils/document.mjs'; +// The documents are parsed, split into entries, and serialised with the +// pipelines of the generators doing so +for (const generator of ['ast', 'metadata', 'json']) { + await loadMarkdownPlugins( + await loadGenerator( + import.meta.resolve(`#generators/${generator}/index.mjs`) + ) + ); +} + const fixtures = new URL('./fixtures/', import.meta.url); const typeMap = { @@ -42,7 +54,7 @@ const buildFixture = async name => { const path = `/${basename(name, '.md')}`; const entries = parseApiDoc( - { path, tree: getRemark().parse(source) }, + { path, tree: getProcessor('ast').parse(source) }, typeMap ); diff --git a/packages/core/src/generators/json/index.mjs b/packages/core/src/generators/json/index.mjs index aa4266ac9..129869796 100644 --- a/packages/core/src/generators/json/index.mjs +++ b/packages/core/src/generators/json/index.mjs @@ -22,6 +22,15 @@ export default { hasParallelProcessor: true, + markdown: { + remarkPlugins: [ + '#plugins/type-annotations/remark.mjs', + 'remark-gfm', + 'remark-stringify', + '...', + ], + }, + generate, processChunk, }; diff --git a/packages/core/src/generators/json/utils/__tests__/entry.test.mjs b/packages/core/src/generators/json/utils/__tests__/entry.test.mjs index 1e4d26a81..918b1400a 100644 --- a/packages/core/src/generators/json/utils/__tests__/entry.test.mjs +++ b/packages/core/src/generators/json/utils/__tests__/entry.test.mjs @@ -3,8 +3,16 @@ import { describe, it } from 'node:test'; import { u } from 'unist-builder'; +import { loadGenerator } from '#generators/loader.mjs'; +import { loadMarkdownPlugins } from '#utils/markdown/plugins.mjs'; + import { buildEntry, entryBody, takeTypedItems } from '../entry.mjs'; +// Markdown is serialised with the pipeline of `json` +await loadMarkdownPlugins( + await loadGenerator(import.meta.resolve('../../index.mjs')) +); + const code = value => u('inlineCode', value); const text = value => u('text', value); const type = value => u('typeAnnotation', { value }); diff --git a/packages/core/src/generators/json/utils/__tests__/lifecycle.test.mjs b/packages/core/src/generators/json/utils/__tests__/lifecycle.test.mjs index 84f0f27b4..ddd5e38e2 100644 --- a/packages/core/src/generators/json/utils/__tests__/lifecycle.test.mjs +++ b/packages/core/src/generators/json/utils/__tests__/lifecycle.test.mjs @@ -3,6 +3,9 @@ import { describe, it } from 'node:test'; import { u } from 'unist-builder'; +import { loadGenerator } from '#generators/loader.mjs'; +import { loadMarkdownPlugins } from '#utils/markdown/plugins.mjs'; + import { toChanges, toNumbers, @@ -10,6 +13,11 @@ import { toVersions, } from '../lifecycle.mjs'; +// Markdown is serialised with the pipeline of `json` +await loadMarkdownPlugins( + await loadGenerator(import.meta.resolve('../../index.mjs')) +); + describe('toVersions', () => { it('always yields an array of strings', () => { assert.deepEqual(toVersions(undefined), []); diff --git a/packages/core/src/generators/json/utils/__tests__/markdown.test.mjs b/packages/core/src/generators/json/utils/__tests__/markdown.test.mjs index 0ce56d8fb..93f9cf0a5 100644 --- a/packages/core/src/generators/json/utils/__tests__/markdown.test.mjs +++ b/packages/core/src/generators/json/utils/__tests__/markdown.test.mjs @@ -3,12 +3,20 @@ import { describe, it } from 'node:test'; import { u } from 'unist-builder'; +import { loadGenerator } from '#generators/loader.mjs'; +import { loadMarkdownPlugins } from '#utils/markdown/plugins.mjs'; + import { blocksToMarkdown, extractExamples, inlineToMarkdown, } from '../markdown.mjs'; +// Markdown is serialised with the pipeline of `json` +await loadMarkdownPlugins( + await loadGenerator(import.meta.resolve('../../index.mjs')) +); + describe('blocksToMarkdown', () => { it('serialises blocks, type annotations included', () => { const nodes = [ diff --git a/packages/core/src/generators/json/utils/__tests__/signature.test.mjs b/packages/core/src/generators/json/utils/__tests__/signature.test.mjs index f03e38ecb..1e5d4741e 100644 --- a/packages/core/src/generators/json/utils/__tests__/signature.test.mjs +++ b/packages/core/src/generators/json/utils/__tests__/signature.test.mjs @@ -3,6 +3,9 @@ import { describe, it } from 'node:test'; import { u } from 'unist-builder'; +import { loadGenerator } from '#generators/loader.mjs'; +import { loadMarkdownPlugins } from '#utils/markdown/plugins.mjs'; + import { buildEventParameters, buildExtends, @@ -10,6 +13,11 @@ import { buildSignature, } from '../signature.mjs'; +// Markdown is serialised with the pipeline of `json` +await loadMarkdownPlugins( + await loadGenerator(import.meta.resolve('../../index.mjs')) +); + const code = value => u('inlineCode', value); const text = value => u('text', value); const type = (value, links = []) => diff --git a/packages/core/src/generators/json/utils/markdown.mjs b/packages/core/src/generators/json/utils/markdown.mjs index 25775b7c2..13d02ed19 100644 --- a/packages/core/src/generators/json/utils/markdown.mjs +++ b/packages/core/src/generators/json/utils/markdown.mjs @@ -3,7 +3,7 @@ import { u as createTree } from 'unist-builder'; import { visit } from 'unist-util-visit'; -import { getRemark, getRemarkMdx } from '#utils/remark.mjs'; +import { getProcessor } from '#utils/markdown/processor.mjs'; import { DISPLAY_NAME } from '../constants.mjs'; @@ -19,7 +19,7 @@ export const blocksToMarkdown = (nodes, mdx = false) => { return ''; } - const processor = mdx ? getRemarkMdx() : getRemark(); + const processor = getProcessor('json', { mdx }); return processor.stringify(createTree('root', nodes)).trim(); }; diff --git a/packages/core/src/generators/metadata/index.mjs b/packages/core/src/generators/metadata/index.mjs index 1c5ff56fb..fcdd965b7 100644 --- a/packages/core/src/generators/metadata/index.mjs +++ b/packages/core/src/generators/metadata/index.mjs @@ -16,6 +16,15 @@ export default { hasParallelProcessor: true, + markdown: { + remarkPlugins: [ + 'remark-parse', + '#plugins/type-annotations/remark.mjs', + 'remark-gfm', + '...', + ], + }, + generate, processChunk, }; diff --git a/packages/core/src/generators/metadata/utils/parse.mjs b/packages/core/src/generators/metadata/utils/parse.mjs index 637023376..c3eb41fb5 100644 --- a/packages/core/src/generators/metadata/utils/parse.mjs +++ b/packages/core/src/generators/metadata/utils/parse.mjs @@ -14,7 +14,6 @@ import { IGNORE_STABILITY_STEMS, } from '#generators/metadata/constants.mjs'; import { UNIST } from '#utils/queries/index.mjs'; -import { getRemark as remark } from '#utils/remark.mjs'; import { relative } from '#utils/url.mjs'; import { resolveTypeAnnotations } from './resolveTypes.mjs'; @@ -147,9 +146,8 @@ export const parseApiDoc = ({ path, tree, mdx = false }, typeMap) => { // Remove processed YAML nodes from the content remove(subTree, [UNIST.isYamlNode]); - // Apply AST transformations - const parsedSubTree = remark().runSync(subTree); - metadata.content = parsedSubTree; + // The remark plugins already ran on the whole document, in `ast` + metadata.content = subTree; // Add to collection metadataCollection.push(metadata); diff --git a/packages/core/src/generators/metadata/utils/visitors.mjs b/packages/core/src/generators/metadata/utils/visitors.mjs index 6fa2d66d4..fb3fc6c2b 100644 --- a/packages/core/src/generators/metadata/utils/visitors.mjs +++ b/packages/core/src/generators/metadata/utils/visitors.mjs @@ -2,8 +2,8 @@ import { SKIP } from 'unist-util-visit'; +import { getProcessor } from '#utils/markdown/processor.mjs'; import { QUERIES } from '#utils/queries/index.mjs'; -import { getRemark as remark } from '#utils/remark.mjs'; import { transformNodesToString } from '#utils/unist.mjs'; import { transformUnixManualToLink } from './transformers.mjs'; @@ -41,7 +41,7 @@ const updateReferences = (query, transformer, node, parent) => { // and adding those nodes to the parent. const { children: [newNode], - } = remark().parse(replacedTypes); + } = getProcessor('metadata').parse(replacedTypes); // Find the index of the original node in the parent const index = parent.children.indexOf(node); From 1cb61531abf00262397b34b12c01e179b217e799 Mon Sep 17 00:00:00 2001 From: Claudio Wunder Date: Thu, 8 Oct 2026 13:00:33 +0200 Subject: [PATCH 06/10] feat(react): use the Markdown pipeline in `jsx-ast` The pages run the configured remark, rehype, and recma plugins, the rehype ones before code is highlighted. Types and signatures are highlighted by the Shiki plugin of the pipeline. Assisted-by: Claude Opus 5.5 --- .../type-annotations/__tests__/hast.test.mjs | 10 +- .../plugins/type-annotations/highlighter.mjs | 31 +++--- .../src/html/__tests__/generate.test.mjs | 7 ++ .../src/html/utils/__tests__/config.test.mjs | 12 +++ packages/react/src/html/utils/config.mjs | 6 +- .../src/jsx-ast/__tests__/generate.test.mjs | 7 ++ .../src/jsx-ast/__tests__/markdown.test.mjs | 101 ++++++++++++++++++ packages/react/src/jsx-ast/index.mjs | 35 ++++++ packages/react/src/jsx-ast/plugins/raw.mjs | 52 +++++++++ .../utils/__tests__/buildContent.test.mjs | 7 ++ .../jsx-ast/utils/__tests__/remark.test.mjs | 53 --------- .../jsx-ast/utils/__tests__/types.test.mjs | 12 +-- .../react/src/jsx-ast/utils/buildContent.mjs | 4 +- packages/react/src/jsx-ast/utils/remark.mjs | 80 -------------- packages/react/src/jsx-ast/utils/render.mjs | 7 +- .../react/src/jsx-ast/utils/signature.mjs | 3 +- 16 files changed, 258 insertions(+), 169 deletions(-) create mode 100644 packages/react/src/jsx-ast/__tests__/markdown.test.mjs create mode 100644 packages/react/src/jsx-ast/plugins/raw.mjs delete mode 100644 packages/react/src/jsx-ast/utils/__tests__/remark.test.mjs delete mode 100644 packages/react/src/jsx-ast/utils/remark.mjs diff --git a/packages/core/src/plugins/type-annotations/__tests__/hast.test.mjs b/packages/core/src/plugins/type-annotations/__tests__/hast.test.mjs index f2ce3971e..9aeb3c791 100644 --- a/packages/core/src/plugins/type-annotations/__tests__/hast.test.mjs +++ b/packages/core/src/plugins/type-annotations/__tests__/hast.test.mjs @@ -4,8 +4,16 @@ import { describe, it } from 'node:test'; import { toHtml } from 'hast-util-to-html'; import { toString } from 'hast-util-to-string'; +import { createHighlighter } from '#plugins/shiki/highlighter.mjs'; + import { typeAnnotationToHast } from '../hast.mjs'; -import { typeAnnotationToHighlightedHast } from '../highlighter.mjs'; +import { createTypeAnnotationHandler } from '../highlighter.mjs'; + +const highlighter = await createHighlighter(); + +const typeAnnotationToHighlightedHast = createTypeAnnotationHandler( + () => highlighter +); // A minimal mdast-util-to-hast state — the handlers only use patch/applyData const state = { patch: () => {}, applyData: (_, result) => result }; diff --git a/packages/core/src/plugins/type-annotations/highlighter.mjs b/packages/core/src/plugins/type-annotations/highlighter.mjs index bca00bcee..f26f9b25c 100644 --- a/packages/core/src/plugins/type-annotations/highlighter.mjs +++ b/packages/core/src/plugins/type-annotations/highlighter.mjs @@ -1,37 +1,32 @@ 'use strict'; -import { highlighter } from '#utils/highlighter.mjs'; - import { typeAnnotationToHast } from './hast.mjs'; -// Kept apart from `./hast.mjs` on purpose: importing this module loads Shiki -// (every grammar plus the regex engine), which only the pipelines that -// highlight should pay for. The `ast` and `metadata` stages never do. -const [lightTheme, darkTheme] = highlighter.shiki.getLoadedThemes(); - /** - * Syntax-highlighted mdast→hast handler for `typeAnnotation` nodes, used by - * the web (JSX) pipeline. The whole type is highlighted as one inline - * fragment, and each resolved identifier's exact character range is wrapped - * in an `` via Shiki decorations. Values that are not TypeScript (display - * names such as `HTTP/2 Headers Object`) are highlighted as plain text, so - * their prose is not coloured as operators and numeric literals. + * Creates the syntax-highlighted mdast→hast handler for `typeAnnotation` + * nodes, used by the web (JSX) pipeline. The whole type is highlighted as one + * inline fragment, and each resolved identifier's exact character range is + * wrapped in an `` via Shiki decorations. Values that are not TypeScript + * (display names such as `HTTP/2 Headers Object`) are highlighted as plain + * text, so their prose is not coloured as operators and numeric literals. * * Falls back to the minimal handler when the type failed to parse or nothing * resolved (no point paying for highlighting then). * - * @param {import('mdast-util-to-hast').State} state - * @param {import('mdast').Node} node - * @returns {import('hast').Element} + * @param {() => import('#plugins/shiki/highlighter.mjs').SyntaxHighlighter} getHighlighter - Gives the highlighter, once a type is highlighted + * @returns {(state: import('mdast-util-to-hast').State, node: import('mdast').Node) => import('hast').Element} */ -export const typeAnnotationToHighlightedHast = (state, node) => { +export const createTypeAnnotationHandler = getHighlighter => (state, node) => { const links = node.data?.links ?? []; if (node.data?.parseError || links.length === 0) { return typeAnnotationToHast(state, node); } - const root = highlighter.shiki.codeToHast(node.value, { + const { shiki } = getHighlighter(); + const [lightTheme, darkTheme] = shiki.getLoadedThemes(); + + const root = shiki.codeToHast(node.value, { lang: node.data?.typescript ? 'typescript' : 'text', themes: { light: lightTheme, dark: darkTheme }, decorations: links.map(({ start, end, href }) => ({ diff --git a/packages/react/src/html/__tests__/generate.test.mjs b/packages/react/src/html/__tests__/generate.test.mjs index cc72980bb..814889c7f 100644 --- a/packages/react/src/html/__tests__/generate.test.mjs +++ b/packages/react/src/html/__tests__/generate.test.mjs @@ -5,7 +5,9 @@ import { join } from 'node:path'; import { describe, it } from 'node:test'; import { pathToFileURL } from 'node:url'; +import { loadGenerator } from '@doc-kit/core/generators/loader.mjs'; import { setConfig } from '@doc-kit/core/utils/configuration/index.mjs'; +import { loadMarkdownPlugins } from '@doc-kit/core/utils/markdown/plugins.mjs'; import { jsx, toJs } from 'estree-util-to-js'; import buildContent from '../../jsx-ast/utils/buildContent.mjs'; @@ -14,6 +16,11 @@ import { generate as chunk } from '../../section-pages/generate.mjs'; import { compile, createViteBundler } from '../bundlers/vite.mjs'; import { generate } from '../generate.mjs'; +// Pages are rendered with the pipeline of `jsx-ast` +await loadMarkdownPlugins( + await loadGenerator(import.meta.resolve('../../jsx-ast/index.mjs')) +); + /** * Converts a page's JSX AST into the `{ data, headings, readingTime, content }` * shape `html` consumes, mirroring the conversion the jsx-ast worker performs. diff --git a/packages/react/src/html/utils/__tests__/config.test.mjs b/packages/react/src/html/utils/__tests__/config.test.mjs index 016b4cf29..94e2e0c38 100644 --- a/packages/react/src/html/utils/__tests__/config.test.mjs +++ b/packages/react/src/html/utils/__tests__/config.test.mjs @@ -2,6 +2,7 @@ import assert from 'node:assert/strict'; import { describe, it, mock } from 'node:test'; import { setConfig } from '@doc-kit/core/utils/configuration/index.mjs'; +import { loadMarkdownPlugins } from '@doc-kit/core/utils/markdown/plugins.mjs'; import { SemVer } from 'semver'; mock.module('@node-core/rehype-shiki', { @@ -11,6 +12,17 @@ mock.module('@node-core/rehype-shiki', { { name: 'typescript', aliases: ['ts'], displayName: 'TypeScript' }, { name: 'python', displayName: 'Python' }, ], + default: async () => ({}), + }, +}); + +// The site's code is highlighted by the Shiki plugin of `jsx-ast` +await loadMarkdownPlugins({ + name: 'jsx-ast', + markdown: { + rehypePlugins: [ + import.meta.resolve('@doc-kit/core/plugins/shiki/rehype.mjs'), + ], }, }); diff --git a/packages/react/src/html/utils/config.mjs b/packages/react/src/html/utils/config.mjs index 8a83dfada..512cbdbf2 100644 --- a/packages/react/src/html/utils/config.mjs +++ b/packages/react/src/html/utils/config.mjs @@ -1,5 +1,6 @@ 'use strict'; +import { getHighlighter } from '@doc-kit/core/plugins/shiki/highlighter.mjs'; import getConfig from '@doc-kit/core/utils/configuration/index.mjs'; import { populate } from '@doc-kit/core/utils/configuration/templates.mjs'; import { @@ -8,7 +9,6 @@ import { } from '@doc-kit/core/utils/generators.mjs'; import { parseInline, renderAsHTML } from '@doc-kit/core/utils/inline.mjs'; import { omitKeys } from '@doc-kit/core/utils/misc.mjs'; -import { LANGS } from '@node-core/rehype-shiki'; import { getSortedHeadNodes } from '../../jsx-ast/utils/getSortedHeadNodes.mjs'; @@ -142,9 +142,11 @@ export function buildDocumentationIndex(input) { * @returns {Array<[string[], string]>} */ export function buildLanguageDisplayNameMap() { + const { langs } = getHighlighter('jsx-ast'); + return [ ...new Map( - LANGS.map(({ name, aliases = [], displayName }) => [ + langs.map(({ name, aliases = [], displayName }) => [ name, [[...aliases, name], displayName], ]) diff --git a/packages/react/src/jsx-ast/__tests__/generate.test.mjs b/packages/react/src/jsx-ast/__tests__/generate.test.mjs index 40466dc41..c179fb962 100644 --- a/packages/react/src/jsx-ast/__tests__/generate.test.mjs +++ b/packages/react/src/jsx-ast/__tests__/generate.test.mjs @@ -1,12 +1,19 @@ import assert from 'node:assert/strict'; import { describe, it } from 'node:test'; +import { loadGenerator } from '@doc-kit/core/generators/loader.mjs'; import getConfig, { setConfig, } from '@doc-kit/core/utils/configuration/index.mjs'; +import { loadMarkdownPlugins } from '@doc-kit/core/utils/markdown/plugins.mjs'; import { generate, processChunk } from '../generate.mjs'; +// Pages are rendered with the pipeline of `jsx-ast` +await loadMarkdownPlugins( + await loadGenerator(import.meta.resolve('../index.mjs')) +); + const createEntry = (api, name, { stabilityIndex = '2' } = {}) => { const heading = { type: 'heading', diff --git a/packages/react/src/jsx-ast/__tests__/markdown.test.mjs b/packages/react/src/jsx-ast/__tests__/markdown.test.mjs new file mode 100644 index 000000000..48942b660 --- /dev/null +++ b/packages/react/src/jsx-ast/__tests__/markdown.test.mjs @@ -0,0 +1,101 @@ +import assert from 'node:assert/strict'; +import { mkdtempSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { describe, it } from 'node:test'; +import { pathToFileURL } from 'node:url'; + +import { loadGenerator } from '@doc-kit/core/generators/loader.mjs'; +import { loadMarkdownPlugins } from '@doc-kit/core/utils/markdown/plugins.mjs'; +import { getProcessor } from '@doc-kit/core/utils/markdown/processor.mjs'; +import dedent from 'dedent'; +import { visit } from 'unist-util-visit'; + +const jsxAst = await loadGenerator(import.meta.resolve('../index.mjs')); + +await loadMarkdownPlugins(jsxAst); + +const getCodeTabsAttributes = tree => { + const attributes = []; + + visit(tree.body[0].expression, 'JSXElement', node => { + if (node.openingElement.name?.name === 'CodeTabs') { + attributes.push( + Object.fromEntries( + node.openingElement.attributes.map(attribute => [ + attribute.name.name, + attribute.value?.value, + ]) + ) + ); + } + }); + + return attributes; +}; + +describe('the Markdown pipeline of jsx-ast', () => { + it('preserves code tab display names when raw HTML is enabled', async () => { + const processor = getProcessor('jsx-ast'); + const tree = await processor.run( + processor.parse(dedent` +
raw html
+ + \`\`\`cjs displayName="main.js" + console.log(1); + \`\`\` + + \`\`\`cjs displayName="main.test.js" + console.log(2); + \`\`\` + `) + ); + + assert.deepEqual(getCodeTabsAttributes(tree), [ + { + languages: 'cjs|cjs', + displayNames: 'main.js|main.test.js', + defaultTab: '0', + }, + ]); + }); + + it('runs the configured rehype plugins before code is highlighted', async t => { + // Records the classes of the code blocks it gets + const recorder = join(mkdtempSync(join(tmpdir(), 'jsx-ast-')), 'rec.mjs'); + + writeFileSync( + recorder, + `export default () => tree => { + for (const node of tree.children) { + if (node.tagName === 'pre') { + globalThis.recorded.push(node.children[0].properties.className); + } + } + };` + ); + + globalThis.recorded = []; + + await loadMarkdownPlugins(jsxAst, { + rehypePlugins: [pathToFileURL(recorder).href], + }); + + t.after(() => loadMarkdownPlugins(jsxAst)); + + const markdown = '```js\nconst a = 1;\n```\n'; + + const processor = getProcessor('jsx-ast'); + + await processor.run(processor.parse(markdown)); + + assert.deepStrictEqual(globalThis.recorded, [['language-js']]); + + // The fragments doc-kit renders itself run without them + const own = getProcessor('jsx-ast', { configured: false }); + + own.runSync(own.parse(markdown)); + + assert.deepStrictEqual(globalThis.recorded, [['language-js']]); + }); +}); diff --git a/packages/react/src/jsx-ast/index.mjs b/packages/react/src/jsx-ast/index.mjs index 191e1934f..b1dd1aeb5 100644 --- a/packages/react/src/jsx-ast/index.mjs +++ b/packages/react/src/jsx-ast/index.mjs @@ -1,5 +1,9 @@ 'use strict'; +import { getHighlighter } from '@doc-kit/core/plugins/shiki/highlighter.mjs'; +import { createTypeAnnotationHandler } from '@doc-kit/core/plugins/type-annotations/highlighter.mjs'; + +import { AST_NODE_TYPES } from './constants.mjs'; import { generate, processChunk } from './generate.mjs'; /** @@ -22,6 +26,37 @@ export default { hasParallelProcessor: true, + markdown: { + // The configured remark plugins run before the alerts, so they can make + // alerts too + remarkPlugins: ['remark-parse', '...', './plugins/alerts.mjs'], + rehypePlugins: [ + [ + 'remark-rehype', + { + // We make Rehype ignore existing HTML nodes, and JSX nodes as these + // are nodes we manually created during the generation process. We + // also allow dangerous HTML to be passed through, since we have HTML + // within our Markdown and we trust the sources of the Markdown files + allowDangerousHtml: true, + passThrough: ['element', ...Object.values(AST_NODE_TYPES.MDX)], + // Types are highlighted, with their links embedded + handlers: { + typeAnnotation: createTypeAnnotationHandler(() => + getHighlighter('jsx-ast') + ), + }, + }, + ], + './plugins/raw.mjs', + // The configured rehype plugins run before code blocks are highlighted + '...', + '@doc-kit/core/plugins/shiki/rehype.mjs', + './plugins/transformer.mjs', + ], + recmaPlugins: ['rehype-recma', 'recma-jsx', '...', 'recma-stringify'], + }, + generate, processChunk, }; diff --git a/packages/react/src/jsx-ast/plugins/raw.mjs b/packages/react/src/jsx-ast/plugins/raw.mjs new file mode 100644 index 000000000..88fb52c56 --- /dev/null +++ b/packages/react/src/jsx-ast/plugins/raw.mjs @@ -0,0 +1,52 @@ +'use strict'; + +import rehypeRaw from 'rehype-raw'; +import { visit } from 'unist-util-visit'; + +import { AST_NODE_TYPES } from '../constants.mjs'; + +const codeMetaProperty = 'codeMeta'; + +/** + * Stores fenced code metadata on properties before rehypeRaw reparses the tree. + */ +const preserveCodeMeta = () => tree => { + visit(tree, 'element', node => { + const meta = node.data?.meta; + + if (node.tagName === 'code' && typeof meta === 'string') { + node.properties ||= {}; + node.properties[codeMetaProperty] = meta; + } + }); +}; + +/** + * Restores fenced code metadata so the Shiki plugin can read displayName. + */ +const restoreCodeMeta = () => tree => { + visit(tree, 'element', node => { + const meta = node.properties?.[codeMetaProperty]; + + if (node.tagName === 'code' && typeof meta === 'string') { + node.data = { ...node.data, meta }; + delete node.properties[codeMetaProperty]; + } + }); +}; + +/** + * Converts any `raw` HTML in the Markdown to AST, in order for Recma to + * understand it, keeping the metadata of fenced code. + * + * @type {import('unified').PluggableList} + */ +export default [ + preserveCodeMeta, + // HTML and JSX nodes pass through, as we created them during the generation + [ + rehypeRaw, + { passThrough: ['element', ...Object.values(AST_NODE_TYPES.MDX)] }, + ], + restoreCodeMeta, +]; diff --git a/packages/react/src/jsx-ast/utils/__tests__/buildContent.test.mjs b/packages/react/src/jsx-ast/utils/__tests__/buildContent.test.mjs index 1be679643..2259b1918 100644 --- a/packages/react/src/jsx-ast/utils/__tests__/buildContent.test.mjs +++ b/packages/react/src/jsx-ast/utils/__tests__/buildContent.test.mjs @@ -1,10 +1,17 @@ import assert from 'node:assert/strict'; import { describe, it } from 'node:test'; +import { loadGenerator } from '@doc-kit/core/generators/loader.mjs'; import { setConfig } from '@doc-kit/core/utils/configuration/index.mjs'; +import { loadMarkdownPlugins } from '@doc-kit/core/utils/markdown/plugins.mjs'; import { transformHeadingNode, gatherChangeEntries } from '../buildContent.mjs'; +// Pages are rendered with the pipeline of `jsx-ast` +await loadMarkdownPlugins( + await loadGenerator(import.meta.resolve('../../index.mjs')) +); + const heading = { type: 'heading', depth: 3, diff --git a/packages/react/src/jsx-ast/utils/__tests__/remark.test.mjs b/packages/react/src/jsx-ast/utils/__tests__/remark.test.mjs deleted file mode 100644 index afea8e9df..000000000 --- a/packages/react/src/jsx-ast/utils/__tests__/remark.test.mjs +++ /dev/null @@ -1,53 +0,0 @@ -import assert from 'node:assert/strict'; -import { describe, it } from 'node:test'; - -import dedent from 'dedent'; -import { visit } from 'unist-util-visit'; - -import { getRemarkRecma } from '../remark.mjs'; - -const getCodeTabsAttributes = tree => { - const attributes = []; - - visit(tree.body[0].expression, 'JSXElement', node => { - if (node.openingElement.name?.name === 'CodeTabs') { - attributes.push( - Object.fromEntries( - node.openingElement.attributes.map(attribute => [ - attribute.name.name, - attribute.value?.value, - ]) - ) - ); - } - }); - - return attributes; -}; - -describe('getRemarkRecma', () => { - it('preserves code tab display names when raw HTML is enabled', async () => { - const processor = getRemarkRecma(); - const tree = await processor.run( - processor.parse(dedent` -
raw html
- - \`\`\`cjs displayName="main.js" - console.log(1); - \`\`\` - - \`\`\`cjs displayName="main.test.js" - console.log(2); - \`\`\` - `) - ); - - assert.deepEqual(getCodeTabsAttributes(tree), [ - { - languages: 'cjs|cjs', - displayNames: 'main.js|main.test.js', - defaultTab: '0', - }, - ]); - }); -}); diff --git a/packages/react/src/jsx-ast/utils/__tests__/types.test.mjs b/packages/react/src/jsx-ast/utils/__tests__/types.test.mjs index 1bafa0cd9..e26b79a96 100644 --- a/packages/react/src/jsx-ast/utils/__tests__/types.test.mjs +++ b/packages/react/src/jsx-ast/utils/__tests__/types.test.mjs @@ -1,15 +1,9 @@ import assert from 'node:assert/strict'; import { describe, it, mock } from 'node:test'; -// Mock remark -mock.module('../remark.mjs', { - exports: { - getRemarkRecma: () => ({ - runSync: () => ({ - body: [{ expression: 'mock-expression' }], - }), - }), - }, +// Mock the rendering of inline nodes +mock.module('../render.mjs', { + exports: { renderAsJSX: () => 'mock-expression' }, }); const { parseListIntoProperties } = await import('../types.mjs'); diff --git a/packages/react/src/jsx-ast/utils/buildContent.mjs b/packages/react/src/jsx-ast/utils/buildContent.mjs index 3a86d8aad..82688fa26 100644 --- a/packages/react/src/jsx-ast/utils/buildContent.mjs +++ b/packages/react/src/jsx-ast/utils/buildContent.mjs @@ -7,6 +7,7 @@ import { populate, } from '@doc-kit/core/utils/configuration/templates.mjs'; import { parseInline } from '@doc-kit/core/utils/inline.mjs'; +import { getProcessor } from '@doc-kit/core/utils/markdown/processor.mjs'; import { annotateOverloads } from '@doc-kit/core/utils/overloads.mjs'; import { splitTypedItems, UNIST } from '@doc-kit/core/utils/queries/index.mjs'; import { removeStabilityPrefix } from '@doc-kit/core/utils/stability.mjs'; @@ -28,7 +29,6 @@ import { } from '../constants.mjs'; import { createJSXElement } from './ast.mjs'; import { extractHeadings, extractTextContent } from './buildBarProps.mjs'; -import { getRemarkRecma as remark } from './remark.mjs'; import { renderAsJSX } from './render.mjs'; import { insertSignatureCodeBlock, @@ -360,7 +360,7 @@ const buildContent = async (metadataEntries, head) => { await createDocumentContent(metadataEntries); // Run remark processor to transform AST (parse markdown, plugins, etc.) - const ast = await remark().run(root); + const ast = await getProcessor('jsx-ast').run(root); // The fragment is the expression in the Program's first body node return { data: head, headings, readingTime, content: ast.body[0].expression }; diff --git a/packages/react/src/jsx-ast/utils/remark.mjs b/packages/react/src/jsx-ast/utils/remark.mjs deleted file mode 100644 index 6e6db0b5d..000000000 --- a/packages/react/src/jsx-ast/utils/remark.mjs +++ /dev/null @@ -1,80 +0,0 @@ -'use strict'; - -import { typeAnnotationToHighlightedHast } from '@doc-kit/core/plugins/type-annotations/highlighter.mjs'; -import { highlighter } from '@doc-kit/core/utils/highlighter.mjs'; -import { lazy } from '@doc-kit/core/utils/misc.mjs'; -import rehypeShikiji from '@node-core/rehype-shiki/plugin'; -import recmaJsx from 'recma-jsx'; -import recmaStringify from 'recma-stringify'; -import rehypeRaw from 'rehype-raw'; -import rehypeRecma from 'rehype-recma'; -import remarkParse from 'remark-parse'; -import remarkRehype from 'remark-rehype'; -import { unified } from 'unified'; -import { visit } from 'unist-util-visit'; - -import { AST_NODE_TYPES } from '../constants.mjs'; -import transformAlerts from '../plugins/alerts.mjs'; -import transformElements from '../plugins/transformer.mjs'; - -const passThrough = ['element', ...Object.values(AST_NODE_TYPES.MDX)]; -const codeMetaProperty = 'codeMeta'; - -/** - * Stores fenced code metadata on properties before rehypeRaw reparses the tree. - */ -const preserveCodeMeta = () => tree => { - visit(tree, 'element', node => { - const meta = node.data?.meta; - - if (node.tagName === 'code' && typeof meta === 'string') { - node.properties ||= {}; - node.properties[codeMetaProperty] = meta; - } - }); -}; - -/** - * Restores fenced code metadata so the Shiki plugin can read displayName. - */ -const restoreCodeMeta = () => tree => { - visit(tree, 'element', node => { - const meta = node.properties?.[codeMetaProperty]; - - if (node.tagName === 'code' && typeof meta === 'string') { - node.data = { ...node.data, meta }; - delete node.properties[codeMetaProperty]; - } - }); -}; - -const singletonShiki = await rehypeShikiji({ highlighter }); - -/** - * Retrieves an instance of Remark configured to output JSX code. - * including parsing Code Boxes with syntax highlighting - */ -export const getRemarkRecma = lazy(() => - unified() - .use(remarkParse) - .use(transformAlerts) - // We make Rehype ignore existing HTML nodes, and JSX nodes - // as these are nodes we manually created during the generation process - // We also allow dangerous HTML to be passed through, since we have HTML within our Markdown - // and we trust the sources of the Markdown files - .use(remarkRehype, { - allowDangerousHtml: true, - passThrough, - // The web pipeline gets Shiki-highlighted types with embedded links - handlers: { typeAnnotation: typeAnnotationToHighlightedHast }, - }) - .use(preserveCodeMeta) - // Any `raw` HTML in the markdown must be converted to AST in order for Recma to understand it - .use(rehypeRaw, { passThrough }) - .use(restoreCodeMeta) - .use(() => singletonShiki) - .use(transformElements) - .use(rehypeRecma) - .use(recmaJsx) - .use(recmaStringify) -); diff --git a/packages/react/src/jsx-ast/utils/render.mjs b/packages/react/src/jsx-ast/utils/render.mjs index bab668ea6..883f8e6e4 100644 --- a/packages/react/src/jsx-ast/utils/render.mjs +++ b/packages/react/src/jsx-ast/utils/render.mjs @@ -1,17 +1,18 @@ 'use strict'; +import { getProcessor } from '@doc-kit/core/utils/markdown/processor.mjs'; import { u as createTree } from 'unist-builder'; import { createJSXElement } from './ast.mjs'; -import { getRemarkRecma as remark } from './remark.mjs'; /** - * Renders inline nodes as a JSX fragment + * Renders inline nodes as a JSX fragment. It runs synchronously, so without + * the configured plugins, which may be asynchronous. * * @param {Array} nodes - The nodes to render. * @returns {import('estree-jsx').JSXFragment} The rendered nodes. */ export const renderAsJSX = nodes => - remark().runSync( + getProcessor('jsx-ast', { configured: false }).runSync( createTree('root', [createJSXElement(null, { children: nodes })]) ).body[0].expression; diff --git a/packages/react/src/jsx-ast/utils/signature.mjs b/packages/react/src/jsx-ast/utils/signature.mjs index d4efdee4a..3edf90156 100644 --- a/packages/react/src/jsx-ast/utils/signature.mjs +++ b/packages/react/src/jsx-ast/utils/signature.mjs @@ -1,4 +1,4 @@ -import { highlighter } from '@doc-kit/core/utils/highlighter.mjs'; +import { getHighlighter } from '@doc-kit/core/plugins/shiki/highlighter.mjs'; import { UNIST } from '@doc-kit/core/utils/queries/index.mjs'; import { parseListItem } from '@doc-kit/core/utils/signature/parseList.mjs'; import parseSignature from '@doc-kit/core/utils/signature/parseSignature.mjs'; @@ -65,6 +65,7 @@ export const generateSignature = ( */ export const createSignatureCodeBlock = (functionName, signature, heading) => { const sig = generateSignature(functionName, signature, heading); + const highlighter = getHighlighter('jsx-ast'); const highlighted = highlighter.highlightToHast(sig, 'typescript'); return createElement('div', { class: 'signature' }, [highlighted]); From 3839ea9e2f5e14da781f4922cbad3025448471e5 Mon Sep 17 00:00:00 2001 From: Claudio Wunder Date: Thu, 8 Oct 2026 13:00:51 +0200 Subject: [PATCH 07/10] refactor(legacy): use Markdown pipelines in the legacy generators The legacy generators declare their pipelines, taking no configured plugins, so their output stays the same. The processors of `@doc-kit/core/utils/remark.mjs` go, and the Shiki step of `utils/highlighter.mjs` moves to `legacy-html`, its last user. Assisted-by: Claude Opus 5.5 --- packages/core/package.json | 1 - packages/core/src/utils/remark-shiki.mjs | 29 ------ packages/core/src/utils/remark.mjs | 91 ------------------- packages/node-legacy/package.json | 3 + .../src/legacy-html-all/generate.mjs | 4 +- .../node-legacy/src/legacy-html/generate.mjs | 6 +- .../node-legacy/src/legacy-html/index.mjs | 12 +++ .../src/legacy-html/plugins/shiki.mjs} | 21 +---- .../utils/__tests__/buildContent.test.mjs | 7 ++ .../src/legacy-html/utils/buildContent.mjs | 8 +- .../legacy-json/__tests__/markdown.test.mjs} | 12 ++- .../node-legacy/src/legacy-json/index.mjs | 10 ++ .../src/legacy-json/utils/buildSection.mjs | 6 +- .../node-legacy/src/utils/rehypeOptions.mjs | 49 ++++++++++ pnpm-lock.yaml | 12 ++- 15 files changed, 114 insertions(+), 157 deletions(-) delete mode 100644 packages/core/src/utils/remark-shiki.mjs delete mode 100644 packages/core/src/utils/remark.mjs rename packages/{core/src/utils/highlighter.mjs => node-legacy/src/legacy-html/plugins/shiki.mjs} (89%) rename packages/{core/src/utils/__tests__/remark.test.mjs => node-legacy/src/legacy-json/__tests__/markdown.test.mjs} (69%) create mode 100644 packages/node-legacy/src/utils/rehypeOptions.mjs diff --git a/packages/core/package.json b/packages/core/package.json index 4bbf018ae..2f2035605 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -61,7 +61,6 @@ "dedent": "^1.7.2", "github-slugger": "^2.0.0", "glob-parent": "^6.0.2", - "hastscript": "^9.0.1", "mdast-util-slice-markdown": "^2.0.1", "mdast-util-to-string": "^4.0.0", "piscina": "^5.3.2", diff --git a/packages/core/src/utils/remark-shiki.mjs b/packages/core/src/utils/remark-shiki.mjs deleted file mode 100644 index 6fe234ea2..000000000 --- a/packages/core/src/utils/remark-shiki.mjs +++ /dev/null @@ -1,29 +0,0 @@ -'use strict'; - -import rehypeStringify from 'rehype-stringify'; -import remarkParse from 'remark-parse'; -import remarkRehype from 'remark-rehype'; -import { unified } from 'unified'; - -import syntaxHighlighter from './highlighter.mjs'; -import { lazy } from './misc.mjs'; -import { rehypeOptions } from './remark.mjs'; - -/** - * Retrieves an instance of Remark configured to output stringified HTML code - * including parsing Code Boxes with syntax highlighting. - * - * This lives apart from `./remark.mjs` because importing it loads Shiki (every - * grammar plus the regex engine, ~250MB per process). Only the generators that - * highlight code — `legacy-html` here — should pay for that. - */ -export const getRemarkRehypeWithShiki = lazy(() => - unified() - .use(remarkParse) - // legacy-html gets the minimal (unhighlighted) type rendering - .use(remarkRehype, rehypeOptions) - // This is a custom ad-hoc within the Shiki Rehype plugin, used to highlight code - // and transform them into HAST nodes - .use(syntaxHighlighter) - .use(rehypeStringify, { allowDangerousHtml: true }) -); diff --git a/packages/core/src/utils/remark.mjs b/packages/core/src/utils/remark.mjs deleted file mode 100644 index afbbe7f5e..000000000 --- a/packages/core/src/utils/remark.mjs +++ /dev/null @@ -1,91 +0,0 @@ -'use strict'; - -import rehypeStringify from 'rehype-stringify'; -import remarkGfm from 'remark-gfm'; -import remarkMdx from 'remark-mdx'; -import remarkParse from 'remark-parse'; -import remarkRehype from 'remark-rehype'; -import remarkStringify from 'remark-stringify'; -import { unified } from 'unified'; - -import { typeAnnotationToHast } from '#plugins/type-annotations/hast.mjs'; -import remarkTypeAnnotations from '#plugins/type-annotations/remark.mjs'; - -import { lazy } from './misc.mjs'; - -// Nothing in this module loads Shiki: the `ast` and `metadata` stages (and -// every worker that runs them) import it, and none of them highlight code. -// The highlighting pipeline lives in `./remark-shiki.mjs`. - -/** - * Renders an MDX JSX element as just its children, so the surrounding prose - * still renders in HTML-string output. - * - * @param {import('mdast-util-to-hast').State} state - * @param {import('unist').Parent} node - */ -const mdxElementToChildren = (state, node) => state.all(node); - -/** - * Drops a node from HTML-string output. - */ -const dropNode = () => undefined; - -/** - * The `remark-rehype` options shared by the HTML-string pipelines. - * - * Existing HTML nodes pass through untouched (they were created during the - * rehype process), and dangerous HTML is allowed since the Markdown sources - * are trusted. The MDX node types cannot be rendered to an HTML string (that - * is the React generators' job): JSX elements degrade to their children so the - * surrounding prose still renders, and expressions/ESM are dropped. - * - * @type {import('remark-rehype').Options} - */ -export const rehypeOptions = { - allowDangerousHtml: true, - passThrough: ['element'], - handlers: { - typeAnnotation: typeAnnotationToHast, - mdxJsxTextElement: mdxElementToChildren, - mdxJsxFlowElement: mdxElementToChildren, - mdxFlowExpression: dropNode, - mdxTextExpression: dropNode, - mdxjsEsm: dropNode, - }, -}; - -/** - * Retrieves an instance of Remark configured to parse GFM (GitHub Flavored Markdown) - * plus `{...}` type annotations (see `./type-annotations`), which only exist - * in non-MDX files — the MDX pipeline below never registers them. - */ -export const getRemark = lazy(() => - unified() - .use(remarkParse) - .use(remarkTypeAnnotations) - .use(remarkGfm) - .use(remarkStringify) -); - -/** - * Retrieves an instance of Remark configured to parse MDX (JSX-in-Markdown). - * - * Unlike {@link getRemark}, this understands `` and `{expression}` - * syntax as real JSX/expression nodes. It is only used for `.mdx` (or - * explicitly opted-in) files, since Node.js core `.md` files use bare `<` and - * `{` for type annotations that MDX would otherwise try to parse. - */ -export const getRemarkMdx = lazy(() => - unified().use(remarkParse).use(remarkMdx).use(remarkGfm).use(remarkStringify) -); - -/** - * Retrieves an instance of Remark configured to output stringified HTML code - */ -export const getRemarkRehype = lazy(() => - unified() - .use(remarkParse) - .use(remarkRehype, rehypeOptions) - .use(rehypeStringify, { allowDangerousHtml: true }) -); diff --git a/packages/node-legacy/package.json b/packages/node-legacy/package.json index 770438c23..9984b535a 100644 --- a/packages/node-legacy/package.json +++ b/packages/node-legacy/package.json @@ -27,6 +27,9 @@ "dependencies": { "@doc-kit/core": "workspace:*", "hastscript": "^9.0.1", + "rehype-stringify": "^10.0.1", + "remark-parse": "^11.0.0", + "remark-rehype": "^11.1.2", "unist-builder": "^4.0.0", "unist-util-visit": "^5.1.0" } diff --git a/packages/node-legacy/src/legacy-html-all/generate.mjs b/packages/node-legacy/src/legacy-html-all/generate.mjs index 9840bc131..307c7a029 100644 --- a/packages/node-legacy/src/legacy-html-all/generate.mjs +++ b/packages/node-legacy/src/legacy-html-all/generate.mjs @@ -5,7 +5,7 @@ import { join } from 'node:path'; import getConfig from '@doc-kit/core/utils/configuration/index.mjs'; import { minifyHTML } from '@doc-kit/core/utils/html-minifier.mjs'; -import { getRemarkRehype as remark } from '@doc-kit/core/utils/remark.mjs'; +import { getProcessor } from '@doc-kit/core/utils/markdown/processor.mjs'; import { replaceTemplateValues } from '../legacy-html/utils/replaceTemplateValues.mjs'; import tableOfContents from '../legacy-html/utils/tableOfContents.mjs'; @@ -38,7 +38,7 @@ export async function generate(input) { })); // Generates the global Table of Contents (Sidebar Navigation) - const parsedSideNav = remark().processSync( + const parsedSideNav = getProcessor('legacy-html').processSync( tableOfContents(sideNavigationFromValues, { maxDepth: 1, parser: tableOfContents.parseNavigationNode, diff --git a/packages/node-legacy/src/legacy-html/generate.mjs b/packages/node-legacy/src/legacy-html/generate.mjs index 6ff25a816..5528b652c 100644 --- a/packages/node-legacy/src/legacy-html/generate.mjs +++ b/packages/node-legacy/src/legacy-html/generate.mjs @@ -7,7 +7,7 @@ import getConfig from '@doc-kit/core/utils/configuration/index.mjs'; import { writeFile } from '@doc-kit/core/utils/file.mjs'; import { groupNodesByModule } from '@doc-kit/core/utils/generators.mjs'; import { minifyHTML } from '@doc-kit/core/utils/html-minifier.mjs'; -import { getRemarkRehypeWithShiki as remark } from '@doc-kit/core/utils/remark-shiki.mjs'; +import { getProcessor } from '@doc-kit/core/utils/markdown/processor.mjs'; import buildContent from './utils/buildContent.mjs'; import { replaceTemplateValues } from './utils/replaceTemplateValues.mjs'; @@ -41,7 +41,7 @@ export async function processChunk(slicedInput, itemIndices, navigation) { ); const toc = String( - remark().processSync( + getProcessor('legacy-html').processSync( tableOfContents(nodes, { maxDepth: 5, parser: tableOfContents.parseToCNode, @@ -93,7 +93,7 @@ export async function* generate(input, worker) { : headNodes; const navigation = String( - remark().processSync( + getProcessor('legacy-html').processSync( tableOfContents(indexOfFiles, { maxDepth: 1, parser: tableOfContents.parseNavigationNode, diff --git a/packages/node-legacy/src/legacy-html/index.mjs b/packages/node-legacy/src/legacy-html/index.mjs index 18ce5f299..1bcb28f62 100644 --- a/packages/node-legacy/src/legacy-html/index.mjs +++ b/packages/node-legacy/src/legacy-html/index.mjs @@ -4,6 +4,7 @@ import { join } from 'node:path'; import { GITHUB_EDIT_URL } from '@doc-kit/core/utils/configuration/templates.mjs'; +import { rehypeOptions } from '../utils/rehypeOptions.mjs'; import { generate, processChunk } from './generate.mjs'; /** @@ -34,6 +35,17 @@ export default { hasParallelProcessor: true, + // Renders the pages' Markdown into HTML, highlighting their code. It takes + // no configured plugins, which would change the legacy output + markdown: { + remarkPlugins: ['remark-parse'], + rehypePlugins: [ + ['remark-rehype', rehypeOptions], + './plugins/shiki.mjs', + ['rehype-stringify', { allowDangerousHtml: true }], + ], + }, + generate, processChunk, }; diff --git a/packages/core/src/utils/highlighter.mjs b/packages/node-legacy/src/legacy-html/plugins/shiki.mjs similarity index 89% rename from packages/core/src/utils/highlighter.mjs rename to packages/node-legacy/src/legacy-html/plugins/shiki.mjs index 9e8fb6e20..188915fdd 100644 --- a/packages/core/src/utils/highlighter.mjs +++ b/packages/node-legacy/src/legacy-html/plugins/shiki.mjs @@ -1,12 +1,11 @@ 'use strict'; -import { endianness } from 'node:os'; - -import createHighlighter from '@node-core/rehype-shiki'; +import { createHighlighter } from '@doc-kit/core/plugins/shiki/highlighter.mjs'; +import shikiConfig from '@doc-kit/core/shiki.config.mjs'; import { h as createElement } from 'hastscript'; import { SKIP, visit } from 'unist-util-visit'; -import shikiConfig from '../../shiki.config.mjs'; +const highlighter = await createHighlighter(); // This is what Remark will use as prefix within a
 className
 // to attribute the current language of the 
 element
@@ -38,22 +37,8 @@ function isCodeBlock(node) {
   );
 }
 
-export const highlighter = await createHighlighter({
-  // riscv64 with sv39 has limited virtual memory space, where creating
-  // too many (>20) wasm memory instances fails.
-  // https://github.com/nodejs/node/pull/60591
-  //
-  // The wasm highlighter is currently not compatible with big endian.
-  // https://github.com/nodejs/node/pull/62512#issuecomment-4243469950
-  wasm: process.arch !== 'riscv64' && endianness() === 'LE',
-});
-
 /**
  * Creates a HAST transformer for Shiki which is used for transforming our codeboxes
- *
- * @deprecated This is used only for the legacy-html generator, please use `@node-core/rehype-shiki` directly instead.
- *
- * @type {import('unified').Plugin}
  */
 export default function rehypeShikiji() {
   /**
diff --git a/packages/node-legacy/src/legacy-html/utils/__tests__/buildContent.test.mjs b/packages/node-legacy/src/legacy-html/utils/__tests__/buildContent.test.mjs
index edb1d149e..5d0bc6a70 100644
--- a/packages/node-legacy/src/legacy-html/utils/__tests__/buildContent.test.mjs
+++ b/packages/node-legacy/src/legacy-html/utils/__tests__/buildContent.test.mjs
@@ -3,10 +3,17 @@
 import assert from 'node:assert/strict';
 import { before, describe, it } from 'node:test';
 
+import { loadGenerator } from '@doc-kit/core/generators/loader.mjs';
 import { setConfig } from '@doc-kit/core/utils/configuration/index.mjs';
+import { loadMarkdownPlugins } from '@doc-kit/core/utils/markdown/plugins.mjs';
 
 import buildContent from '../buildContent.mjs';
 
+// The content is rendered with the pipeline of `legacy-html`
+await loadMarkdownPlugins(
+  await loadGenerator(import.meta.resolve('../../index.mjs'))
+);
+
 const createEntry = slug => {
   const text = 'DEP0001: deprecated API';
   const heading = {
diff --git a/packages/node-legacy/src/legacy-html/utils/buildContent.mjs b/packages/node-legacy/src/legacy-html/utils/buildContent.mjs
index 37abac651..8584f47e0 100644
--- a/packages/node-legacy/src/legacy-html/utils/buildContent.mjs
+++ b/packages/node-legacy/src/legacy-html/utils/buildContent.mjs
@@ -5,8 +5,8 @@ import {
   GITHUB_BLOB_URL,
   populate,
 } from '@doc-kit/core/utils/configuration/templates.mjs';
+import { getProcessor } from '@doc-kit/core/utils/markdown/processor.mjs';
 import { UNIST } from '@doc-kit/core/utils/queries/index.mjs';
-import { getRemarkRehypeWithShiki as remark } from '@doc-kit/core/utils/remark-shiki.mjs';
 import { h as createElement } from 'hastscript';
 import { u as createTree } from 'unist-builder';
 import { SKIP, visit } from 'unist-util-visit';
@@ -80,7 +80,7 @@ const buildStability = ({ children, data }, index, parent) => {
  * @param {import('@doc-kit/core/generators/metadata/types').ChangeEntry} change
  */
 const createHistoryTableRow = ({ version: changeVersions, description }) => {
-  const descriptionNode = remark().parse(description);
+  const descriptionNode = getProcessor('legacy-html').parse(description);
 
   return createElement('tr', [
     createElement(
@@ -241,8 +241,8 @@ export default (headNodes, metadataEntries) => {
     })
   );
 
-  const processedNodes = remark().runSync(parsedNodes);
+  const processedNodes = getProcessor('legacy-html').runSync(parsedNodes);
 
   // Stringifies the processed nodes to return the final Markdown content
-  return remark().stringify(processedNodes);
+  return getProcessor('legacy-html').stringify(processedNodes);
 };
diff --git a/packages/core/src/utils/__tests__/remark.test.mjs b/packages/node-legacy/src/legacy-json/__tests__/markdown.test.mjs
similarity index 69%
rename from packages/core/src/utils/__tests__/remark.test.mjs
rename to packages/node-legacy/src/legacy-json/__tests__/markdown.test.mjs
index 40250885d..f5f271443 100644
--- a/packages/core/src/utils/__tests__/remark.test.mjs
+++ b/packages/node-legacy/src/legacy-json/__tests__/markdown.test.mjs
@@ -1,11 +1,17 @@
 import assert from 'node:assert/strict';
 import { describe, it } from 'node:test';
 
-import { getRemarkRehype } from '../remark.mjs';
+import { loadGenerator } from '@doc-kit/core/generators/loader.mjs';
+import { loadMarkdownPlugins } from '@doc-kit/core/utils/markdown/plugins.mjs';
+import { getProcessor } from '@doc-kit/core/utils/markdown/processor.mjs';
 
-describe('getRemarkRehype', () => {
+await loadMarkdownPlugins(
+  await loadGenerator(import.meta.resolve('../index.mjs'))
+);
+
+describe('the Markdown pipeline of legacy-json', () => {
   it('degrades MDX nodes instead of crashing rehype-stringify', () => {
-    const processor = getRemarkRehype();
+    const processor = getProcessor('legacy-json');
 
     const tree = {
       type: 'root',
diff --git a/packages/node-legacy/src/legacy-json/index.mjs b/packages/node-legacy/src/legacy-json/index.mjs
index 2109e7a33..841e54659 100644
--- a/packages/node-legacy/src/legacy-json/index.mjs
+++ b/packages/node-legacy/src/legacy-json/index.mjs
@@ -1,5 +1,6 @@
 'use strict';
 
+import { rehypeOptions } from '../utils/rehypeOptions.mjs';
 import { generate, processChunk } from './generate.mjs';
 
 /**
@@ -28,6 +29,15 @@ export default {
 
   hasParallelProcessor: true,
 
+  // Renders the descriptions' Markdown into HTML. It takes no configured
+  // plugins, which would change the legacy output
+  markdown: {
+    rehypePlugins: [
+      ['remark-rehype', rehypeOptions],
+      ['rehype-stringify', { allowDangerousHtml: true }],
+    ],
+  },
+
   generate,
   processChunk,
 };
diff --git a/packages/node-legacy/src/legacy-json/utils/buildSection.mjs b/packages/node-legacy/src/legacy-json/utils/buildSection.mjs
index 0c02d5291..c22474598 100644
--- a/packages/node-legacy/src/legacy-json/utils/buildSection.mjs
+++ b/packages/node-legacy/src/legacy-json/utils/buildSection.mjs
@@ -1,7 +1,7 @@
 import { enforceArray } from '@doc-kit/core/utils/array.mjs';
 import { populate } from '@doc-kit/core/utils/configuration/templates.mjs';
 import { buildHierarchy } from '@doc-kit/core/utils/hierarchy.mjs';
-import { getRemarkRehype as remark } from '@doc-kit/core/utils/remark.mjs';
+import { getProcessor } from '@doc-kit/core/utils/markdown/processor.mjs';
 import { parseList } from '@doc-kit/core/utils/signature/parseList.mjs';
 import { transformNodesToString } from '@doc-kit/core/utils/unist.mjs';
 
@@ -125,8 +125,8 @@ export const createSectionBuilder = () => {
       return;
     }
 
-    const rendered = remark().stringify(
-      remark().runSync({ type: 'root', children: nodes })
+    const rendered = getProcessor('legacy-json').stringify(
+      getProcessor('legacy-json').runSync({ type: 'root', children: nodes })
     );
 
     section.shortDesc = section.desc || undefined;
diff --git a/packages/node-legacy/src/utils/rehypeOptions.mjs b/packages/node-legacy/src/utils/rehypeOptions.mjs
new file mode 100644
index 000000000..f2387ccaf
--- /dev/null
+++ b/packages/node-legacy/src/utils/rehypeOptions.mjs
@@ -0,0 +1,49 @@
+'use strict';
+
+import { typeAnnotationToHast } from '@doc-kit/core/plugins/type-annotations/hast.mjs';
+
+/**
+ * A `remark-rehype` handler, turning an mdast node into hast.
+ *
+ * @typedef {NonNullable} Handler
+ */
+
+/**
+ * Renders an MDX JSX element as just its children, so the surrounding prose
+ * still renders in HTML-string output.
+ *
+ * @type {Handler}
+ */
+const mdxElementToChildren = (state, node) => state.all(node);
+
+/**
+ * Drops a node from HTML-string output.
+ *
+ * @type {Handler}
+ */
+const dropNode = () => undefined;
+
+/**
+ * The `remark-rehype` options of the legacy generators, which render Markdown
+ * into HTML strings.
+ *
+ * Existing HTML nodes pass through untouched (they were created during the
+ * rehype process), and dangerous HTML is allowed since the Markdown sources
+ * are trusted. The MDX node types cannot be rendered to an HTML string (that
+ * is the React generators' job): JSX elements degrade to their children so the
+ * surrounding prose still renders, and expressions/ESM are dropped.
+ *
+ * @type {import('remark-rehype').Options}
+ */
+export const rehypeOptions = {
+  allowDangerousHtml: true,
+  passThrough: ['element'],
+  handlers: {
+    typeAnnotation: typeAnnotationToHast,
+    mdxJsxTextElement: mdxElementToChildren,
+    mdxJsxFlowElement: mdxElementToChildren,
+    mdxFlowExpression: dropNode,
+    mdxTextExpression: dropNode,
+    mdxjsEsm: dropNode,
+  },
+};
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index d2d1ec1f4..af67ac0e6 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -104,9 +104,6 @@ importers:
       glob-parent:
         specifier: ^6.0.2
         version: 6.0.2
-      hastscript:
-        specifier: ^9.0.1
-        version: 9.0.1
       mdast-util-slice-markdown:
         specifier: ^2.0.1
         version: 2.0.1
@@ -208,6 +205,15 @@ importers:
       hastscript:
         specifier: ^9.0.1
         version: 9.0.1
+      rehype-stringify:
+        specifier: ^10.0.1
+        version: 10.0.1
+      remark-parse:
+        specifier: ^11.0.0
+        version: 11.0.0(supports-color@7.2.0)
+      remark-rehype:
+        specifier: ^11.1.2
+        version: 11.1.2
       unist-builder:
         specifier: ^4.0.0
         version: 4.0.0

From fc40dff55307fe36d09464fa0499653a929f1906 Mon Sep 17 00:00:00 2001
From: Claudio Wunder 
Date: Thu, 8 Oct 2026 13:01:05 +0200
Subject: [PATCH 08/10] docs: document Markdown plugins

Documents the `markdown` option, the Shiki plugin options, and how
generators declare their pipelines. The changeset also lists the
`@doc-kit/core` modules that are removed or moved.

Assisted-by: Claude Opus 5.5 
---
 .changeset/markdown-plugins.md |  7 +++
 docs/configuration.md          | 94 +++++++++++++++++++++++++++++++++-
 docs/creating-generators.md    | 58 +++++++++++++++++++++
 3 files changed, 158 insertions(+), 1 deletion(-)
 create mode 100644 .changeset/markdown-plugins.md

diff --git a/.changeset/markdown-plugins.md b/.changeset/markdown-plugins.md
new file mode 100644
index 000000000..402484ea8
--- /dev/null
+++ b/.changeset/markdown-plugins.md
@@ -0,0 +1,7 @@
+---
+'@doc-kit/core': minor
+'@doc-kit/generator-react': minor
+'@node-core/doc-kit-legacy': patch
+---
+
+Add a `markdown` option to add remark, rehype, and recma plugins to the generators processing Markdown, or configure the ones they use, such as Shiki. The `@doc-kit/core` modules the generators' pipelines replace are removed (`utils/remark.mjs`, `utils/remark-shiki.mjs`, and `utils/highlighter.mjs`), and `utils/type-annotations` moves to `plugins/type-annotations`
diff --git a/docs/configuration.md b/docs/configuration.md
index 8aa63a494..1865a918c 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -123,9 +123,11 @@ Everything under the `global` key applies to every generator:
   a URL or path to parse, or a pre-parsed array. **Default:** `[]`
   (single-version output).
 - `index` {string|URL|Array} Index URL.
+- `markdown` {Object} [Markdown plugins](#markdown-plugins) for every
+  generator processing Markdown.
 
 A generator's own section (e.g., `html`, `legacy-json`) can override any of
-these for that generator alone.
+these for that generator alone, except `markdown`, which it adds to.
 
 ## Execution options
 
@@ -155,6 +157,96 @@ export default {
 };
 ```
 
+## Markdown plugins
+
+`doc-kit` processes Markdown with [unified](https://unifiedjs.com/).
+`global.markdown` adds plugins to every generator processing Markdown, and a
+generator's own `markdown` adds plugins to it alone, after the global ones.
+
+```mjs displayName="doc-kit.config.mjs"
+export default {
+  global: {
+    markdown: {
+      remarkPlugins: ['remark-math'],
+    },
+  },
+
+  // The generator rendering the site's pages
+  'jsx-ast': {
+    markdown: {
+      rehypePlugins: [
+        'rehype-katex',
+        ['./plugins/rehype-diagrams.mjs', { theme: 'neutral' }],
+      ],
+    },
+  },
+};
+```
+
+Each plugin is a module specifier (a package name, or a path relative to the
+configuration file), alone or in a `[specifier, options]` pair. The module
+default-exports the plugin, a list of plugins, or a unified preset. Markdown is
+processed in worker threads, which import the plugins themselves, so options
+must be serializable: configure a plugin taking functions in a module of your
+own.
+
+- `remarkPlugins` {Array} The global ones run on each document once the `ast`
+  generator parses it, so every output sees their changes. A generator
+  rendering Markdown, such as `jsx-ast`, runs its own on what it renders.
+- `rehypePlugins` {Array} Run on the HTML of the generators rendering
+  Markdown, such as `jsx-ast`, before code is highlighted.
+- `recmaPlugins` {Array} Run on the JavaScript `jsx-ast` compiles the pages to.
+
+A generator only takes the plugins its pipeline has a place for: `jsx-ast`
+takes all three kinds, `ast`, `metadata`, and `json` take remark plugins, and
+generators such as `html` take none. Plugins listed for a generator that
+doesn't take them are ignored with a warning.
+
+A generator runs each plugin once: listing one it already has, its own or a
+global one, configures it instead, merging the options (arrays add up, and
+objects merge).
+
+The plugins configured for `jsx-ast` don't apply to the fragments it renders
+on their own: types, parameter descriptions, and change history notes.
+
+### Code highlighting
+
+`jsx-ast` highlights code with the `@doc-kit/core/plugins/shiki/rehype.mjs`
+plugin, which runs [Shiki](https://shiki.style/) with every language it
+bundles. To configure it, list it with options in
+`jsx-ast.markdown.rehypePlugins` (with pnpm, add `@doc-kit/core` to your
+dependencies, so your configuration file can resolve it):
+
+- `langs` {Array} More languages: grammars, or modules default-exporting them.
+- `langAlias` {Object} Aliases of languages, such as `{ conf: 'ini' }`.
+- `themes` {Object} The `light` and `dark` themes: names of themes Shiki
+  bundles, modules default-exporting themes, or themes.
+- `transformers` {Array} Modules default-exporting
+  [Shiki transformers](https://shiki.style/guide/transformers), or lists of
+  them.
+
+These modules are paths relative to the working directory, or URLs, such as
+the ones `import.meta.resolve()` gives:
+
+```mjs displayName="doc-kit.config.mjs"
+export default {
+  'jsx-ast': {
+    markdown: {
+      rehypePlugins: [
+        [
+          '@doc-kit/core/plugins/shiki/rehype.mjs',
+          {
+            langAlias: { conf: 'ini' },
+            themes: { light: 'github-light', dark: 'github-dark' },
+            transformers: [import.meta.resolve('./shiki-transformers.mjs')],
+          },
+        ],
+      ],
+    },
+  },
+};
+```
+
 ## Configuration Merging
 
 Configurations are merged in the following order (higher sources take
diff --git a/docs/creating-generators.md b/docs/creating-generators.md
index 2842a6458..e81585586 100644
--- a/docs/creating-generators.md
+++ b/docs/creating-generators.md
@@ -468,6 +468,64 @@ Use a dependent when a generator transforms an intermediate representation
 format of its own. See the [`section-pages`](./generators/section-pages.md) generator for
 a worked example.
 
+## Markdown pipelines
+
+A generator processing Markdown declares its [unified](https://unifiedjs.com/)
+pipeline as `markdown`, listing plugins as the
+[`markdown` option](./configuration.md#markdown-plugins) does, with paths
+relative to its module. The rehype list starts with `remark-rehype`, and the
+recma list with `rehype-recma`. Each list takes the configured plugins in place
+of its `'...'`, and a list without one takes none. As every thread running the
+generator imports it, options may be anything, functions included.
+
+```mjs displayName="index.mjs"
+export default {
+  name: 'my-generator',
+
+  dependsOn: '@doc-kit/core/metadata',
+
+  markdown: {
+    remarkPlugins: ['remark-parse', 'remark-gfm', '...'],
+    rehypePlugins: [
+      ['remark-rehype', { allowDangerousHtml: true }],
+      '...',
+      ['rehype-stringify', { allowDangerousHtml: true }],
+    ],
+  },
+
+  generate,
+};
+```
+
+`getProcessor(name)` gives the generator's processor, on each thread it runs
+on:
+
+```mjs displayName="generate.mjs"
+import { getProcessor } from '@doc-kit/core/utils/markdown/processor.mjs';
+
+export async function generate(input) {
+  const processor = getProcessor('my-generator');
+
+  return Promise.all(
+    input.map(async ({ content }) =>
+      processor.stringify(await processor.run(content))
+    )
+  );
+}
+```
+
+`getProcessor(name, { mdx: true })` returns an MDX processor (plugins can
+check `this.data('mdx')`), and `getProcessor(name, { configured: false })` one
+without the configured plugins. A generator rendering Markdown (with rehype or
+recma plugins) only takes its own remark plugins, as the global ones already
+ran when `ast` parsed the Markdown.
+
+A plugin needing asynchronous setup can export an async `load(options)`
+returning the plugin, instead of a default export. The Shiki plugin does, and
+`getHighlighter(name)` (from `@doc-kit/core/plugins/shiki/highlighter.mjs`)
+returns the highlighter of a generator's Shiki plugin, to highlight code as its
+pipeline does.
+
 ## File Output
 
 ### Writing Output Files

From 0919affea12e92523e6c9e00562217001c90c0a5 Mon Sep 17 00:00:00 2001
From: avivkeller 
Date: Thu, 8 Oct 2026 13:08:31 -0400
Subject: [PATCH 09/10] chore: dry

---
 .../core/src/plugins/shiki/highlighter.mjs    | 41 +++++++++--------
 .../core/src/utils/configuration/index.mjs    |  7 +--
 packages/core/src/utils/markdown/plugins.mjs  | 46 ++++++++++++-------
 .../src/legacy-html/utils/buildContent.mjs    |  5 +-
 .../src/legacy-json/utils/buildSection.mjs    |  5 +-
 5 files changed, 61 insertions(+), 43 deletions(-)

diff --git a/packages/core/src/plugins/shiki/highlighter.mjs b/packages/core/src/plugins/shiki/highlighter.mjs
index dbfc8112b..a4c3a8513 100644
--- a/packages/core/src/plugins/shiki/highlighter.mjs
+++ b/packages/core/src/plugins/shiki/highlighter.mjs
@@ -63,15 +63,11 @@ const highlighters = new Map();
  * @returns {Promise>}
  */
 const importList = async options => {
-  const imported = [];
-
-  for (const option of options) {
-    if (typeof option === 'string') {
-      imported.push(await importFromURL(option));
-    } else {
-      imported.push(option);
-    }
-  }
+  const imported = await Promise.all(
+    options.map(option =>
+      typeof option === 'string' ? importFromURL(option) : option
+    )
+  );
 
   return imported.flat();
 };
@@ -113,23 +109,32 @@ const importHighlighter = async ({
 }) => {
   engine ??= createEngine();
 
+  const [regexEngine, importedLangs, importedTransformers, importedThemes] =
+    await Promise.all([
+      engine,
+      importList(langs),
+      importList(transformers),
+      themes &&
+        Promise.all([
+          importTheme(themes.light, 'light'),
+          importTheme(themes.dark, 'dark'),
+        ]),
+    ]);
+
   const coreOptions = {
-    engine: await engine,
-    langs: [...LANGS, ...(await importList(langs))],
+    engine: regexEngine,
+    langs: [...LANGS, ...importedLangs],
     // A copy, as Shiki adds the aliases of the languages it bundles to it
     langAlias: { ...langAlias },
   };
 
-  const highlighterOptions = {
-    transformers: await importList(transformers),
-  };
+  const highlighterOptions = { transformers: importedTransformers };
 
   // Without themes of its own, the highlighter has a default light and dark one
-  if (themes) {
-    const light = await importTheme(themes.light, 'light');
-    const dark = await importTheme(themes.dark, 'dark');
+  if (importedThemes) {
+    const [light, dark] = importedThemes;
 
-    coreOptions.themes = [light, dark];
+    coreOptions.themes = importedThemes;
     highlighterOptions.themes = { light: light.name, dark: dark.name };
     highlighterOptions.defaultColor = 'light';
   }
diff --git a/packages/core/src/utils/configuration/index.mjs b/packages/core/src/utils/configuration/index.mjs
index ac61fff35..185c0c24b 100644
--- a/packages/core/src/utils/configuration/index.mjs
+++ b/packages/core/src/utils/configuration/index.mjs
@@ -19,10 +19,7 @@ import {
   CONFIGURED_PLUGINS,
   PLUGIN_LISTS,
 } from '#utils/markdown/constants.mjs';
-import {
-  resolveMarkdown,
-  resolveMarkdownPipeline,
-} from '#utils/markdown/plugins.mjs';
+import { resolveMarkdown } from '#utils/markdown/plugins.mjs';
 import { deepMerge } from '#utils/misc.mjs';
 
 import { DEFAULT_CHUNK_SIZE, DEFAULT_MAX_THREADS } from './constants.mjs';
@@ -157,7 +154,7 @@ export const loadConfigFile = async filePath => {
  * @returns {Partial | undefined}
  */
 const configureMarkdown = (generator, markdown = {}, global) => {
-  const pipeline = resolveMarkdownPipeline(generator);
+  const pipeline = generator.markdown ?? {};
   const renders = Boolean(pipeline.rehypePlugins || pipeline.recmaPlugins);
   const configured = {};
 
diff --git a/packages/core/src/utils/markdown/plugins.mjs b/packages/core/src/utils/markdown/plugins.mjs
index 033251ad3..1ab58c2ef 100644
--- a/packages/core/src/utils/markdown/plugins.mjs
+++ b/packages/core/src/utils/markdown/plugins.mjs
@@ -63,7 +63,7 @@ export const resolveMarkdown = (markdown, label, filePath) => {
  * @param {GeneratorMetadata} generator - The generator
  * @returns {import('../configuration/types').MarkdownPipeline}
  */
-export const resolveMarkdownPipeline = generator => {
+const resolveMarkdownPipeline = generator => {
   const module = getGeneratorModule(generator);
 
   return resolveMarkdown(
@@ -112,19 +112,30 @@ const configureList = (own = [], configured = []) => {
   const plugins = own.map(entry => ({ entry, added: false }));
   const added = [];
 
+  // Each plugin by its specifier: the first listing of it takes the options
+  const listed = new Map();
+
+  for (const plugin of plugins) {
+    const [specifier] = enforceArray(plugin.entry);
+
+    if (!listed.has(specifier)) {
+      listed.set(specifier, plugin);
+    }
+  }
+
   for (const entry of configured) {
     const [specifier, options] = enforceArray(entry);
+    const plugin = listed.get(specifier);
 
-    const listed = [...plugins, ...added].find(
-      plugin => enforceArray(plugin.entry)[0] === specifier
-    );
+    if (!plugin) {
+      const addedPlugin = { entry, added: true };
 
-    if (!listed) {
-      added.push({ entry, added: true });
+      added.push(addedPlugin);
+      listed.set(specifier, addedPlugin);
     } else if (options !== undefined) {
-      const [, listedOptions] = enforceArray(listed.entry);
+      const [, listedOptions] = enforceArray(plugin.entry);
 
-      listed.entry = [specifier, mergeOptions(listedOptions, options)];
+      plugin.entry = [specifier, mergeOptions(listedOptions, options)];
     }
   }
 
@@ -183,16 +194,19 @@ export const loadMarkdownPlugins = async (generator, markdown = {}) => {
   const configured = {};
   const own = {};
 
-  for (const list of PLUGIN_LISTS) {
-    const plugins = configureList(pipeline[list], markdown[list]);
+  // The lists import at once, so a slow plugin (Shiki) doesn't hold the rest
+  await Promise.all(
+    PLUGIN_LISTS.map(async list => {
+      const plugins = configureList(pipeline[list], markdown[list]);
 
-    const imported = await Promise.all(
-      plugins.map(({ entry }) => importPlugin(entry))
-    );
+      const imported = await Promise.all(
+        plugins.map(({ entry }) => importPlugin(entry))
+      );
 
-    configured[list] = imported;
-    own[list] = imported.filter((_, index) => !plugins[index].added);
-  }
+      configured[list] = imported;
+      own[list] = imported.filter((_, index) => !plugins[index].added);
+    })
+  );
 
   loadedPipelines.set(generator.name, { generator, key, configured, own });
 };
diff --git a/packages/node-legacy/src/legacy-html/utils/buildContent.mjs b/packages/node-legacy/src/legacy-html/utils/buildContent.mjs
index 8584f47e0..ae0cad1a0 100644
--- a/packages/node-legacy/src/legacy-html/utils/buildContent.mjs
+++ b/packages/node-legacy/src/legacy-html/utils/buildContent.mjs
@@ -241,8 +241,9 @@ export default (headNodes, metadataEntries) => {
     })
   );
 
-  const processedNodes = getProcessor('legacy-html').runSync(parsedNodes);
+  const processor = getProcessor('legacy-html');
+  const processedNodes = processor.runSync(parsedNodes);
 
   // Stringifies the processed nodes to return the final Markdown content
-  return getProcessor('legacy-html').stringify(processedNodes);
+  return processor.stringify(processedNodes);
 };
diff --git a/packages/node-legacy/src/legacy-json/utils/buildSection.mjs b/packages/node-legacy/src/legacy-json/utils/buildSection.mjs
index c22474598..5aad62039 100644
--- a/packages/node-legacy/src/legacy-json/utils/buildSection.mjs
+++ b/packages/node-legacy/src/legacy-json/utils/buildSection.mjs
@@ -125,8 +125,9 @@ export const createSectionBuilder = () => {
       return;
     }
 
-    const rendered = getProcessor('legacy-json').stringify(
-      getProcessor('legacy-json').runSync({ type: 'root', children: nodes })
+    const processor = getProcessor('legacy-json');
+    const rendered = processor.stringify(
+      processor.runSync({ type: 'root', children: nodes })
     );
 
     section.shortDesc = section.desc || undefined;

From c1ff71751faf70f19aea104e4b035bc05fe8018f Mon Sep 17 00:00:00 2001
From: Claudio Wunder 
Date: Fri, 9 Oct 2026 13:48:47 +0200
Subject: [PATCH 10/10] refactor(core): split up `Promise.all` calls and
 ternaries

Builds each list of promises before awaiting it, imports the Shiki
themes in a step of their own, and replaces the ternaries resolving and
configuring Markdown plugins with `if` statements, so each step reads
on its own.

Assisted-by: Claude Opus 5.5 
---
 .../core/src/plugins/shiki/highlighter.mjs    | 37 +++++++++----------
 .../core/src/utils/configuration/index.mjs    | 34 ++++++++---------
 packages/core/src/utils/markdown/plugins.mjs  | 19 +++++-----
 3 files changed, 44 insertions(+), 46 deletions(-)

diff --git a/packages/core/src/plugins/shiki/highlighter.mjs b/packages/core/src/plugins/shiki/highlighter.mjs
index a4c3a8513..6ecdef8de 100644
--- a/packages/core/src/plugins/shiki/highlighter.mjs
+++ b/packages/core/src/plugins/shiki/highlighter.mjs
@@ -63,12 +63,12 @@ const highlighters = new Map();
  * @returns {Promise>}
  */
 const importList = async options => {
-  const imported = await Promise.all(
-    options.map(option =>
-      typeof option === 'string' ? importFromURL(option) : option
-    )
+  const imports = options.map(option =>
+    typeof option === 'string' ? importFromURL(option) : option
   );
 
+  const imported = await Promise.all(imports);
+
   return imported.flat();
 };
 
@@ -86,7 +86,9 @@ const importTheme = async (theme, scheme) => {
 
   if (typeof theme === 'string') {
     if (theme in bundledThemes) {
-      imported = (await bundledThemes[theme]()).default;
+      const { default: bundledTheme } = await bundledThemes[theme]();
+
+      imported = bundledTheme;
     } else {
       imported = await importFromURL(theme);
     }
@@ -109,17 +111,11 @@ const importHighlighter = async ({
 }) => {
   engine ??= createEngine();
 
-  const [regexEngine, importedLangs, importedTransformers, importedThemes] =
-    await Promise.all([
-      engine,
-      importList(langs),
-      importList(transformers),
-      themes &&
-        Promise.all([
-          importTheme(themes.light, 'light'),
-          importTheme(themes.dark, 'dark'),
-        ]),
-    ]);
+  const [regexEngine, importedLangs, importedTransformers] = await Promise.all([
+    engine,
+    importList(langs),
+    importList(transformers),
+  ]);
 
   const coreOptions = {
     engine: regexEngine,
@@ -131,10 +127,13 @@ const importHighlighter = async ({
   const highlighterOptions = { transformers: importedTransformers };
 
   // Without themes of its own, the highlighter has a default light and dark one
-  if (importedThemes) {
-    const [light, dark] = importedThemes;
+  if (themes) {
+    const [light, dark] = await Promise.all([
+      importTheme(themes.light, 'light'),
+      importTheme(themes.dark, 'dark'),
+    ]);
 
-    coreOptions.themes = importedThemes;
+    coreOptions.themes = [light, dark];
     highlighterOptions.themes = { light: light.name, dark: dark.name };
     highlighterOptions.defaultColor = 'light';
   }
diff --git a/packages/core/src/utils/configuration/index.mjs b/packages/core/src/utils/configuration/index.mjs
index 185c0c24b..5ab508039 100644
--- a/packages/core/src/utils/configuration/index.mjs
+++ b/packages/core/src/utils/configuration/index.mjs
@@ -93,19 +93,16 @@ export const getDefaultConfig = (generators, config) =>
  */
 const resolveMarkdownPlugins = (config, filePath) =>
   Object.fromEntries(
-    Object.entries(config).map(([name, value]) => [
-      name,
-      value?.markdown
-        ? {
-            ...value,
-            markdown: resolveMarkdown(
-              value.markdown,
-              `${name}.markdown`,
-              filePath
-            ),
-          }
-        : value,
-    ])
+    Object.entries(config).map(([name, value]) => {
+      if (value?.markdown) {
+        const label = `${name}.markdown`;
+        const markdown = resolveMarkdown(value.markdown, label, filePath);
+
+        return [name, { ...value, markdown }];
+      }
+
+      return [name, value];
+    })
   );
 
 /**
@@ -170,10 +167,13 @@ const configureMarkdown = (generator, markdown = {}, global) => {
       continue;
     }
 
-    configured[list] = [
-      ...(list === 'remarkPlugins' && renders ? [] : global.markdown[list]),
-      ...(markdown[list] ?? []),
-    ];
+    const plugins = markdown[list] ?? [];
+
+    if (list === 'remarkPlugins' && renders) {
+      configured[list] = plugins;
+    } else {
+      configured[list] = [...global.markdown[list], ...plugins];
+    }
   }
 
   return generator.markdown && configured;
diff --git a/packages/core/src/utils/markdown/plugins.mjs b/packages/core/src/utils/markdown/plugins.mjs
index 1ab58c2ef..effaa4134 100644
--- a/packages/core/src/utils/markdown/plugins.mjs
+++ b/packages/core/src/utils/markdown/plugins.mjs
@@ -195,18 +195,17 @@ export const loadMarkdownPlugins = async (generator, markdown = {}) => {
   const own = {};
 
   // The lists import at once, so a slow plugin (Shiki) doesn't hold the rest
-  await Promise.all(
-    PLUGIN_LISTS.map(async list => {
-      const plugins = configureList(pipeline[list], markdown[list]);
+  const loading = PLUGIN_LISTS.map(async list => {
+    const plugins = configureList(pipeline[list], markdown[list]);
+    const imports = plugins.map(({ entry }) => importPlugin(entry));
 
-      const imported = await Promise.all(
-        plugins.map(({ entry }) => importPlugin(entry))
-      );
+    const imported = await Promise.all(imports);
 
-      configured[list] = imported;
-      own[list] = imported.filter((_, index) => !plugins[index].added);
-    })
-  );
+    configured[list] = imported;
+    own[list] = imported.filter((_, index) => !plugins[index].added);
+  });
+
+  await Promise.all(loading);
 
   loadedPipelines.set(generator.name, { generator, key, configured, own });
 };