` element
+ * @property {(code: string, lang: string, meta?: Record) => ReturnType} highlightToHast - Highlights code, returning a HAST tree
+ * @property {Array} langs - The languages it highlights
+ */
+
+/**
+ * Creates Shiki's regular expression engine: the wasm (Oniguruma) one where
+ * it can, the JavaScript one otherwise.
+ *
+ * @returns {Promise}
+ */
+const createEngine = async () => {
+ // riscv64 with sv39 has limited virtual memory space, where creating
+ // too many (>20) wasm memory instances fails.
+ // https://github.com/nodejs/node/pull/60591
+ //
+ // The wasm highlighter is currently not compatible with big endian.
+ // https://github.com/nodejs/node/pull/62512#issuecomment-4243469950
+ if (process.arch !== 'riscv64' && endianness() === 'LE') {
+ const { createOnigurumaEngine } = await import('shiki/engine/oniguruma');
+
+ return createOnigurumaEngine(import('shiki/wasm'));
+ }
+
+ const { createJavaScriptRegexEngine } =
+ await import('shiki/engine/javascript');
+
+ // Not every bundled grammar compiles under the JavaScript engine,
+ // so skip the patterns it cannot handle instead of throwing.
+ return createJavaScriptRegexEngine({ forgiving: true });
+};
+
+// The regular expression engine of this thread, for all its highlighters
+let engine;
+
+// The highlighters of the options given, by their JSON
+const highlighters = new Map();
+
+/**
+ * Imports a list of options, each given as it is, or as a module (a path
+ * relative to the working directory, or a URL) default-exporting it, or a list
+ * of them.
+ *
+ * @template T
+ * @param {Array} options - The options
+ * @returns {Promise>}
+ */
+const importList = async options => {
+ const imports = options.map(option =>
+ typeof option === 'string' ? importFromURL(option) : option
+ );
+
+ const imported = await Promise.all(imports);
+
+ return imported.flat();
+};
+
+/**
+ * Imports a theme: the name of one Shiki bundles, a module default-exporting
+ * one, or the theme itself. Shiki tells themes apart by their name, so a theme
+ * without one is named after its color scheme.
+ *
+ * @param {string | import('shiki').ThemeRegistration} theme - The theme
+ * @param {'light' | 'dark'} scheme - Its color scheme
+ * @returns {Promise}
+ */
+const importTheme = async (theme, scheme) => {
+ let imported = theme;
+
+ if (typeof theme === 'string') {
+ if (theme in bundledThemes) {
+ const { default: bundledTheme } = await bundledThemes[theme]();
+
+ imported = bundledTheme;
+ } else {
+ imported = await importFromURL(theme);
+ }
+ }
+
+ return { name: `custom-${scheme}`, ...imported };
+};
+
+/**
+ * Imports the options of a highlighter, and creates it.
+ *
+ * @param {import('./rehype.mjs').ShikiOptions} options - The options
+ * @returns {Promise}
+ */
+const importHighlighter = async ({
+ langs = [],
+ langAlias = {},
+ themes,
+ transformers = [],
+}) => {
+ engine ??= createEngine();
+
+ const [regexEngine, importedLangs, importedTransformers] = await Promise.all([
+ engine,
+ importList(langs),
+ importList(transformers),
+ ]);
+
+ const coreOptions = {
+ engine: regexEngine,
+ langs: [...LANGS, ...importedLangs],
+ // A copy, as Shiki adds the aliases of the languages it bundles to it
+ langAlias: { ...langAlias },
+ };
+
+ const highlighterOptions = { transformers: importedTransformers };
+
+ // Without themes of its own, the highlighter has a default light and dark one
+ if (themes) {
+ const [light, dark] = await Promise.all([
+ importTheme(themes.light, 'light'),
+ importTheme(themes.dark, 'dark'),
+ ]);
+
+ coreOptions.themes = [light, dark];
+ highlighterOptions.themes = { light: light.name, dark: dark.name };
+ highlighterOptions.defaultColor = 'light';
+ }
+
+ let highlighter;
+
+ /**
+ * Gives the Shiki highlighter, creating it on first use.
+ */
+ const current = () =>
+ (highlighter ??= createSyntaxHighlighter({
+ coreOptions,
+ highlighterOptions,
+ }));
+
+ return {
+ langs: coreOptions.langs,
+
+ /**
+ * The Shiki instance.
+ */
+ get shiki() {
+ return current().shiki;
+ },
+
+ /**
+ * Resolves a language, falling back to plain text for unknown ones.
+ *
+ * @param {string} [languageId]
+ */
+ resolveLanguage: languageId => current().resolveLanguage(languageId),
+
+ /**
+ * Highlights code, returning the inner HTML of its `` element.
+ *
+ * @param {...any} args - The code, its language, and its metadata
+ */
+ highlightToHtml: (...args) => current().highlightToHtml(...args),
+
+ /**
+ * Highlights code, returning a HAST tree.
+ *
+ * @param {...any} args - The code, its language, and its metadata
+ */
+ highlightToHast: (...args) => current().highlightToHast(...args),
+ };
+};
+
+/**
+ * Creates a highlighter of every language Shiki bundles, with the given
+ * options (see `./rehype.mjs`). Its Shiki instance is created on first use,
+ * and the same options give the same highlighter.
+ *
+ * @param {import('./rehype.mjs').ShikiOptions} [options] - The options
+ * @returns {Promise}
+ */
+export const createHighlighter = (options = {}) => {
+ const key = JSON.stringify(options);
+
+ if (!highlighters.has(key)) {
+ highlighters.set(key, importHighlighter(options));
+ }
+
+ return highlighters.get(key);
+};
+
+/**
+ * Gets the highlighter of the Shiki plugin of a generator's Markdown pipeline,
+ * as loaded on the current thread, to highlight code as the pipeline does.
+ *
+ * @param {string} generator - The name of the generator
+ * @returns {SyntaxHighlighter}
+ */
+export const getHighlighter = generator => {
+ const shiki = getMarkdownPlugins(generator).rehypePlugins.find(
+ plugin => Array.isArray(plugin) && plugin[0].highlighter
+ );
+
+ if (!shiki) {
+ throw new Error(
+ `The Markdown pipeline of "${generator}" does not highlight code`
+ );
+ }
+
+ return shiki[0].highlighter;
+};
diff --git a/packages/core/src/plugins/shiki/rehype.mjs b/packages/core/src/plugins/shiki/rehype.mjs
new file mode 100644
index 000000000..284a77cb1
--- /dev/null
+++ b/packages/core/src/plugins/shiki/rehype.mjs
@@ -0,0 +1,38 @@
+'use strict';
+
+import rehypeShikiji from '@node-core/rehype-shiki/plugin';
+
+import { createHighlighter } from './highlighter.mjs';
+
+/**
+ * The options of the Shiki plugin. Its modules are paths relative to the
+ * working directory, or URLs.
+ *
+ * @typedef {Object} ShikiOptions
+ * @property {Array} [langs] - More languages: grammars, or modules default-exporting grammars
+ * @property {Record} [langAlias] - Aliases of languages, e.g. `{ conf: 'ini' }`
+ * @property {{ light: string | import('shiki').ThemeRegistration, dark: string | import('shiki').ThemeRegistration }} [themes] - The light and dark themes: names of themes Shiki bundles, modules default-exporting themes, or themes
+ * @property {Array} [transformers] - Modules default-exporting Shiki transformers, or lists of them
+ */
+
+/**
+ * Loads the rehype plugin highlighting code blocks with Shiki, in every
+ * language it bundles. The plugin exposes its highlighter as `highlighter`
+ * (see `getHighlighter`).
+ *
+ * @param {ShikiOptions} [options] - The options
+ * @returns {Promise}
+ */
+export async function load(options) {
+ const highlighter = await createHighlighter(options);
+ const transformer = await rehypeShikiji({ highlighter });
+
+ /**
+ * Highlights the code blocks of a tree.
+ */
+ const shiki = () => transformer;
+
+ shiki.highlighter = highlighter;
+
+ return shiki;
+}
diff --git a/packages/core/src/utils/type-annotations/__tests__/hast.test.mjs b/packages/core/src/plugins/type-annotations/__tests__/hast.test.mjs
similarity index 94%
rename from packages/core/src/utils/type-annotations/__tests__/hast.test.mjs
rename to packages/core/src/plugins/type-annotations/__tests__/hast.test.mjs
index 34124ab9d..9aeb3c791 100644
--- a/packages/core/src/utils/type-annotations/__tests__/hast.test.mjs
+++ b/packages/core/src/plugins/type-annotations/__tests__/hast.test.mjs
@@ -4,8 +4,16 @@ import { describe, it } from 'node:test';
import { toHtml } from 'hast-util-to-html';
import { toString } from 'hast-util-to-string';
+import { createHighlighter } from '#plugins/shiki/highlighter.mjs';
+
import { typeAnnotationToHast } from '../hast.mjs';
-import { typeAnnotationToHighlightedHast } from '../highlighted.mjs';
+import { createTypeAnnotationHandler } from '../highlighter.mjs';
+
+const highlighter = await createHighlighter();
+
+const typeAnnotationToHighlightedHast = createTypeAnnotationHandler(
+ () => highlighter
+);
// A minimal mdast-util-to-hast state — the handlers only use patch/applyData
const state = { patch: () => {}, applyData: (_, result) => result };
diff --git a/packages/core/src/utils/type-annotations/__tests__/remark.test.mjs b/packages/core/src/plugins/type-annotations/__tests__/remark.test.mjs
similarity index 100%
rename from packages/core/src/utils/type-annotations/__tests__/remark.test.mjs
rename to packages/core/src/plugins/type-annotations/__tests__/remark.test.mjs
diff --git a/packages/core/src/utils/type-annotations/hast.mjs b/packages/core/src/plugins/type-annotations/hast.mjs
similarity index 100%
rename from packages/core/src/utils/type-annotations/hast.mjs
rename to packages/core/src/plugins/type-annotations/hast.mjs
diff --git a/packages/core/src/utils/type-annotations/highlighted.mjs b/packages/core/src/plugins/type-annotations/highlighter.mjs
similarity index 52%
rename from packages/core/src/utils/type-annotations/highlighted.mjs
rename to packages/core/src/plugins/type-annotations/highlighter.mjs
index bca00bcee..f26f9b25c 100644
--- a/packages/core/src/utils/type-annotations/highlighted.mjs
+++ b/packages/core/src/plugins/type-annotations/highlighter.mjs
@@ -1,37 +1,32 @@
'use strict';
-import { highlighter } from '#utils/highlighter.mjs';
-
import { typeAnnotationToHast } from './hast.mjs';
-// Kept apart from `./hast.mjs` on purpose: importing this module loads Shiki
-// (every grammar plus the regex engine), which only the pipelines that
-// highlight should pay for. The `ast` and `metadata` stages never do.
-const [lightTheme, darkTheme] = highlighter.shiki.getLoadedThemes();
-
/**
- * Syntax-highlighted mdast→hast handler for `typeAnnotation` nodes, used by
- * the web (JSX) pipeline. The whole type is highlighted as one inline
- * fragment, and each resolved identifier's exact character range is wrapped
- * in an `` via Shiki decorations. Values that are not TypeScript (display
- * names such as `HTTP/2 Headers Object`) are highlighted as plain text, so
- * their prose is not coloured as operators and numeric literals.
+ * Creates the syntax-highlighted mdast→hast handler for `typeAnnotation`
+ * nodes, used by the web (JSX) pipeline. The whole type is highlighted as one
+ * inline fragment, and each resolved identifier's exact character range is
+ * wrapped in an `` via Shiki decorations. Values that are not TypeScript
+ * (display names such as `HTTP/2 Headers Object`) are highlighted as plain
+ * text, so their prose is not coloured as operators and numeric literals.
*
* Falls back to the minimal handler when the type failed to parse or nothing
* resolved (no point paying for highlighting then).
*
- * @param {import('mdast-util-to-hast').State} state
- * @param {import('mdast').Node} node
- * @returns {import('hast').Element}
+ * @param {() => import('#plugins/shiki/highlighter.mjs').SyntaxHighlighter} getHighlighter - Gives the highlighter, once a type is highlighted
+ * @returns {(state: import('mdast-util-to-hast').State, node: import('mdast').Node) => import('hast').Element}
*/
-export const typeAnnotationToHighlightedHast = (state, node) => {
+export const createTypeAnnotationHandler = getHighlighter => (state, node) => {
const links = node.data?.links ?? [];
if (node.data?.parseError || links.length === 0) {
return typeAnnotationToHast(state, node);
}
- const root = highlighter.shiki.codeToHast(node.value, {
+ const { shiki } = getHighlighter();
+ const [lightTheme, darkTheme] = shiki.getLoadedThemes();
+
+ const root = shiki.codeToHast(node.value, {
lang: node.data?.typescript ? 'typescript' : 'text',
themes: { light: lightTheme, dark: darkTheme },
decorations: links.map(({ start, end, href }) => ({
diff --git a/packages/core/src/utils/type-annotations/mdast.mjs b/packages/core/src/plugins/type-annotations/mdast.mjs
similarity index 100%
rename from packages/core/src/utils/type-annotations/mdast.mjs
rename to packages/core/src/plugins/type-annotations/mdast.mjs
diff --git a/packages/core/src/utils/type-annotations/remark.mjs b/packages/core/src/plugins/type-annotations/remark.mjs
similarity index 82%
rename from packages/core/src/utils/type-annotations/remark.mjs
rename to packages/core/src/plugins/type-annotations/remark.mjs
index 82279efeb..1ef8c351a 100644
--- a/packages/core/src/utils/type-annotations/remark.mjs
+++ b/packages/core/src/plugins/type-annotations/remark.mjs
@@ -10,12 +10,16 @@ import { typeAnnotationSyntax } from './syntax.mjs';
* Remark plugin that teaches the parser to treat any balanced `{...}` span in
* text as a `typeAnnotation` node whose value is a TypeScript type expression.
*
- * Only registered on the non-MDX pipeline — in MDX, `{...}` is a real
- * expression and is handled by remark-mdx instead.
+ * It does nothing in MDX, where `{...}` is a real expression, handled by
+ * remark-mdx instead.
*
* @this {import('unified').Processor}
*/
export default function remarkTypeAnnotations() {
+ if (this.data('mdx')) {
+ return;
+ }
+
const data = this.data();
(data.micromarkExtensions ??= []).push(typeAnnotationSyntax());
diff --git a/packages/core/src/utils/type-annotations/syntax.mjs b/packages/core/src/plugins/type-annotations/syntax.mjs
similarity index 100%
rename from packages/core/src/utils/type-annotations/syntax.mjs
rename to packages/core/src/plugins/type-annotations/syntax.mjs
diff --git a/packages/core/src/threading/__tests__/fixtures/markdown-plugins-reporter.mjs b/packages/core/src/threading/__tests__/fixtures/markdown-plugins-reporter.mjs
new file mode 100644
index 000000000..19eb4cadc
--- /dev/null
+++ b/packages/core/src/threading/__tests__/fixtures/markdown-plugins-reporter.mjs
@@ -0,0 +1,24 @@
+import { getMarkdownPlugins } from '#utils/markdown/plugins.mjs';
+
+/**
+ * Test generator that reports the rehype plugins of its Markdown pipeline as
+ * loaded inside the worker, so their loading there can be asserted.
+ *
+ * @type {GeneratorMetadata}
+ */
+export default {
+ name: 'markdown-plugins-reporter',
+ version: '1.0.0',
+ description: 'Reports the rehype plugins loaded inside the worker',
+ dependsOn: 'ast',
+ markdown: { rehypePlugins: ['./rehype-plugin.mjs', '...'] },
+ processChunk: async (_input, itemIndices) =>
+ itemIndices.map(() =>
+ getMarkdownPlugins('markdown-plugins-reporter').rehypePlugins.map(
+ ([plugin, options]) => `${plugin.name} ${JSON.stringify(options)}`
+ )
+ ),
+ async generate() {
+ return [];
+ },
+};
diff --git a/packages/core/src/threading/__tests__/fixtures/rehype-plugin.mjs b/packages/core/src/threading/__tests__/fixtures/rehype-plugin.mjs
new file mode 100644
index 000000000..047c6b46e
--- /dev/null
+++ b/packages/core/src/threading/__tests__/fixtures/rehype-plugin.mjs
@@ -0,0 +1,4 @@
+/**
+ * A rehype plugin doing nothing, for workers to load.
+ */
+export default function rehypeFixture() {}
diff --git a/packages/core/src/threading/__tests__/index.test.mjs b/packages/core/src/threading/__tests__/index.test.mjs
index b5f6b63fa..dc6b63a3c 100644
--- a/packages/core/src/threading/__tests__/index.test.mjs
+++ b/packages/core/src/threading/__tests__/index.test.mjs
@@ -1,4 +1,4 @@
-import { strictEqual } from 'node:assert';
+import { deepStrictEqual, strictEqual } from 'node:assert';
import { describe, it } from 'node:test';
import { fileURLToPath } from 'node:url';
@@ -10,6 +10,10 @@ const reporterSpecifier = fileURLToPath(
import.meta.resolve('./fixtures/log-level-reporter.mjs')
);
+const pluginsReporterSpecifier = fileURLToPath(
+ import.meta.resolve('./fixtures/markdown-plugins-reporter.mjs')
+);
+
/**
* Runs a function with the logger temporarily set to the given level.
*
@@ -80,4 +84,34 @@ describe('createWorkerPool', () => {
await pool.destroy();
}
});
+
+ it('should load the Markdown pipeline of the generator in the worker', async () => {
+ const pool = createWorkerPool(1);
+
+ try {
+ const [plugins] = await pool.run({
+ generatorSpecifier: pluginsReporterSpecifier,
+ input: [null],
+ itemIndices: [0],
+ extra: {},
+ configuration: {
+ 'markdown-plugins-reporter': {
+ markdown: {
+ rehypePlugins: [
+ [
+ import.meta.resolve('./fixtures/rehype-plugin.mjs'),
+ { configured: true },
+ ],
+ ],
+ },
+ },
+ },
+ });
+
+ // The configured plugin is the one its pipeline has, which it configures
+ deepStrictEqual(plugins, ['rehypeFixture {"configured":true}']);
+ } finally {
+ await pool.destroy();
+ }
+ });
});
diff --git a/packages/core/src/threading/chunk-worker.mjs b/packages/core/src/threading/chunk-worker.mjs
index 7bfbae112..a0ff75015 100644
--- a/packages/core/src/threading/chunk-worker.mjs
+++ b/packages/core/src/threading/chunk-worker.mjs
@@ -3,6 +3,7 @@ import { workerData } from 'node:worker_threads';
import { loadGenerator } from '#generators/loader.mjs';
import logger from '#logger/index.mjs';
import { setConfig } from '#utils/configuration/index.mjs';
+import { loadMarkdownPlugins } from '#utils/markdown/plugins.mjs';
if (workerData?.logLevel !== undefined) {
logger.setLogLevel(workerData.logLevel);
@@ -26,5 +27,8 @@ export default async ({
const generator = await loadGenerator(generatorSpecifier);
+ // Plugins can't be sent to workers, which import the pipeline themselves
+ await loadMarkdownPlugins(generator, configuration[generator.name]?.markdown);
+
return generator.processChunk(input, itemIndices, extra);
};
diff --git a/packages/core/src/utils/configuration/__tests__/index.test.mjs b/packages/core/src/utils/configuration/__tests__/index.test.mjs
index c08023cf9..140c0f350 100644
--- a/packages/core/src/utils/configuration/__tests__/index.test.mjs
+++ b/packages/core/src/utils/configuration/__tests__/index.test.mjs
@@ -3,6 +3,9 @@ import { mkdtempSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { describe, it, mock, beforeEach } from 'node:test';
+import { pathToFileURL } from 'node:url';
+
+import logger from '../../../logger/index.mjs';
// Mock dependencies
const mockParseChangelog = mock.fn(async changelog => [changelog]);
@@ -22,7 +25,11 @@ const createMockConfig = (overrides = {}) => ({
// Synthetic generators keyed by specifier; the identity resolver below means
// shorthand names and specifiers are the same thing in these tests.
const mockGenerators = {
- json: { name: 'json', defaultConfiguration: { format: 'json' } },
+ json: {
+ name: 'json',
+ defaultConfiguration: { format: 'json' },
+ markdown: { remarkPlugins: ['file:///syntax.mjs', '...'] },
+ },
html: { name: 'html', defaultConfiguration: { format: 'html' } },
markdown: { name: 'markdown' },
web: {
@@ -31,6 +38,11 @@ const mockGenerators = {
showSearchBox:
Array.isArray(config.target) && config.target.includes('orama-db'),
}),
+ markdown: {
+ remarkPlugins: ['...'],
+ rehypePlugins: ['...'],
+ recmaPlugins: ['...'],
+ },
},
};
@@ -39,6 +51,7 @@ mock.module('../../../generators/loader.mjs', {
exports: {
resolveGeneratorSpecifier: specifier => specifier,
loadGenerator: async specifier => mockGenerators[specifier],
+ getGeneratorModule: () => undefined,
// Defaults are computed from the loaded generators; returning the full
// set regardless of targets keeps the assertions below simple.
loadGenerators: async () => new Map(Object.entries(mockGenerators)),
@@ -140,6 +153,48 @@ describe('config.mjs', () => {
assert.strictEqual(result.global.project, 'Node.js');
assert.strictEqual(result.global.repository, 'nodejs/node');
});
+
+ it('should resolve Markdown plugins from the file declaring them', async () => {
+ const dir = mkdtempSync(join(tmpdir(), 'doc-kit-config-'));
+
+ writeFileSync(
+ join(dir, 'preset.mjs'),
+ 'export default { "jsx-ast": { markdown: { rehypePlugins: ["./rehype.mjs"] } } };'
+ );
+
+ mockConfigLoad.mock.mockImplementationOnce(async () => ({
+ config: {
+ extends: '../preset.mjs',
+ global: {
+ markdown: { remarkPlugins: [['../remark.mjs', { a: 1 }]] },
+ },
+ },
+ filepath: join(dir, 'config', 'doc-kit.config.mjs'),
+ }));
+
+ const result = await loadConfigFile('any');
+
+ assert.deepStrictEqual(result.global.markdown.remarkPlugins, [
+ [pathToFileURL(join(dir, 'remark.mjs')).href, { a: 1 }],
+ ]);
+ // A preset's plugins resolve from the preset
+ assert.deepStrictEqual(result['jsx-ast'].markdown.rehypePlugins, [
+ pathToFileURL(join(dir, 'rehype.mjs')).href,
+ ]);
+ });
+
+ it('should reject Markdown plugins that are not module specifiers', async () => {
+ mockConfigLoad.mock.mockImplementationOnce(async () => ({
+ config: { global: { markdown: { rehypePlugins: [() => {}] } } },
+ filepath: '/doc-kit.config.mjs',
+ }));
+
+ await assert.rejects(loadConfigFile('any'), {
+ name: 'TypeError',
+ message:
+ /^global\.markdown\.rehypePlugins\[0\] in \/doc-kit\.config\.mjs must be a module specifier/,
+ });
+ });
});
describe('createConfigFromCLIOptions', () => {
@@ -322,6 +377,64 @@ describe('config.mjs', () => {
assert.ok(config.html);
assert.ok(config.markdown);
});
+
+ /**
+ * Creates a run configuration from a configuration file's contents.
+ *
+ * @param {object} contents - The configuration file's contents
+ */
+ const configureWith = contents => {
+ mockConfigLoad.mock.mockImplementationOnce(async () => ({
+ config: createMockConfig(contents),
+ filepath: '/doc-kit.config.mjs',
+ }));
+
+ return createRunConfiguration({ configFile: '/doc-kit.config.mjs' });
+ };
+
+ const url = name => pathToFileURL(`/plugins/${name}.mjs`).href;
+
+ it('should give each generator the Markdown plugins its pipeline takes', async () => {
+ const config = await configureWith({
+ global: {
+ markdown: {
+ remarkPlugins: ['/plugins/math.mjs'],
+ rehypePlugins: ['/plugins/katex.mjs'],
+ },
+ },
+ web: { markdown: { remarkPlugins: ['/plugins/web.mjs'] } },
+ });
+
+ assert.deepStrictEqual(config.json.markdown, {
+ remarkPlugins: [url('math')],
+ });
+ // The global remark plugins ran when `ast` parsed what `web` renders
+ assert.deepStrictEqual(config.web.markdown, {
+ remarkPlugins: [url('web')],
+ rehypePlugins: [url('katex')],
+ recmaPlugins: [],
+ });
+ assert.strictEqual(config.html.markdown, undefined);
+ });
+
+ it('should ignore the Markdown plugins a generator does not take', async t => {
+ const warn = t.mock.method(logger, 'warn', () => {});
+
+ const config = await configureWith({
+ json: { markdown: { rehypePlugins: ['/plugins/rehype.mjs'] } },
+ html: { markdown: { remarkPlugins: ['/plugins/remark.mjs'] } },
+ });
+
+ assert.deepStrictEqual(config.json.markdown, { remarkPlugins: [] });
+ assert.strictEqual(config.html.markdown, undefined);
+ assert.deepStrictEqual(
+ warn.mock.calls.map(({ arguments: [message] }) => message),
+ [
+ 'Ignoring `json.markdown.rehypePlugins`: `json` does not take them.',
+ 'Ignoring `html.markdown.remarkPlugins`: `html` does not take them.',
+ ]
+ );
+ });
});
describe('setConfig and getConfig', () => {
diff --git a/packages/core/src/utils/configuration/index.mjs b/packages/core/src/utils/configuration/index.mjs
index af7d2b4a6..5ab508039 100644
--- a/packages/core/src/utils/configuration/index.mjs
+++ b/packages/core/src/utils/configuration/index.mjs
@@ -1,8 +1,6 @@
import { readFileSync } from 'node:fs';
-import { createRequire } from 'node:module';
import { cpus } from 'node:os';
-import { dirname, isAbsolute, resolve } from 'node:path';
-import { pathToFileURL } from 'node:url';
+import { fileURLToPath } from 'node:url';
import { isMainThread } from 'node:worker_threads';
import { cosmiconfig } from 'cosmiconfig';
@@ -16,6 +14,12 @@ import logger from '#logger/index.mjs';
import { parseChangelog, parseIndex } from '#parsers/markdown.mjs';
import { enforceArray } from '#utils/array.mjs';
import { leftHandAssign } from '#utils/generators.mjs';
+import { resolveSpecifier } from '#utils/loaders.mjs';
+import {
+ CONFIGURED_PLUGINS,
+ PLUGIN_LISTS,
+} from '#utils/markdown/constants.mjs';
+import { resolveMarkdown } from '#utils/markdown/plugins.mjs';
import { deepMerge } from '#utils/misc.mjs';
import { DEFAULT_CHUNK_SIZE, DEFAULT_MAX_THREADS } from './constants.mjs';
@@ -64,6 +68,7 @@ export const getDefaultConfig = (generators, config) =>
// from, so generators render single-version output.
changelog: [],
pathsToCopy: ['assets', 'public', 'static'],
+ markdown: { remarkPlugins: [], rehypePlugins: [], recmaPlugins: [] },
},
// The number of wasm memory instances is severely limited on
@@ -79,21 +84,26 @@ export const getDefaultConfig = (generators, config) =>
);
/**
- * Resolves an `extends` entry of a configuration file into an importable
- * URL: relative paths resolve against the configuration file, anything else
- * resolves as a package import specifier (e.g. `@node-core/doc-kit/config`).
+ * Resolves the Markdown plugins of a configuration (in `global`, and each
+ * generator's section) from the file declaring them.
*
- * @param {string} specifier - The `extends` entry
- * @param {string} configFilePath - The configuration file it appears in
- * @returns {string} A `file:` URL to import
+ * @param {Partial} config - The configuration
+ * @param {string} filePath - The file declaring it
+ * @returns {Partial}
*/
-const resolveConfigExtends = (specifier, configFilePath) => {
- if (specifier.startsWith('.') || isAbsolute(specifier)) {
- return pathToFileURL(resolve(dirname(configFilePath), specifier)).href;
- }
+const resolveMarkdownPlugins = (config, filePath) =>
+ Object.fromEntries(
+ Object.entries(config).map(([name, value]) => {
+ if (value?.markdown) {
+ const label = `${name}.markdown`;
+ const markdown = resolveMarkdown(value.markdown, label, filePath);
- return pathToFileURL(createRequire(configFilePath).resolve(specifier)).href;
-};
+ return [name, { ...value, markdown }];
+ }
+
+ return [name, value];
+ })
+ );
/**
* Loads an explicit configuration file or searches for one using cosmiconfig.
@@ -112,15 +122,63 @@ export const loadConfigFile = async filePath => {
let { extends: presets, ...config } = result.config ?? {};
+ config = resolveMarkdownPlugins(config, result.filepath);
+
for (const preset of enforceArray(presets ?? []).toReversed()) {
- const module = await import(resolveConfigExtends(preset, result.filepath));
+ const url = resolveSpecifier(preset, result.filepath);
+ const module = await import(url);
- config = deepMerge(module.default ?? module, config);
+ // A preset's Markdown plugins resolve from the preset
+ config = deepMerge(
+ resolveMarkdownPlugins(module.default ?? module, fileURLToPath(url)),
+ config
+ );
}
return config;
};
+/**
+ * Returns the Markdown plugins a generator takes: for each list of its
+ * pipeline with a `'...'`, the global ones, then its own. Generators rendering
+ * Markdown (with rehype or recma plugins) skip the global remark plugins,
+ * which already ran in `ast`. Its own plugins it doesn't take are ignored with
+ * a warning.
+ *
+ * @param {GeneratorMetadata} generator - The generator
+ * @param {Partial} [markdown] - Its own plugins
+ * @param {import('./types').GlobalConfiguration} global - The global configuration
+ * @returns {Partial | undefined}
+ */
+const configureMarkdown = (generator, markdown = {}, global) => {
+ const pipeline = generator.markdown ?? {};
+ const renders = Boolean(pipeline.rehypePlugins || pipeline.recmaPlugins);
+ const configured = {};
+
+ for (const list of PLUGIN_LISTS) {
+ if (!pipeline[list]?.includes(CONFIGURED_PLUGINS)) {
+ if (markdown[list]?.length > 0) {
+ logger.warn(
+ `Ignoring \`${generator.name}.markdown.${list}\`: ` +
+ `\`${generator.name}\` does not take them.`
+ );
+ }
+
+ continue;
+ }
+
+ const plugins = markdown[list] ?? [];
+
+ if (list === 'remarkPlugins' && renders) {
+ configured[list] = plugins;
+ } else {
+ configured[list] = [...global.markdown[list], ...plugins];
+ }
+ }
+
+ return generator.markdown && configured;
+};
+
/**
* Transforms configuration values that need async processing or coercion.
* Only processes values that haven't been transformed yet (strings for changelog/index, non-coerced versions).
@@ -232,12 +290,20 @@ export const createRunConfiguration = async options => {
// Now assign to each generator config (they inherit from global)
await Promise.all(
- [...generators.values()].map(async ({ name }) => {
- const value = merged[name];
+ [...generators.values()].map(async generator => {
+ const value = merged[generator.name];
// Transform generator-specific overrides
await transformConfig(value);
+ // A generator's own Markdown plugins add to the global ones. Set even
+ // when undefined, so the assignment below doesn't copy the global ones
+ value.markdown = configureMarkdown(
+ generator,
+ value.markdown,
+ merged.global
+ );
+
// Assign from global (this populates missing values from global)
leftHandAssign(value, merged.global);
})
diff --git a/packages/core/src/utils/configuration/types.d.ts b/packages/core/src/utils/configuration/types.d.ts
index b9b4b6299..2437302bc 100644
--- a/packages/core/src/utils/configuration/types.d.ts
+++ b/packages/core/src/utils/configuration/types.d.ts
@@ -62,4 +62,30 @@ export type GlobalConfiguration = {
// Git ref (i.e. HEAD)
ref: string;
+
+ // Markdown plugins added to the pipeline of each generator processing
+ // Markdown. A generator's own `markdown` adds to them, rather than
+ // replacing them.
+ markdown: MarkdownConfiguration;
+};
+
+// A plugin's module specifier (a package name, or a path relative to the file
+// declaring it), alone or with its options
+export type PluginEntry = string | [specifier: string, options?: unknown];
+
+export type MarkdownConfiguration = {
+ // remark plugins, run on Markdown syntax trees
+ remarkPlugins: Array;
+
+ // rehype plugins, run on HTML syntax trees
+ rehypePlugins: Array;
+
+ // recma plugins, run on JavaScript syntax trees
+ recmaPlugins: Array;
+};
+
+// A generator's Markdown pipeline, where each list takes the configured
+// plugins in place of its `'...'`
+export type MarkdownPipeline = {
+ [List in keyof MarkdownConfiguration]?: Array;
};
diff --git a/packages/core/src/utils/loaders.mjs b/packages/core/src/utils/loaders.mjs
index 0d350c94b..b2a7fe78a 100644
--- a/packages/core/src/utils/loaders.mjs
+++ b/packages/core/src/utils/loaders.mjs
@@ -1,5 +1,6 @@
import { readFile } from 'node:fs/promises';
-import { extname, isAbsolute, join } from 'node:path';
+import { createRequire } from 'node:module';
+import { dirname, extname, isAbsolute, join, resolve } from 'node:path';
import { pathToFileURL } from 'node:url';
/**
@@ -28,6 +29,27 @@ export const loadFromURL = async url => {
}
};
+/**
+ * Resolves an import specifier into a `file:` URL: relative paths resolve
+ * against the file it appears in, anything else as a package import
+ * specifier (e.g. `@node-core/doc-kit/config`).
+ *
+ * @param {string} specifier - A path, a package specifier, or a `file:` URL
+ * @param {string} filePath - The file it appears in
+ * @returns {string} A `file:` URL to import
+ */
+export const resolveSpecifier = (specifier, filePath) => {
+ if (specifier.startsWith('file:')) {
+ return specifier;
+ }
+
+ if (specifier.startsWith('.') || isAbsolute(specifier)) {
+ return pathToFileURL(resolve(dirname(filePath), specifier)).href;
+ }
+
+ return pathToFileURL(createRequire(filePath).resolve(specifier)).href;
+};
+
/**
* Dynamically imports a module from a URL, using JSON import assertion if applicable.
*
diff --git a/packages/core/src/utils/markdown/__tests__/plugins.test.mjs b/packages/core/src/utils/markdown/__tests__/plugins.test.mjs
new file mode 100644
index 000000000..3cd4517ff
--- /dev/null
+++ b/packages/core/src/utils/markdown/__tests__/plugins.test.mjs
@@ -0,0 +1,197 @@
+import assert from 'node:assert/strict';
+import { mkdtempSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { describe, it } from 'node:test';
+import { pathToFileURL } from 'node:url';
+
+import { getMarkdownPlugins, loadMarkdownPlugins } from '../plugins.mjs';
+
+const dir = mkdtempSync(join(tmpdir(), 'doc-kit-markdown-plugins-'));
+
+/**
+ * Writes a module, returning its URL, as configuration resolves it.
+ *
+ * @param {string} name
+ * @param {string} source
+ */
+const writeModule = (name, source) => {
+ writeFileSync(join(dir, name), source);
+
+ return pathToFileURL(join(dir, name)).href;
+};
+
+const plugin = writeModule('plugin.mjs', 'export default function plugin() {}');
+const other = writeModule('other.mjs', 'export default function other() {}');
+const list = writeModule(
+ 'list.mjs',
+ `import plugin from './plugin.mjs';
+ export default [[plugin, { from: 'list' }]];`
+);
+const preset = writeModule(
+ 'preset.mjs',
+ `import plugin from './plugin.mjs';
+ export default { plugins: [plugin] };`
+);
+const setUp = writeModule(
+ 'set-up.mjs',
+ `export async function load(options) {
+ return Object.assign(function setUp() {}, { options });
+ }`
+);
+
+const { default: pluginFunction } = await import(plugin);
+const { default: otherFunction } = await import(other);
+
+/**
+ * A generator whose Markdown pipeline only has the configured plugins.
+ *
+ * @param {string} name
+ */
+const configurable = name => ({
+ name,
+ markdown: {
+ remarkPlugins: ['...'],
+ rehypePlugins: ['...'],
+ recmaPlugins: ['...'],
+ },
+});
+
+describe('loadMarkdownPlugins', () => {
+ it('loads plugins as `processor.use()` takes them', async () => {
+ await loadMarkdownPlugins(configurable('formats'), {
+ remarkPlugins: [plugin, [other, { a: 1 }]],
+ rehypePlugins: [list],
+ recmaPlugins: [preset],
+ });
+
+ assert.deepStrictEqual(getMarkdownPlugins('formats'), {
+ remarkPlugins: [[pluginFunction], [otherFunction, { a: 1 }]],
+ // A list is used as a preset
+ rehypePlugins: [{ plugins: [[pluginFunction, { from: 'list' }]] }],
+ recmaPlugins: [{ plugins: [pluginFunction] }],
+ });
+ });
+
+ it('loads a plugin through its `load`, given the options', async () => {
+ await loadMarkdownPlugins(configurable('setting-up'), {
+ rehypePlugins: [[setUp, { ready: true }]],
+ });
+
+ const [[setUpPlugin, ...options]] =
+ getMarkdownPlugins('setting-up').rehypePlugins;
+
+ assert.equal(setUpPlugin.name, 'setUp');
+ assert.deepStrictEqual(setUpPlugin.options, { ready: true });
+ assert.deepStrictEqual(options, []);
+ });
+
+ it('puts the configured plugins in place of `...`', async () => {
+ await loadMarkdownPlugins(
+ {
+ name: 'pipeline',
+ markdown: {
+ remarkPlugins: [other, '...', [other, { last: true }]],
+ // Without `...`, a list takes no configured plugins
+ rehypePlugins: [other],
+ },
+ },
+ { remarkPlugins: [plugin], rehypePlugins: [plugin] }
+ );
+
+ assert.deepStrictEqual(getMarkdownPlugins('pipeline'), {
+ remarkPlugins: [
+ [otherFunction],
+ [pluginFunction],
+ [otherFunction, { last: true }],
+ ],
+ rehypePlugins: [[otherFunction]],
+ recmaPlugins: [],
+ });
+
+ // The generator's own plugins, without the configured ones
+ assert.deepStrictEqual(getMarkdownPlugins('pipeline', false), {
+ remarkPlugins: [[otherFunction], [otherFunction, { last: true }]],
+ rehypePlugins: [[otherFunction]],
+ recmaPlugins: [],
+ });
+ });
+
+ it('configures the plugins it has, rather than running them twice', async () => {
+ await loadMarkdownPlugins(
+ {
+ name: 'configuring',
+ markdown: {
+ rehypePlugins: [[plugin, { list: [1], nested: { a: 1 } }], '...'],
+ },
+ },
+ {
+ rehypePlugins: [
+ [plugin, { list: [2], nested: { b: 2 } }],
+ // As a generator's own plugins add to the global ones
+ [setUp, { list: [1] }],
+ other,
+ [setUp, { list: [2] }],
+ ],
+ }
+ );
+
+ const [own, [setUpPlugin], ...rest] =
+ getMarkdownPlugins('configuring').rehypePlugins;
+
+ // Options merge: lists add up, and objects merge
+ assert.deepStrictEqual(own, [
+ pluginFunction,
+ { list: [1, 2], nested: { a: 1, b: 2 } },
+ ]);
+ assert.deepStrictEqual(setUpPlugin.options, { list: [1, 2] });
+ assert.deepStrictEqual(rest, [[otherFunction]]);
+
+ // Its own plugins are configured without the configured ones too
+ assert.deepStrictEqual(getMarkdownPlugins('configuring', false), {
+ remarkPlugins: [],
+ rehypePlugins: [own],
+ recmaPlugins: [],
+ });
+ });
+
+ it('rejects the plugins it cannot import', async () => {
+ await assert.rejects(
+ loadMarkdownPlugins(configurable('missing'), {
+ remarkPlugins: [`${plugin}-missing`],
+ }),
+ { code: 'ERR_MODULE_NOT_FOUND' }
+ );
+ });
+
+ it('loads each generator its own plugins, once', async () => {
+ const one = configurable('one');
+ const markdown = { rehypePlugins: [[plugin, { for: 'one' }]] };
+
+ await loadMarkdownPlugins(one, markdown);
+ await loadMarkdownPlugins(configurable('two'), {
+ rehypePlugins: [[plugin, { for: 'two' }]],
+ });
+
+ const plugins = getMarkdownPlugins('one');
+
+ assert.deepStrictEqual(plugins.rehypePlugins, [
+ [pluginFunction, { for: 'one' }],
+ ]);
+ assert.deepStrictEqual(getMarkdownPlugins('two').rehypePlugins, [
+ [pluginFunction, { for: 'two' }],
+ ]);
+
+ // As workers do for each task, from a copy of the configuration
+ await loadMarkdownPlugins(one, structuredClone(markdown));
+
+ assert.equal(getMarkdownPlugins('one'), plugins);
+
+ // A generator without a pipeline has none to load
+ await loadMarkdownPlugins({ name: 'none' }, markdown);
+
+ assert.throws(() => getMarkdownPlugins('none'), {
+ message: 'The Markdown pipeline of "none" is not loaded on this thread',
+ });
+ });
+});
diff --git a/packages/core/src/utils/markdown/__tests__/processor.test.mjs b/packages/core/src/utils/markdown/__tests__/processor.test.mjs
new file mode 100644
index 000000000..4d6fb8f57
--- /dev/null
+++ b/packages/core/src/utils/markdown/__tests__/processor.test.mjs
@@ -0,0 +1,105 @@
+import assert from 'node:assert/strict';
+import { mkdtempSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { beforeEach, describe, it } from 'node:test';
+import { pathToFileURL } from 'node:url';
+
+import { loadMarkdownPlugins } from '../plugins.mjs';
+import { getProcessor } from '../processor.mjs';
+
+const dir = mkdtempSync(join(tmpdir(), 'doc-kit-processor-'));
+
+// What the plugins below saw, in the order they ran
+const seen = [];
+
+globalThis.seenByPlugins = seen;
+
+/**
+ * Writes a plugin recording the type of the first node of its tree, and
+ * whether it processes MDX, returning its URL.
+ *
+ * @param {string} name
+ */
+const recorder = name => {
+ writeFileSync(
+ join(dir, `${name}.mjs`),
+ `export default function ${name}() {
+ const mdx = this.data('mdx') ? ' (MDX)' : '';
+
+ return tree => {
+ globalThis.seenByPlugins.push('${name}: ' + tree.children[0].type + mdx);
+ };
+ }`
+ );
+
+ return pathToFileURL(join(dir, `${name}.mjs`)).href;
+};
+
+// A pipeline turning Markdown into HTML, with doc-kit's Markdown syntax
+await loadMarkdownPlugins(
+ {
+ name: 'to-html',
+ markdown: {
+ remarkPlugins: [
+ import.meta.resolve('remark-parse'),
+ import.meta.resolve('#plugins/type-annotations/remark.mjs'),
+ import.meta.resolve('remark-gfm'),
+ '...',
+ ],
+ rehypePlugins: [
+ import.meta.resolve('remark-rehype'),
+ '...',
+ import.meta.resolve('rehype-stringify'),
+ ],
+ },
+ },
+ { remarkPlugins: [recorder('remark')], rehypePlugins: [recorder('rehype')] }
+);
+
+describe('getProcessor', () => {
+ beforeEach(() => {
+ seen.length = 0;
+ });
+
+ it('runs the plugins of each syntax tree in turn', async () => {
+ const html = await getProcessor('to-html').process('# Hi ~~there~~');
+
+ assert.equal(String(html), 'Hi there
');
+ assert.deepStrictEqual(seen, ['remark: heading', 'rehype: element']);
+ });
+
+ it('processes MDX, telling the plugins so', async () => {
+ const processor = getProcessor('to-html', { mdx: true });
+
+ // `{...}` is an expression in MDX, and a type annotation in Markdown
+ const [paragraph] = processor.parse('A {string}').children;
+
+ assert.equal(paragraph.children[1].type, 'mdxTextExpression');
+ assert.equal(
+ getProcessor('to-html').parse('A {string}').children[0].children[1].type,
+ 'typeAnnotation'
+ );
+
+ await processor.process('');
+
+ assert.deepStrictEqual(seen, [
+ 'remark: mdxJsxFlowElement (MDX)',
+ 'rehype: element (MDX)',
+ ]);
+ });
+
+ it('runs without the configured plugins, if asked', async () => {
+ await getProcessor('to-html', { configured: false }).process('# Hi');
+
+ assert.deepStrictEqual(seen, []);
+ });
+
+ it('gives the same processor for the same pipeline', () => {
+ assert.equal(getProcessor('to-html'), getProcessor('to-html'));
+ assert.notEqual(
+ getProcessor('to-html'),
+ getProcessor('to-html', { mdx: true })
+ );
+ });
+});
diff --git a/packages/core/src/utils/markdown/constants.mjs b/packages/core/src/utils/markdown/constants.mjs
new file mode 100644
index 000000000..dc25c88c7
--- /dev/null
+++ b/packages/core/src/utils/markdown/constants.mjs
@@ -0,0 +1,7 @@
+'use strict';
+
+// The plugin lists of a Markdown pipeline, one for each syntax tree
+export const PLUGIN_LISTS = ['remarkPlugins', 'rehypePlugins', 'recmaPlugins'];
+
+// Marks where a list of a generator's pipeline takes the configured plugins
+export const CONFIGURED_PLUGINS = '...';
diff --git a/packages/core/src/utils/markdown/plugins.mjs b/packages/core/src/utils/markdown/plugins.mjs
new file mode 100644
index 000000000..effaa4134
--- /dev/null
+++ b/packages/core/src/utils/markdown/plugins.mjs
@@ -0,0 +1,232 @@
+'use strict';
+
+import { fileURLToPath } from 'node:url';
+
+import { getGeneratorModule } from '#generators/loader.mjs';
+import { enforceArray } from '#utils/array.mjs';
+import { resolveSpecifier } from '#utils/loaders.mjs';
+import { isPlainObject } from '#utils/misc.mjs';
+
+import { CONFIGURED_PLUGINS, PLUGIN_LISTS } from './constants.mjs';
+
+/**
+ * The plugins of a Markdown pipeline, as `processor.use()` takes them.
+ *
+ * @typedef {Record<'remarkPlugins' | 'rehypePlugins' | 'recmaPlugins', import('unified').PluggableList>} MarkdownPlugins
+ */
+
+// The pipeline each generator runs on this thread, by its name
+const loadedPipelines = new Map();
+
+/**
+ * Resolves the module specifiers of Markdown plugins (in the `markdown`
+ * option, or a generator's pipeline) from the file declaring them into
+ * `file:` URLs, which any thread can import.
+ *
+ * @param {import('../configuration/types').MarkdownPipeline} markdown - The plugins
+ * @param {string} label - Where they are declared, for errors (e.g. `global.markdown`)
+ * @param {string} filePath - The file declaring them
+ * @returns {import('../configuration/types').MarkdownPipeline}
+ */
+export const resolveMarkdown = (markdown, label, filePath) => {
+ const resolved = { ...markdown };
+
+ for (const list of PLUGIN_LISTS) {
+ resolved[list] &&= enforceArray(markdown[list]).map((entry, index) => {
+ if (entry === CONFIGURED_PLUGINS) {
+ return entry;
+ }
+
+ const [specifier, ...options] = enforceArray(entry);
+
+ if (typeof specifier !== 'string') {
+ throw new TypeError(
+ `${label}.${list}[${index}] in ${filePath} must be a module ` +
+ 'specifier (a package name or a path), as Markdown is processed ' +
+ 'in worker threads, which import the plugins themselves.'
+ );
+ }
+
+ const url = resolveSpecifier(specifier, filePath);
+
+ return options.length === 0 ? url : [url, ...options];
+ });
+ }
+
+ return resolved;
+};
+
+/**
+ * Resolves the module specifiers of a generator's Markdown pipeline from the
+ * module it was loaded from.
+ *
+ * @param {GeneratorMetadata} generator - The generator
+ * @returns {import('../configuration/types').MarkdownPipeline}
+ */
+const resolveMarkdownPipeline = generator => {
+ const module = getGeneratorModule(generator);
+
+ return resolveMarkdown(
+ generator.markdown,
+ `${generator.name}.markdown`,
+ module && fileURLToPath(module)
+ );
+};
+
+/**
+ * Merges options into others: arrays add up, plain objects merge, and other
+ * values replace the ones they are merged into.
+ *
+ * @param {unknown} options - The options merged into
+ * @param {unknown} added - The options merged
+ * @returns {unknown}
+ */
+const mergeOptions = (options, added) => {
+ if (Array.isArray(options) && Array.isArray(added)) {
+ return [...options, ...added];
+ }
+
+ if (!isPlainObject(options) || !isPlainObject(added)) {
+ return added;
+ }
+
+ const merged = { ...options };
+
+ for (const [key, value] of Object.entries(added)) {
+ merged[key] = mergeOptions(options[key], value);
+ }
+
+ return merged;
+};
+
+/**
+ * Builds a list of a pipeline, with the configured plugins in place of its
+ * `'...'`. A configured plugin the list already has, or added earlier, isn't
+ * added again: its options merge into the ones it has.
+ *
+ * @param {Array} [own] - The list
+ * @param {Array} [configured] - The configured plugins
+ * @returns {Array<{ entry: import('../configuration/types').PluginEntry, added: boolean }>}
+ */
+const configureList = (own = [], configured = []) => {
+ const plugins = own.map(entry => ({ entry, added: false }));
+ const added = [];
+
+ // Each plugin by its specifier: the first listing of it takes the options
+ const listed = new Map();
+
+ for (const plugin of plugins) {
+ const [specifier] = enforceArray(plugin.entry);
+
+ if (!listed.has(specifier)) {
+ listed.set(specifier, plugin);
+ }
+ }
+
+ for (const entry of configured) {
+ const [specifier, options] = enforceArray(entry);
+ const plugin = listed.get(specifier);
+
+ if (!plugin) {
+ const addedPlugin = { entry, added: true };
+
+ added.push(addedPlugin);
+ listed.set(specifier, addedPlugin);
+ } else if (options !== undefined) {
+ const [, listedOptions] = enforceArray(plugin.entry);
+
+ plugin.entry = [specifier, mergeOptions(listedOptions, options)];
+ }
+ }
+
+ return plugins.flatMap(plugin =>
+ plugin.entry === CONFIGURED_PLUGINS ? added : [plugin]
+ );
+};
+
+/**
+ * Imports a plugin as `processor.use()` takes it. Its module default-exports
+ * the plugin, a list of plugins, or a preset, or exports an async `load`,
+ * taking the plugin's options and returning it, for setup unified plugins
+ * can't do themselves.
+ *
+ * @param {import('../configuration/types').PluginEntry} entry - The plugin
+ * @returns {Promise}
+ */
+const importPlugin = async entry => {
+ const [specifier, ...options] = enforceArray(entry);
+ const { default: plugin, load } = await import(specifier);
+
+ if (typeof load === 'function') {
+ return [await load(...options)];
+ }
+
+ if (typeof plugin === 'function') {
+ return [plugin, ...options];
+ }
+
+ // A list is used as a preset, not to be taken for a [plugin, options] pair
+ return Array.isArray(plugin) ? { plugins: plugin } : plugin;
+};
+
+/**
+ * Loads a generator's Markdown pipeline on the current thread, with the
+ * plugins configured for it. Loading the same configuration again does
+ * nothing.
+ *
+ * @param {GeneratorMetadata} generator - The generator
+ * @param {Partial} [markdown] - The plugins configured for it
+ * @returns {Promise}
+ */
+export const loadMarkdownPlugins = async (generator, markdown = {}) => {
+ if (!generator.markdown) {
+ return;
+ }
+
+ const key = JSON.stringify(markdown);
+ const loaded = loadedPipelines.get(generator.name);
+
+ if (loaded?.generator === generator && loaded.key === key) {
+ return;
+ }
+
+ const pipeline = resolveMarkdownPipeline(generator);
+ const configured = {};
+ const own = {};
+
+ // The lists import at once, so a slow plugin (Shiki) doesn't hold the rest
+ const loading = PLUGIN_LISTS.map(async list => {
+ const plugins = configureList(pipeline[list], markdown[list]);
+ const imports = plugins.map(({ entry }) => importPlugin(entry));
+
+ const imported = await Promise.all(imports);
+
+ configured[list] = imported;
+ own[list] = imported.filter((_, index) => !plugins[index].added);
+ });
+
+ await Promise.all(loading);
+
+ loadedPipelines.set(generator.name, { generator, key, configured, own });
+};
+
+/**
+ * Gets the plugins of a generator's Markdown pipeline, as loaded on the
+ * current thread: with the configured plugins, or only its own (with their
+ * configured options).
+ *
+ * @param {string} generator - The name of the generator
+ * @param {boolean} [configured] - Whether with the configured plugins
+ * @returns {MarkdownPlugins}
+ */
+export const getMarkdownPlugins = (generator, configured = true) => {
+ const loaded = loadedPipelines.get(generator);
+
+ if (!loaded) {
+ throw new Error(
+ `The Markdown pipeline of "${generator}" is not loaded on this thread`
+ );
+ }
+
+ return configured ? loaded.configured : loaded.own;
+};
diff --git a/packages/core/src/utils/markdown/processor.mjs b/packages/core/src/utils/markdown/processor.mjs
new file mode 100644
index 000000000..5fbaf01c5
--- /dev/null
+++ b/packages/core/src/utils/markdown/processor.mjs
@@ -0,0 +1,60 @@
+'use strict';
+
+import remarkMdx from 'remark-mdx';
+import { unified } from 'unified';
+
+import { PLUGIN_LISTS } from './constants.mjs';
+import { getMarkdownPlugins } from './plugins.mjs';
+
+// The processors of each loaded pipeline, of Markdown and of MDX
+const processors = new WeakMap();
+
+/**
+ * Creates the processor of a pipeline: its remark plugins, then its rehype
+ * ones, then its recma ones. A processor of MDX knows its syntax too, and
+ * tells plugins it processes MDX through `this.data('mdx')`.
+ *
+ * @param {import('./plugins.mjs').MarkdownPlugins} plugins - The pipeline
+ * @param {boolean} mdx - Whether it processes MDX
+ * @returns {import('unified').Processor}
+ */
+const createProcessor = (plugins, mdx) => {
+ const processor = unified().data('mdx', mdx);
+
+ if (mdx) {
+ processor.use(remarkMdx);
+ }
+
+ for (const list of PLUGIN_LISTS) {
+ processor.use(plugins[list]);
+ }
+
+ return processor;
+};
+
+/**
+ * Gets the processor of a generator's Markdown pipeline, as loaded on the
+ * current thread (see `loadMarkdownPlugins`).
+ *
+ * @param {string} generator - The name of the generator
+ * @param {Object} [options]
+ * @param {boolean} [options.mdx] - Whether it processes MDX
+ * @param {boolean} [options.configured] - Whether it runs the configured
+ * plugins, rather than only the generator's own
+ * @returns {import('unified').Processor}
+ */
+export const getProcessor = (
+ generator,
+ { mdx = false, configured = true } = {}
+) => {
+ const plugins = getMarkdownPlugins(generator, configured);
+
+ if (!processors.has(plugins)) {
+ processors.set(plugins, {});
+ }
+
+ const created = processors.get(plugins);
+ const syntax = mdx ? 'mdx' : 'markdown';
+
+ return (created[syntax] ??= createProcessor(plugins, mdx));
+};
diff --git a/packages/core/src/utils/remark-shiki.mjs b/packages/core/src/utils/remark-shiki.mjs
deleted file mode 100644
index 6fe234ea2..000000000
--- a/packages/core/src/utils/remark-shiki.mjs
+++ /dev/null
@@ -1,29 +0,0 @@
-'use strict';
-
-import rehypeStringify from 'rehype-stringify';
-import remarkParse from 'remark-parse';
-import remarkRehype from 'remark-rehype';
-import { unified } from 'unified';
-
-import syntaxHighlighter from './highlighter.mjs';
-import { lazy } from './misc.mjs';
-import { rehypeOptions } from './remark.mjs';
-
-/**
- * Retrieves an instance of Remark configured to output stringified HTML code
- * including parsing Code Boxes with syntax highlighting.
- *
- * This lives apart from `./remark.mjs` because importing it loads Shiki (every
- * grammar plus the regex engine, ~250MB per process). Only the generators that
- * highlight code — `legacy-html` here — should pay for that.
- */
-export const getRemarkRehypeWithShiki = lazy(() =>
- unified()
- .use(remarkParse)
- // legacy-html gets the minimal (unhighlighted) type rendering
- .use(remarkRehype, rehypeOptions)
- // This is a custom ad-hoc within the Shiki Rehype plugin, used to highlight code
- // and transform them into HAST nodes
- .use(syntaxHighlighter)
- .use(rehypeStringify, { allowDangerousHtml: true })
-);
diff --git a/packages/core/src/utils/remark.mjs b/packages/core/src/utils/remark.mjs
deleted file mode 100644
index 022e55b20..000000000
--- a/packages/core/src/utils/remark.mjs
+++ /dev/null
@@ -1,90 +0,0 @@
-'use strict';
-
-import rehypeStringify from 'rehype-stringify';
-import remarkGfm from 'remark-gfm';
-import remarkMdx from 'remark-mdx';
-import remarkParse from 'remark-parse';
-import remarkRehype from 'remark-rehype';
-import remarkStringify from 'remark-stringify';
-import { unified } from 'unified';
-
-import { lazy } from './misc.mjs';
-import { typeAnnotationToHast } from './type-annotations/hast.mjs';
-import remarkTypeAnnotations from './type-annotations/remark.mjs';
-
-// Nothing in this module loads Shiki: the `ast` and `metadata` stages (and
-// every worker that runs them) import it, and none of them highlight code.
-// The highlighting pipeline lives in `./remark-shiki.mjs`.
-
-/**
- * Renders an MDX JSX element as just its children, so the surrounding prose
- * still renders in HTML-string output.
- *
- * @param {import('mdast-util-to-hast').State} state
- * @param {import('unist').Parent} node
- */
-const mdxElementToChildren = (state, node) => state.all(node);
-
-/**
- * Drops a node from HTML-string output.
- */
-const dropNode = () => undefined;
-
-/**
- * The `remark-rehype` options shared by the HTML-string pipelines.
- *
- * Existing HTML nodes pass through untouched (they were created during the
- * rehype process), and dangerous HTML is allowed since the Markdown sources
- * are trusted. The MDX node types cannot be rendered to an HTML string (that
- * is the React generators' job): JSX elements degrade to their children so the
- * surrounding prose still renders, and expressions/ESM are dropped.
- *
- * @type {import('remark-rehype').Options}
- */
-export const rehypeOptions = {
- allowDangerousHtml: true,
- passThrough: ['element'],
- handlers: {
- typeAnnotation: typeAnnotationToHast,
- mdxJsxTextElement: mdxElementToChildren,
- mdxJsxFlowElement: mdxElementToChildren,
- mdxFlowExpression: dropNode,
- mdxTextExpression: dropNode,
- mdxjsEsm: dropNode,
- },
-};
-
-/**
- * Retrieves an instance of Remark configured to parse GFM (GitHub Flavored Markdown)
- * plus `{...}` type annotations (see `./type-annotations`), which only exist
- * in non-MDX files — the MDX pipeline below never registers them.
- */
-export const getRemark = lazy(() =>
- unified()
- .use(remarkParse)
- .use(remarkTypeAnnotations)
- .use(remarkGfm)
- .use(remarkStringify)
-);
-
-/**
- * Retrieves an instance of Remark configured to parse MDX (JSX-in-Markdown).
- *
- * Unlike {@link getRemark}, this understands ` className
// to attribute the current language of the element
@@ -38,22 +37,8 @@ function isCodeBlock(node) {
);
}
-export const highlighter = await createHighlighter({
- // riscv64 with sv39 has limited virtual memory space, where creating
- // too many (>20) wasm memory instances fails.
- // https://github.com/nodejs/node/pull/60591
- //
- // The wasm highlighter is currently not compatible with big endian.
- // https://github.com/nodejs/node/pull/62512#issuecomment-4243469950
- wasm: process.arch !== 'riscv64' && endianness() === 'LE',
-});
-
/**
* Creates a HAST transformer for Shiki which is used for transforming our codeboxes
- *
- * @deprecated This is used only for the legacy-html generator, please use `@node-core/rehype-shiki` directly instead.
- *
- * @type {import('unified').Plugin}
*/
export default function rehypeShikiji() {
/**
diff --git a/packages/node-legacy/src/legacy-html/utils/__tests__/buildContent.test.mjs b/packages/node-legacy/src/legacy-html/utils/__tests__/buildContent.test.mjs
index edb1d149e..5d0bc6a70 100644
--- a/packages/node-legacy/src/legacy-html/utils/__tests__/buildContent.test.mjs
+++ b/packages/node-legacy/src/legacy-html/utils/__tests__/buildContent.test.mjs
@@ -3,10 +3,17 @@
import assert from 'node:assert/strict';
import { before, describe, it } from 'node:test';
+import { loadGenerator } from '@doc-kit/core/generators/loader.mjs';
import { setConfig } from '@doc-kit/core/utils/configuration/index.mjs';
+import { loadMarkdownPlugins } from '@doc-kit/core/utils/markdown/plugins.mjs';
import buildContent from '../buildContent.mjs';
+// The content is rendered with the pipeline of `legacy-html`
+await loadMarkdownPlugins(
+ await loadGenerator(import.meta.resolve('../../index.mjs'))
+);
+
const createEntry = slug => {
const text = 'DEP0001: deprecated API';
const heading = {
diff --git a/packages/node-legacy/src/legacy-html/utils/buildContent.mjs b/packages/node-legacy/src/legacy-html/utils/buildContent.mjs
index 37abac651..ae0cad1a0 100644
--- a/packages/node-legacy/src/legacy-html/utils/buildContent.mjs
+++ b/packages/node-legacy/src/legacy-html/utils/buildContent.mjs
@@ -5,8 +5,8 @@ import {
GITHUB_BLOB_URL,
populate,
} from '@doc-kit/core/utils/configuration/templates.mjs';
+import { getProcessor } from '@doc-kit/core/utils/markdown/processor.mjs';
import { UNIST } from '@doc-kit/core/utils/queries/index.mjs';
-import { getRemarkRehypeWithShiki as remark } from '@doc-kit/core/utils/remark-shiki.mjs';
import { h as createElement } from 'hastscript';
import { u as createTree } from 'unist-builder';
import { SKIP, visit } from 'unist-util-visit';
@@ -80,7 +80,7 @@ const buildStability = ({ children, data }, index, parent) => {
* @param {import('@doc-kit/core/generators/metadata/types').ChangeEntry} change
*/
const createHistoryTableRow = ({ version: changeVersions, description }) => {
- const descriptionNode = remark().parse(description);
+ const descriptionNode = getProcessor('legacy-html').parse(description);
return createElement('tr', [
createElement(
@@ -241,8 +241,9 @@ export default (headNodes, metadataEntries) => {
})
);
- const processedNodes = remark().runSync(parsedNodes);
+ const processor = getProcessor('legacy-html');
+ const processedNodes = processor.runSync(parsedNodes);
// Stringifies the processed nodes to return the final Markdown content
- return remark().stringify(processedNodes);
+ return processor.stringify(processedNodes);
};
diff --git a/packages/core/src/utils/__tests__/remark.test.mjs b/packages/node-legacy/src/legacy-json/__tests__/markdown.test.mjs
similarity index 69%
rename from packages/core/src/utils/__tests__/remark.test.mjs
rename to packages/node-legacy/src/legacy-json/__tests__/markdown.test.mjs
index 40250885d..f5f271443 100644
--- a/packages/core/src/utils/__tests__/remark.test.mjs
+++ b/packages/node-legacy/src/legacy-json/__tests__/markdown.test.mjs
@@ -1,11 +1,17 @@
import assert from 'node:assert/strict';
import { describe, it } from 'node:test';
-import { getRemarkRehype } from '../remark.mjs';
+import { loadGenerator } from '@doc-kit/core/generators/loader.mjs';
+import { loadMarkdownPlugins } from '@doc-kit/core/utils/markdown/plugins.mjs';
+import { getProcessor } from '@doc-kit/core/utils/markdown/processor.mjs';
-describe('getRemarkRehype', () => {
+await loadMarkdownPlugins(
+ await loadGenerator(import.meta.resolve('../index.mjs'))
+);
+
+describe('the Markdown pipeline of legacy-json', () => {
it('degrades MDX nodes instead of crashing rehype-stringify', () => {
- const processor = getRemarkRehype();
+ const processor = getProcessor('legacy-json');
const tree = {
type: 'root',
diff --git a/packages/node-legacy/src/legacy-json/index.mjs b/packages/node-legacy/src/legacy-json/index.mjs
index 2109e7a33..841e54659 100644
--- a/packages/node-legacy/src/legacy-json/index.mjs
+++ b/packages/node-legacy/src/legacy-json/index.mjs
@@ -1,5 +1,6 @@
'use strict';
+import { rehypeOptions } from '../utils/rehypeOptions.mjs';
import { generate, processChunk } from './generate.mjs';
/**
@@ -28,6 +29,15 @@ export default {
hasParallelProcessor: true,
+ // Renders the descriptions' Markdown into HTML. It takes no configured
+ // plugins, which would change the legacy output
+ markdown: {
+ rehypePlugins: [
+ ['remark-rehype', rehypeOptions],
+ ['rehype-stringify', { allowDangerousHtml: true }],
+ ],
+ },
+
generate,
processChunk,
};
diff --git a/packages/node-legacy/src/legacy-json/utils/buildSection.mjs b/packages/node-legacy/src/legacy-json/utils/buildSection.mjs
index 0c02d5291..5aad62039 100644
--- a/packages/node-legacy/src/legacy-json/utils/buildSection.mjs
+++ b/packages/node-legacy/src/legacy-json/utils/buildSection.mjs
@@ -1,7 +1,7 @@
import { enforceArray } from '@doc-kit/core/utils/array.mjs';
import { populate } from '@doc-kit/core/utils/configuration/templates.mjs';
import { buildHierarchy } from '@doc-kit/core/utils/hierarchy.mjs';
-import { getRemarkRehype as remark } from '@doc-kit/core/utils/remark.mjs';
+import { getProcessor } from '@doc-kit/core/utils/markdown/processor.mjs';
import { parseList } from '@doc-kit/core/utils/signature/parseList.mjs';
import { transformNodesToString } from '@doc-kit/core/utils/unist.mjs';
@@ -125,8 +125,9 @@ export const createSectionBuilder = () => {
return;
}
- const rendered = remark().stringify(
- remark().runSync({ type: 'root', children: nodes })
+ const processor = getProcessor('legacy-json');
+ const rendered = processor.stringify(
+ processor.runSync({ type: 'root', children: nodes })
);
section.shortDesc = section.desc || undefined;
diff --git a/packages/node-legacy/src/utils/rehypeOptions.mjs b/packages/node-legacy/src/utils/rehypeOptions.mjs
new file mode 100644
index 000000000..f2387ccaf
--- /dev/null
+++ b/packages/node-legacy/src/utils/rehypeOptions.mjs
@@ -0,0 +1,49 @@
+'use strict';
+
+import { typeAnnotationToHast } from '@doc-kit/core/plugins/type-annotations/hast.mjs';
+
+/**
+ * A `remark-rehype` handler, turning an mdast node into hast.
+ *
+ * @typedef {NonNullable} Handler
+ */
+
+/**
+ * Renders an MDX JSX element as just its children, so the surrounding prose
+ * still renders in HTML-string output.
+ *
+ * @type {Handler}
+ */
+const mdxElementToChildren = (state, node) => state.all(node);
+
+/**
+ * Drops a node from HTML-string output.
+ *
+ * @type {Handler}
+ */
+const dropNode = () => undefined;
+
+/**
+ * The `remark-rehype` options of the legacy generators, which render Markdown
+ * into HTML strings.
+ *
+ * Existing HTML nodes pass through untouched (they were created during the
+ * rehype process), and dangerous HTML is allowed since the Markdown sources
+ * are trusted. The MDX node types cannot be rendered to an HTML string (that
+ * is the React generators' job): JSX elements degrade to their children so the
+ * surrounding prose still renders, and expressions/ESM are dropped.
+ *
+ * @type {import('remark-rehype').Options}
+ */
+export const rehypeOptions = {
+ allowDangerousHtml: true,
+ passThrough: ['element'],
+ handlers: {
+ typeAnnotation: typeAnnotationToHast,
+ mdxJsxTextElement: mdxElementToChildren,
+ mdxJsxFlowElement: mdxElementToChildren,
+ mdxFlowExpression: dropNode,
+ mdxTextExpression: dropNode,
+ mdxjsEsm: dropNode,
+ },
+};
diff --git a/packages/react/src/html/__tests__/generate.test.mjs b/packages/react/src/html/__tests__/generate.test.mjs
index cc72980bb..814889c7f 100644
--- a/packages/react/src/html/__tests__/generate.test.mjs
+++ b/packages/react/src/html/__tests__/generate.test.mjs
@@ -5,7 +5,9 @@ import { join } from 'node:path';
import { describe, it } from 'node:test';
import { pathToFileURL } from 'node:url';
+import { loadGenerator } from '@doc-kit/core/generators/loader.mjs';
import { setConfig } from '@doc-kit/core/utils/configuration/index.mjs';
+import { loadMarkdownPlugins } from '@doc-kit/core/utils/markdown/plugins.mjs';
import { jsx, toJs } from 'estree-util-to-js';
import buildContent from '../../jsx-ast/utils/buildContent.mjs';
@@ -14,6 +16,11 @@ import { generate as chunk } from '../../section-pages/generate.mjs';
import { compile, createViteBundler } from '../bundlers/vite.mjs';
import { generate } from '../generate.mjs';
+// Pages are rendered with the pipeline of `jsx-ast`
+await loadMarkdownPlugins(
+ await loadGenerator(import.meta.resolve('../../jsx-ast/index.mjs'))
+);
+
/**
* Converts a page's JSX AST into the `{ data, headings, readingTime, content }`
* shape `html` consumes, mirroring the conversion the jsx-ast worker performs.
diff --git a/packages/react/src/html/utils/__tests__/config.test.mjs b/packages/react/src/html/utils/__tests__/config.test.mjs
index 016b4cf29..94e2e0c38 100644
--- a/packages/react/src/html/utils/__tests__/config.test.mjs
+++ b/packages/react/src/html/utils/__tests__/config.test.mjs
@@ -2,6 +2,7 @@ import assert from 'node:assert/strict';
import { describe, it, mock } from 'node:test';
import { setConfig } from '@doc-kit/core/utils/configuration/index.mjs';
+import { loadMarkdownPlugins } from '@doc-kit/core/utils/markdown/plugins.mjs';
import { SemVer } from 'semver';
mock.module('@node-core/rehype-shiki', {
@@ -11,6 +12,17 @@ mock.module('@node-core/rehype-shiki', {
{ name: 'typescript', aliases: ['ts'], displayName: 'TypeScript' },
{ name: 'python', displayName: 'Python' },
],
+ default: async () => ({}),
+ },
+});
+
+// The site's code is highlighted by the Shiki plugin of `jsx-ast`
+await loadMarkdownPlugins({
+ name: 'jsx-ast',
+ markdown: {
+ rehypePlugins: [
+ import.meta.resolve('@doc-kit/core/plugins/shiki/rehype.mjs'),
+ ],
},
});
diff --git a/packages/react/src/html/utils/config.mjs b/packages/react/src/html/utils/config.mjs
index 8a83dfada..512cbdbf2 100644
--- a/packages/react/src/html/utils/config.mjs
+++ b/packages/react/src/html/utils/config.mjs
@@ -1,5 +1,6 @@
'use strict';
+import { getHighlighter } from '@doc-kit/core/plugins/shiki/highlighter.mjs';
import getConfig from '@doc-kit/core/utils/configuration/index.mjs';
import { populate } from '@doc-kit/core/utils/configuration/templates.mjs';
import {
@@ -8,7 +9,6 @@ import {
} from '@doc-kit/core/utils/generators.mjs';
import { parseInline, renderAsHTML } from '@doc-kit/core/utils/inline.mjs';
import { omitKeys } from '@doc-kit/core/utils/misc.mjs';
-import { LANGS } from '@node-core/rehype-shiki';
import { getSortedHeadNodes } from '../../jsx-ast/utils/getSortedHeadNodes.mjs';
@@ -142,9 +142,11 @@ export function buildDocumentationIndex(input) {
* @returns {Array<[string[], string]>}
*/
export function buildLanguageDisplayNameMap() {
+ const { langs } = getHighlighter('jsx-ast');
+
return [
...new Map(
- LANGS.map(({ name, aliases = [], displayName }) => [
+ langs.map(({ name, aliases = [], displayName }) => [
name,
[[...aliases, name], displayName],
])
diff --git a/packages/react/src/jsx-ast/__tests__/generate.test.mjs b/packages/react/src/jsx-ast/__tests__/generate.test.mjs
index 40466dc41..c179fb962 100644
--- a/packages/react/src/jsx-ast/__tests__/generate.test.mjs
+++ b/packages/react/src/jsx-ast/__tests__/generate.test.mjs
@@ -1,12 +1,19 @@
import assert from 'node:assert/strict';
import { describe, it } from 'node:test';
+import { loadGenerator } from '@doc-kit/core/generators/loader.mjs';
import getConfig, {
setConfig,
} from '@doc-kit/core/utils/configuration/index.mjs';
+import { loadMarkdownPlugins } from '@doc-kit/core/utils/markdown/plugins.mjs';
import { generate, processChunk } from '../generate.mjs';
+// Pages are rendered with the pipeline of `jsx-ast`
+await loadMarkdownPlugins(
+ await loadGenerator(import.meta.resolve('../index.mjs'))
+);
+
const createEntry = (api, name, { stabilityIndex = '2' } = {}) => {
const heading = {
type: 'heading',
diff --git a/packages/react/src/jsx-ast/__tests__/markdown.test.mjs b/packages/react/src/jsx-ast/__tests__/markdown.test.mjs
new file mode 100644
index 000000000..48942b660
--- /dev/null
+++ b/packages/react/src/jsx-ast/__tests__/markdown.test.mjs
@@ -0,0 +1,101 @@
+import assert from 'node:assert/strict';
+import { mkdtempSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { describe, it } from 'node:test';
+import { pathToFileURL } from 'node:url';
+
+import { loadGenerator } from '@doc-kit/core/generators/loader.mjs';
+import { loadMarkdownPlugins } from '@doc-kit/core/utils/markdown/plugins.mjs';
+import { getProcessor } from '@doc-kit/core/utils/markdown/processor.mjs';
+import dedent from 'dedent';
+import { visit } from 'unist-util-visit';
+
+const jsxAst = await loadGenerator(import.meta.resolve('../index.mjs'));
+
+await loadMarkdownPlugins(jsxAst);
+
+const getCodeTabsAttributes = tree => {
+ const attributes = [];
+
+ visit(tree.body[0].expression, 'JSXElement', node => {
+ if (node.openingElement.name?.name === 'CodeTabs') {
+ attributes.push(
+ Object.fromEntries(
+ node.openingElement.attributes.map(attribute => [
+ attribute.name.name,
+ attribute.value?.value,
+ ])
+ )
+ );
+ }
+ });
+
+ return attributes;
+};
+
+describe('the Markdown pipeline of jsx-ast', () => {
+ it('preserves code tab display names when raw HTML is enabled', async () => {
+ const processor = getProcessor('jsx-ast');
+ const tree = await processor.run(
+ processor.parse(dedent`
+ raw html
+
+ \`\`\`cjs displayName="main.js"
+ console.log(1);
+ \`\`\`
+
+ \`\`\`cjs displayName="main.test.js"
+ console.log(2);
+ \`\`\`
+ `)
+ );
+
+ assert.deepEqual(getCodeTabsAttributes(tree), [
+ {
+ languages: 'cjs|cjs',
+ displayNames: 'main.js|main.test.js',
+ defaultTab: '0',
+ },
+ ]);
+ });
+
+ it('runs the configured rehype plugins before code is highlighted', async t => {
+ // Records the classes of the code blocks it gets
+ const recorder = join(mkdtempSync(join(tmpdir(), 'jsx-ast-')), 'rec.mjs');
+
+ writeFileSync(
+ recorder,
+ `export default () => tree => {
+ for (const node of tree.children) {
+ if (node.tagName === 'pre') {
+ globalThis.recorded.push(node.children[0].properties.className);
+ }
+ }
+ };`
+ );
+
+ globalThis.recorded = [];
+
+ await loadMarkdownPlugins(jsxAst, {
+ rehypePlugins: [pathToFileURL(recorder).href],
+ });
+
+ t.after(() => loadMarkdownPlugins(jsxAst));
+
+ const markdown = '```js\nconst a = 1;\n```\n';
+
+ const processor = getProcessor('jsx-ast');
+
+ await processor.run(processor.parse(markdown));
+
+ assert.deepStrictEqual(globalThis.recorded, [['language-js']]);
+
+ // The fragments doc-kit renders itself run without them
+ const own = getProcessor('jsx-ast', { configured: false });
+
+ own.runSync(own.parse(markdown));
+
+ assert.deepStrictEqual(globalThis.recorded, [['language-js']]);
+ });
+});
diff --git a/packages/react/src/jsx-ast/index.mjs b/packages/react/src/jsx-ast/index.mjs
index 191e1934f..b1dd1aeb5 100644
--- a/packages/react/src/jsx-ast/index.mjs
+++ b/packages/react/src/jsx-ast/index.mjs
@@ -1,5 +1,9 @@
'use strict';
+import { getHighlighter } from '@doc-kit/core/plugins/shiki/highlighter.mjs';
+import { createTypeAnnotationHandler } from '@doc-kit/core/plugins/type-annotations/highlighter.mjs';
+
+import { AST_NODE_TYPES } from './constants.mjs';
import { generate, processChunk } from './generate.mjs';
/**
@@ -22,6 +26,37 @@ export default {
hasParallelProcessor: true,
+ markdown: {
+ // The configured remark plugins run before the alerts, so they can make
+ // alerts too
+ remarkPlugins: ['remark-parse', '...', './plugins/alerts.mjs'],
+ rehypePlugins: [
+ [
+ 'remark-rehype',
+ {
+ // We make Rehype ignore existing HTML nodes, and JSX nodes as these
+ // are nodes we manually created during the generation process. We
+ // also allow dangerous HTML to be passed through, since we have HTML
+ // within our Markdown and we trust the sources of the Markdown files
+ allowDangerousHtml: true,
+ passThrough: ['element', ...Object.values(AST_NODE_TYPES.MDX)],
+ // Types are highlighted, with their links embedded
+ handlers: {
+ typeAnnotation: createTypeAnnotationHandler(() =>
+ getHighlighter('jsx-ast')
+ ),
+ },
+ },
+ ],
+ './plugins/raw.mjs',
+ // The configured rehype plugins run before code blocks are highlighted
+ '...',
+ '@doc-kit/core/plugins/shiki/rehype.mjs',
+ './plugins/transformer.mjs',
+ ],
+ recmaPlugins: ['rehype-recma', 'recma-jsx', '...', 'recma-stringify'],
+ },
+
generate,
processChunk,
};
diff --git a/packages/react/src/jsx-ast/utils/plugins/__tests__/alerts.test.mjs b/packages/react/src/jsx-ast/plugins/__tests__/alerts.test.mjs
similarity index 100%
rename from packages/react/src/jsx-ast/utils/plugins/__tests__/alerts.test.mjs
rename to packages/react/src/jsx-ast/plugins/__tests__/alerts.test.mjs
diff --git a/packages/react/src/jsx-ast/utils/plugins/__tests__/transformer.test.mjs b/packages/react/src/jsx-ast/plugins/__tests__/transformer.test.mjs
similarity index 100%
rename from packages/react/src/jsx-ast/utils/plugins/__tests__/transformer.test.mjs
rename to packages/react/src/jsx-ast/plugins/__tests__/transformer.test.mjs
diff --git a/packages/react/src/jsx-ast/utils/plugins/alerts.mjs b/packages/react/src/jsx-ast/plugins/alerts.mjs
similarity index 91%
rename from packages/react/src/jsx-ast/utils/plugins/alerts.mjs
rename to packages/react/src/jsx-ast/plugins/alerts.mjs
index 4ede03e56..3823ad46c 100644
--- a/packages/react/src/jsx-ast/utils/plugins/alerts.mjs
+++ b/packages/react/src/jsx-ast/plugins/alerts.mjs
@@ -2,9 +2,9 @@
import { SKIP, visit } from 'unist-util-visit';
-import { JSX_IMPORTS } from '../../../html/constants.mjs';
-import { ALERT_MARKER, GITHUB_ALERT_TYPES } from '../../constants.mjs';
-import { createJSXElement } from '../ast.mjs';
+import { JSX_IMPORTS } from '../../html/constants.mjs';
+import { ALERT_MARKER, GITHUB_ALERT_TYPES } from '../constants.mjs';
+import { createJSXElement } from '../utils/ast.mjs';
/**
* Converts a marker keyword into a human-readable title (e.g. `NOTE` -> `Note`).
diff --git a/packages/react/src/jsx-ast/plugins/raw.mjs b/packages/react/src/jsx-ast/plugins/raw.mjs
new file mode 100644
index 000000000..88fb52c56
--- /dev/null
+++ b/packages/react/src/jsx-ast/plugins/raw.mjs
@@ -0,0 +1,52 @@
+'use strict';
+
+import rehypeRaw from 'rehype-raw';
+import { visit } from 'unist-util-visit';
+
+import { AST_NODE_TYPES } from '../constants.mjs';
+
+const codeMetaProperty = 'codeMeta';
+
+/**
+ * Stores fenced code metadata on properties before rehypeRaw reparses the tree.
+ */
+const preserveCodeMeta = () => tree => {
+ visit(tree, 'element', node => {
+ const meta = node.data?.meta;
+
+ if (node.tagName === 'code' && typeof meta === 'string') {
+ node.properties ||= {};
+ node.properties[codeMetaProperty] = meta;
+ }
+ });
+};
+
+/**
+ * Restores fenced code metadata so the Shiki plugin can read displayName.
+ */
+const restoreCodeMeta = () => tree => {
+ visit(tree, 'element', node => {
+ const meta = node.properties?.[codeMetaProperty];
+
+ if (node.tagName === 'code' && typeof meta === 'string') {
+ node.data = { ...node.data, meta };
+ delete node.properties[codeMetaProperty];
+ }
+ });
+};
+
+/**
+ * Converts any `raw` HTML in the Markdown to AST, in order for Recma to
+ * understand it, keeping the metadata of fenced code.
+ *
+ * @type {import('unified').PluggableList}
+ */
+export default [
+ preserveCodeMeta,
+ // HTML and JSX nodes pass through, as we created them during the generation
+ [
+ rehypeRaw,
+ { passThrough: ['element', ...Object.values(AST_NODE_TYPES.MDX)] },
+ ],
+ restoreCodeMeta,
+];
diff --git a/packages/react/src/jsx-ast/utils/plugins/transformer.mjs b/packages/react/src/jsx-ast/plugins/transformer.mjs
similarity index 97%
rename from packages/react/src/jsx-ast/utils/plugins/transformer.mjs
rename to packages/react/src/jsx-ast/plugins/transformer.mjs
index 819b16307..cbb768357 100644
--- a/packages/react/src/jsx-ast/utils/plugins/transformer.mjs
+++ b/packages/react/src/jsx-ast/plugins/transformer.mjs
@@ -1,7 +1,7 @@
import { toString } from 'hast-util-to-string';
import { visit } from 'unist-util-visit';
-import { TAG_TRANSFORMS } from '../../constants.mjs';
+import { TAG_TRANSFORMS } from '../constants.mjs';
/**
* Checks whether a HAST node is the generated GFM footnotes section.
diff --git a/packages/react/src/jsx-ast/utils/__tests__/buildContent.test.mjs b/packages/react/src/jsx-ast/utils/__tests__/buildContent.test.mjs
index 1be679643..2259b1918 100644
--- a/packages/react/src/jsx-ast/utils/__tests__/buildContent.test.mjs
+++ b/packages/react/src/jsx-ast/utils/__tests__/buildContent.test.mjs
@@ -1,10 +1,17 @@
import assert from 'node:assert/strict';
import { describe, it } from 'node:test';
+import { loadGenerator } from '@doc-kit/core/generators/loader.mjs';
import { setConfig } from '@doc-kit/core/utils/configuration/index.mjs';
+import { loadMarkdownPlugins } from '@doc-kit/core/utils/markdown/plugins.mjs';
import { transformHeadingNode, gatherChangeEntries } from '../buildContent.mjs';
+// Pages are rendered with the pipeline of `jsx-ast`
+await loadMarkdownPlugins(
+ await loadGenerator(import.meta.resolve('../../index.mjs'))
+);
+
const heading = {
type: 'heading',
depth: 3,
diff --git a/packages/react/src/jsx-ast/utils/__tests__/remark.test.mjs b/packages/react/src/jsx-ast/utils/__tests__/remark.test.mjs
deleted file mode 100644
index afea8e9df..000000000
--- a/packages/react/src/jsx-ast/utils/__tests__/remark.test.mjs
+++ /dev/null
@@ -1,53 +0,0 @@
-import assert from 'node:assert/strict';
-import { describe, it } from 'node:test';
-
-import dedent from 'dedent';
-import { visit } from 'unist-util-visit';
-
-import { getRemarkRecma } from '../remark.mjs';
-
-const getCodeTabsAttributes = tree => {
- const attributes = [];
-
- visit(tree.body[0].expression, 'JSXElement', node => {
- if (node.openingElement.name?.name === 'CodeTabs') {
- attributes.push(
- Object.fromEntries(
- node.openingElement.attributes.map(attribute => [
- attribute.name.name,
- attribute.value?.value,
- ])
- )
- );
- }
- });
-
- return attributes;
-};
-
-describe('getRemarkRecma', () => {
- it('preserves code tab display names when raw HTML is enabled', async () => {
- const processor = getRemarkRecma();
- const tree = await processor.run(
- processor.parse(dedent`
- raw html
-
- \`\`\`cjs displayName="main.js"
- console.log(1);
- \`\`\`
-
- \`\`\`cjs displayName="main.test.js"
- console.log(2);
- \`\`\`
- `)
- );
-
- assert.deepEqual(getCodeTabsAttributes(tree), [
- {
- languages: 'cjs|cjs',
- displayNames: 'main.js|main.test.js',
- defaultTab: '0',
- },
- ]);
- });
-});
diff --git a/packages/react/src/jsx-ast/utils/__tests__/types.test.mjs b/packages/react/src/jsx-ast/utils/__tests__/types.test.mjs
index 1bafa0cd9..e26b79a96 100644
--- a/packages/react/src/jsx-ast/utils/__tests__/types.test.mjs
+++ b/packages/react/src/jsx-ast/utils/__tests__/types.test.mjs
@@ -1,15 +1,9 @@
import assert from 'node:assert/strict';
import { describe, it, mock } from 'node:test';
-// Mock remark
-mock.module('../remark.mjs', {
- exports: {
- getRemarkRecma: () => ({
- runSync: () => ({
- body: [{ expression: 'mock-expression' }],
- }),
- }),
- },
+// Mock the rendering of inline nodes
+mock.module('../render.mjs', {
+ exports: { renderAsJSX: () => 'mock-expression' },
});
const { parseListIntoProperties } = await import('../types.mjs');
diff --git a/packages/react/src/jsx-ast/utils/buildContent.mjs b/packages/react/src/jsx-ast/utils/buildContent.mjs
index 3a86d8aad..82688fa26 100644
--- a/packages/react/src/jsx-ast/utils/buildContent.mjs
+++ b/packages/react/src/jsx-ast/utils/buildContent.mjs
@@ -7,6 +7,7 @@ import {
populate,
} from '@doc-kit/core/utils/configuration/templates.mjs';
import { parseInline } from '@doc-kit/core/utils/inline.mjs';
+import { getProcessor } from '@doc-kit/core/utils/markdown/processor.mjs';
import { annotateOverloads } from '@doc-kit/core/utils/overloads.mjs';
import { splitTypedItems, UNIST } from '@doc-kit/core/utils/queries/index.mjs';
import { removeStabilityPrefix } from '@doc-kit/core/utils/stability.mjs';
@@ -28,7 +29,6 @@ import {
} from '../constants.mjs';
import { createJSXElement } from './ast.mjs';
import { extractHeadings, extractTextContent } from './buildBarProps.mjs';
-import { getRemarkRecma as remark } from './remark.mjs';
import { renderAsJSX } from './render.mjs';
import {
insertSignatureCodeBlock,
@@ -360,7 +360,7 @@ const buildContent = async (metadataEntries, head) => {
await createDocumentContent(metadataEntries);
// Run remark processor to transform AST (parse markdown, plugins, etc.)
- const ast = await remark().run(root);
+ const ast = await getProcessor('jsx-ast').run(root);
// The fragment is the expression in the Program's first body node
return { data: head, headings, readingTime, content: ast.body[0].expression };
diff --git a/packages/react/src/jsx-ast/utils/remark.mjs b/packages/react/src/jsx-ast/utils/remark.mjs
deleted file mode 100644
index 59bafd01c..000000000
--- a/packages/react/src/jsx-ast/utils/remark.mjs
+++ /dev/null
@@ -1,80 +0,0 @@
-'use strict';
-
-import { highlighter } from '@doc-kit/core/utils/highlighter.mjs';
-import { lazy } from '@doc-kit/core/utils/misc.mjs';
-import { typeAnnotationToHighlightedHast } from '@doc-kit/core/utils/type-annotations/highlighted.mjs';
-import rehypeShikiji from '@node-core/rehype-shiki/plugin';
-import recmaJsx from 'recma-jsx';
-import recmaStringify from 'recma-stringify';
-import rehypeRaw from 'rehype-raw';
-import rehypeRecma from 'rehype-recma';
-import remarkParse from 'remark-parse';
-import remarkRehype from 'remark-rehype';
-import { unified } from 'unified';
-import { visit } from 'unist-util-visit';
-
-import { AST_NODE_TYPES } from '../constants.mjs';
-import transformAlerts from './plugins/alerts.mjs';
-import transformElements from './plugins/transformer.mjs';
-
-const passThrough = ['element', ...Object.values(AST_NODE_TYPES.MDX)];
-const codeMetaProperty = 'codeMeta';
-
-/**
- * Stores fenced code metadata on properties before rehypeRaw reparses the tree.
- */
-const preserveCodeMeta = () => tree => {
- visit(tree, 'element', node => {
- const meta = node.data?.meta;
-
- if (node.tagName === 'code' && typeof meta === 'string') {
- node.properties ||= {};
- node.properties[codeMetaProperty] = meta;
- }
- });
-};
-
-/**
- * Restores fenced code metadata so the Shiki plugin can read displayName.
- */
-const restoreCodeMeta = () => tree => {
- visit(tree, 'element', node => {
- const meta = node.properties?.[codeMetaProperty];
-
- if (node.tagName === 'code' && typeof meta === 'string') {
- node.data = { ...node.data, meta };
- delete node.properties[codeMetaProperty];
- }
- });
-};
-
-const singletonShiki = await rehypeShikiji({ highlighter });
-
-/**
- * Retrieves an instance of Remark configured to output JSX code.
- * including parsing Code Boxes with syntax highlighting
- */
-export const getRemarkRecma = lazy(() =>
- unified()
- .use(remarkParse)
- .use(transformAlerts)
- // We make Rehype ignore existing HTML nodes, and JSX nodes
- // as these are nodes we manually created during the generation process
- // We also allow dangerous HTML to be passed through, since we have HTML within our Markdown
- // and we trust the sources of the Markdown files
- .use(remarkRehype, {
- allowDangerousHtml: true,
- passThrough,
- // The web pipeline gets Shiki-highlighted types with embedded links
- handlers: { typeAnnotation: typeAnnotationToHighlightedHast },
- })
- .use(preserveCodeMeta)
- // Any `raw` HTML in the markdown must be converted to AST in order for Recma to understand it
- .use(rehypeRaw, { passThrough })
- .use(restoreCodeMeta)
- .use(() => singletonShiki)
- .use(transformElements)
- .use(rehypeRecma)
- .use(recmaJsx)
- .use(recmaStringify)
-);
diff --git a/packages/react/src/jsx-ast/utils/render.mjs b/packages/react/src/jsx-ast/utils/render.mjs
index bab668ea6..883f8e6e4 100644
--- a/packages/react/src/jsx-ast/utils/render.mjs
+++ b/packages/react/src/jsx-ast/utils/render.mjs
@@ -1,17 +1,18 @@
'use strict';
+import { getProcessor } from '@doc-kit/core/utils/markdown/processor.mjs';
import { u as createTree } from 'unist-builder';
import { createJSXElement } from './ast.mjs';
-import { getRemarkRecma as remark } from './remark.mjs';
/**
- * Renders inline nodes as a JSX fragment
+ * Renders inline nodes as a JSX fragment. It runs synchronously, so without
+ * the configured plugins, which may be asynchronous.
*
* @param {Array} nodes - The nodes to render.
* @returns {import('estree-jsx').JSXFragment} The rendered nodes.
*/
export const renderAsJSX = nodes =>
- remark().runSync(
+ getProcessor('jsx-ast', { configured: false }).runSync(
createTree('root', [createJSXElement(null, { children: nodes })])
).body[0].expression;
diff --git a/packages/react/src/jsx-ast/utils/signature.mjs b/packages/react/src/jsx-ast/utils/signature.mjs
index d4efdee4a..3edf90156 100644
--- a/packages/react/src/jsx-ast/utils/signature.mjs
+++ b/packages/react/src/jsx-ast/utils/signature.mjs
@@ -1,4 +1,4 @@
-import { highlighter } from '@doc-kit/core/utils/highlighter.mjs';
+import { getHighlighter } from '@doc-kit/core/plugins/shiki/highlighter.mjs';
import { UNIST } from '@doc-kit/core/utils/queries/index.mjs';
import { parseListItem } from '@doc-kit/core/utils/signature/parseList.mjs';
import parseSignature from '@doc-kit/core/utils/signature/parseSignature.mjs';
@@ -65,6 +65,7 @@ export const generateSignature = (
*/
export const createSignatureCodeBlock = (functionName, signature, heading) => {
const sig = generateSignature(functionName, signature, heading);
+ const highlighter = getHighlighter('jsx-ast');
const highlighted = highlighter.highlightToHast(sig, 'typescript');
return createElement('div', { class: 'signature' }, [highlighted]);
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index d2d1ec1f4..af67ac0e6 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -104,9 +104,6 @@ importers:
glob-parent:
specifier: ^6.0.2
version: 6.0.2
- hastscript:
- specifier: ^9.0.1
- version: 9.0.1
mdast-util-slice-markdown:
specifier: ^2.0.1
version: 2.0.1
@@ -208,6 +205,15 @@ importers:
hastscript:
specifier: ^9.0.1
version: 9.0.1
+ rehype-stringify:
+ specifier: ^10.0.1
+ version: 10.0.1
+ remark-parse:
+ specifier: ^11.0.0
+ version: 11.0.0(supports-color@7.2.0)
+ remark-rehype:
+ specifier: ^11.1.2
+ version: 11.1.2
unist-builder:
specifier: ^4.0.0
version: 4.0.0