// MDX → plain markdown, for the /llms.txt endpoints. // // The docs are MDX: prose is already markdown, but the parts that carry the most // value for a reader (the actual Tabler markup) sit inside slots and // component props. Stripping components wholesale — the usual llms.txt recipe — // would delete exactly that. So the components that hold content are unwrapped // into fenced code blocks instead, and only the decorative ones are dropped. // A component whose content is data rather than markup (, ) // renders its own markdown in a comment, see shared/lib/markdown-source.ts. // // Prose comes from the MDX source, but example markup comes from the *rendered* // page (see renderedPage below): a third of the examples are written with // docs components — , , — and the source form of those // is of no use to a reader who wants the html. import type { CollectionEntry } from 'astro:content' import { render } from 'astro:content' import { loadRenderers } from 'astro:container' import { experimental_AstroContainer as AstroContainer } from 'astro/container' import { getContainerRenderer } from '@astrojs/mdx/container-renderer' import { beautifyHtml, extractMarkedSnippet } from '@shared/lib/code-example' import { extractMarkdownSources } from '@shared/lib/markdown-source' import { site } from '@shared/lib/site' import { callouts } from '@components/callouts/index.ts' import docs from '@data/docs.json' import packageManagers from '@data/package-managers.json' import payments from '@data/payments.json' import { cdnCssTag, cdnJsTag, cdnPackageSnippet, cdnPluginSnippet, cdnThemeJsTag } from './cdn-snippets.ts' // Lazy raw imports, same as CodeDocs.astro — node:fs paths break once this is // bundled into dist/.prerender. const scssSources = import.meta.glob('../../core/scss/**/*.scss', { query: '?raw', import: 'default' }) const jsSources = import.meta.glob('../../core/js/**/*.{js,ts}', { query: '?raw', import: 'default' }) const fence = (code: string, lang = 'html') => `\`\`\`${lang}\n${code.trim()}\n\`\`\`` /** * `` blocks are read as source text, never evaluated, so the few interpolations * the docs use inside them are resolved by hand. Anything not listed here is left as written. */ function resolveCodeTokens(snippet: string): string { const tokens: Record string> = { '${cdnCssTag()}': cdnCssTag, '${cdnJsTag()}': cdnJsTag, '${cdnThemeJsTag()}': cdnThemeJsTag, } return Object.entries(tokens) .reduce((text, [token, resolve]) => text.replaceAll(token, resolve()), snippet) .replace(/\$\{site\.(\w+)\}/g, (match, key: string) => siteValue(key) ?? match) } /** A string or number field of `site`, as the MDX pages read it; undefined for anything else. */ function siteValue(key: string): string | undefined { const value = (site as Record)[key] return typeof value === 'string' || typeof value === 'number' ? String(value) : undefined } /** * JSX expressions the pages use in prose — `{ site.iconsCount }`, * `{ payments.length }` — and `${site.…}` inside template literals. MDX * evaluates them in the browser; the markdown mirror has to do it here. */ function resolveExpressions(text: string): string { return resolveCodeTokens(text) .replace(/\{\s*site\.(\w+)\s*\}/g, (match, key: string) => siteValue(key) ?? match) .replace(/\{\s*payments\.length\s*\}/g, String(payments.length)) } /** Strip the common leading indentation from a block and trim blank edges. */ const dedent = (text: string) => { const lines = text.replace(/^\n+|\s+$/g, '').split('\n') const indents = lines.filter((line) => line.trim()).map((line) => (line.match(/^[ \t]*/)?.[0] ?? '').length) const min = indents.length ? Math.min(...indents) : 0 return lines.map((line) => line.slice(min)).join('\n') } /** Replace every match of `pattern`, awaiting an async replacer for each one. */ async function replaceAsync(input: string, pattern: RegExp, replacer: (match: RegExpExecArray) => Promise) { const matches = [...input.matchAll(pattern)] if (!matches.length) return input const replacements = await Promise.all(matches.map((match) => replacer(match))) let result = '' let last = 0 matches.forEach((match, index) => { result += input.slice(last, match.index) + replacements[index] last = (match.index ?? 0) + match[0].length }) return result + input.slice(last) } const attr = (tag: string, name: string) => tag.match(new RegExp(`${name}=["']([^"']*)["']`))?.[1] ?? null /** `` → the marked snippet from the real source file. */ async function resolveCodeDocs(tag: string): Promise { const file = attr(tag, 'file') const name = attr(tag, 'name') if (!file || !name) return '' const extension = file.split('.').pop() const kind = extension === 'scss' ? { sources: scssSources, marker: 'scss-docs', lang: 'scss' } : extension === 'ts' || extension === 'js' ? { sources: jsSources, marker: 'js-docs', lang: extension } : null if (!kind) return '' const load = kind.sources[`../../${file}`] if (!load) return '' let source = await load() if (kind.lang === 'scss') source = source.replaceAll(' !default', '') const snippet = extractMarkedSnippet(source, kind.marker, name) return snippet ? `${fence(snippet, kind.lang)}\n\n_Source: \`${file}\`_` : '' } // Placeholder sentinel for code that must survive the component stripping below. // A private-use code point cannot appear in the docs source, so a placeholder can // never collide with prose. const SENTINEL = String.fromCharCode(0xe000) const placeholder = (index: number) => `${SENTINEL}${index}${SENTINEL}` const placeholderPattern = new RegExp(`${SENTINEL}(\\d+)${SENTINEL}`, 'g') /** * Swap fenced blocks and inline code for placeholders, so the component stripping * never reaches inside them. Without this, an `` shown as example markup * and an `import App from './App.vue'` line in a framework guide would both be * deleted as if they were real MDX. */ function protectCode(text: string, store: string[]) { return text.replace(/```[\s\S]*?```|`[^`\n]*`/g, (match) => { store.push(match) return placeholder(store.length - 1) }) } /** Placeholders can nest (a rewritten block may contain protected code), so repeat until stable. */ function restoreCode(text: string, store: string[]) { let result = text while (result.includes(SENTINEL)) { const next = result.replace(placeholderPattern, (_match, index: string) => store[Number(index)] ?? '') // a sentinel that resolves to nothing would otherwise loop forever if (next === result) break result = next } return result } // Marker Example.astro puts in front of every example it renders. const EXAMPLE_MARKER = '' const decodeEntities = (value: string) => value .replace(/"/g, '"') .replace(/'/g, "'") .replace(/</g, '<') .replace(/>/g, '>') .replace(/&/g, '&') // One container for the whole build: creating it loads the MDX renderer. let container: Promise | undefined const getContainer = () => (container ??= loadRenderers([getContainerRenderer()]).then((renderers) => AstroContainer.create({ renderers }))) type RenderedPage = { /** final markup of every , in document order; null when the example renders nothing */ examples: (string | null)[] /** markdown shipped by components via , per component name */ sources: Map } const EMPTY_PAGE: RenderedPage = { examples: [], sources: new Map() } /** * What the rendered page carries for the markdown mirror: the final markup of * every — the same html the page's copy button hands out, so * components are already expanded — and the markdown sources of data * components. Both lists are empty when the page cannot be rendered. */ async function renderedPage(entry: CollectionEntry<'docs'>): Promise { try { const { Content } = await render(entry) // Same components the page route passes, or every page using a callout throws. // The local tells this render is for the mirror. const html = await (await getContainer()).renderToString(Content, { props: { components: callouts }, locals: { markdownMirror: true } }) const examples = html .split(EXAMPLE_MARKER) .slice(1) .map((block) => { // the copy button, or the wrapper attribute when the example hides its code panel const markup = block.match(/data-bs-text="([^"]*)"/) ?? block.match(/data-example-markup="([^"]*)"/) return markup ? beautifyHtml(decodeEntities(markup[1]!)) : null }) return { examples, sources: extractMarkdownSources(html) } } catch (error) { console.warn(`[llms] Rendering ${entry.id} failed, examples fall back to their MDX source:`, error) return EMPTY_PAGE } } /** * Turn one page's MDX body into plain markdown. `examples` and `sources` come * from renderedPage(): the examples replace the source of each slot, * the sources replace the tags of the components that shipped them. Either is * ignored unless it lines up one-to-one with the tags in the source. */ export async function mdxToMarkdown(body: string, examples: (string | null)[] = [], sources: Map = new Map()): Promise { // `` first: its template literal contains backticks, which // would otherwise be mistaken for markdown code spans by protectCode() below. let text = body.replace(/]*?code=\{`([\s\S]*?)`\}[\s\S]*?\/>/g, (match, snippet: string) => { const lang = attr(match, 'lang') ?? 'html' return `\n${fence(resolveCodeTokens(snippet), lang)}\n` }) const code: string[] = [] // code the page already had, before anything is rewritten text = protectCode(text, code) // top-level imports are wiring, not content text = text.replace(/^import\s+.+?from\s+['"][^'"]+['"];?[ \t]*$/gm, '') // slots hold the markup the page is actually documenting: either // wrapped in the tag, or passed as `html={\`…\`}` on a self-closing one (that // template literal may contain `>`, so it is matched before the attribute scan) const examplePattern = /]*?html=\{`([\s\S]*?)`\})?[^>]*?(?:\/>|>([\s\S]*?)<\/Example>)/g const useRendered = examples.length === (text.match(examplePattern)?.length ?? 0) let exampleIndex = 0 text = text.replace(examplePattern, (_match, html: string | undefined, inner: string | undefined) => { const snippet = (useRendered ? examples[exampleIndex++] : null) ?? dedent(html ?? inner ?? '') return snippet ? `\n${fence(snippet)}\n` : '' }) text = await replaceAsync(text, /]*\/>/g, (match) => resolveCodeDocs(match[0])) // install instructions rendered as tabs in the browser text = text.replace(/]*\/>/g, (match: string) => { const name = attr(match, 'name') if (!name) return '' const commands = packageManagers.map((manager) => `${manager.command} ${manager.install} ${name}`) return `\n${fence(commands.join('\n'), 'shell')}\n` }) text = text.replace(/]*\/>/g, () => `\n${fence(cdnPackageSnippet())}\n`) text = text.replace(/]*\/>/g, (match: string) => { const plugins = [...match.matchAll(/'([^']+)'/g)].map((plugin) => plugin[1]).filter((plugin): plugin is string => Boolean(plugin)) if (!plugins.length) return '' return `\n${fence(cdnPluginSnippet(plugins))}\n` }) // data components that shipped their own markdown (, , …) for (const [name, markdowns] of sources) { const tagPattern = new RegExp(`<${name}\\b[^>]*/>`, 'g') if ((text.match(tagPattern)?.length ?? 0) !== markdowns.length) continue let sourceIndex = 0 text = text.replace(tagPattern, () => `\n${markdowns[sourceIndex++]}\n`) } // the blocks just produced must be protected too, for the same reason text = protectCode(text, code) // decorative self-closing components (icons, illustrations, charts, …) text = text.replace(/<[A-Z][A-Za-z]*\b[^>]*\/>/g, '') // anything else that wraps content: drop the tags, keep what is inside text = text.replace(/<\/?[A-Z][A-Za-z]*\b[^>]*>/g, '') // MDX comments and leftover JSX expressions text = text.replace(/\{\/\*[\s\S]*?\*\/\}/g, '') return resolveExpressions(restoreCode(text, code)) .replace(/\n{3,}/g, '\n\n') .trim() } async function buildPageMarkdown(entry: CollectionEntry<'docs'>, url: string): Promise { const { title, summary, description } = entry.data const header = [`# ${title}`, '', `> ${summary}`, '', description, '', `Source: ${url}`, '', '---', ''].join('\n') const { examples, sources } = await renderedPage(entry) return `${header}\n${await mdxToMarkdown(entry.body ?? '', examples, sources)}\n` } // [...slug].md.ts and llms-full.txt.ts render the same pages in one build. const pageCache = new Map>() /** A single docs page as a standalone markdown document. */ export function pageMarkdown(entry: CollectionEntry<'docs'>, url: string): Promise { const key = `${entry.id}\n${url}` let cached = pageCache.get(key) if (!cached) { cached = buildPageMarkdown(entry, url) pageCache.set(key, cached) } return cached } type MenuNode = { url?: string; children?: MenuNode[] } /** Docs urls in sidebar (docs.json) order — the shared reading order of llms.txt and llms-full.txt. */ export function menuOrderedUrls(): string[] { const normalize = (url: string) => { const parts = url.split('/').filter(Boolean) return parts.length ? `/${parts.join('/')}` : '/' } const walk = (nodes: MenuNode[]): string[] => nodes.flatMap((node) => [...(node.url ? [normalize(node.url)] : []), ...walk(node.children ?? [])]) return [...new Set(walk(docs.menu))] } /** Absolute in production, root-relative in dev — same rule as sitemap.xml.ts. */ export const baseUrl = () => (process.env.NODE_ENV === 'development' ? '' : site.docsUrl)