feat: add build-time doc index generator

Created scripts/generate-doc-index.js that:
- Scans all .md files under docs/
- Extracts titles, breadcrumbs, and URLs
- Strips Markdown syntax and tokenizes content with jieba
- Outputs structured index to static/doc-index.json

Also added static/doc-index.json to .gitignore
This commit is contained in:
z2_cc 2026-05-13 11:38:22 +08:00
parent bec48dca72
commit d28b46c86f
2 changed files with 111 additions and 0 deletions

4
.gitignore vendored
View File

@ -27,3 +27,7 @@ yarn-error.log*
# Office temp files
.~*
.superpowers/
# Generated search index
static/doc-index.json

View File

@ -0,0 +1,107 @@
const fs = require('fs');
const path = require('path');
// @node-rs/jieba is already a project dependency
let jieba;
try {
jieba = require('@node-rs/jieba');
} catch {
console.warn('Warning: @node-rs/jieba not available, using simple character split');
jieba = { cut: (text) => [...new Set(text.replace(/[^一-龥a-zA-Z0-9]/g, ' ').split(/\s+/).filter(t => t.length > 1))] };
}
const DOCS_DIR = path.join(__dirname, '..', 'docs');
const OUTPUT_PATH = path.join(__dirname, '..', 'static', 'doc-index.json');
function stripMarkdown(md) {
return md
.replace(/^---[\s\S]*?---/m, '') // frontmatter
.replace(/!\[.*?\]\(.*?\)/g, '') // images
.replace(/\[([^\]]*)\]\(.*?\)/g, '$1') // links → text
.replace(/#{1,6}\s/g, '') // headings
.replace(/(\*{1,3}|_{1,3})(.*?)\1/g, '$2') // bold/italic
.replace(/`{1,3}[^`]*`{1,3}/g, '') // inline code
.replace(/<[^>]+>/g, '') // HTML tags
.replace(/\n{2,}/g, '\n') // extra newlines
.trim();
}
function getBreadcrumb(docDir) {
const parts = [];
let current = docDir;
while (current !== DOCS_DIR) {
const categoryFile = path.join(current, '_category_.json');
if (fs.existsSync(categoryFile)) {
const cat = JSON.parse(fs.readFileSync(categoryFile, 'utf-8'));
parts.unshift(cat.label);
}
current = path.dirname(current);
}
return parts.join(' > ');
}
function getUrl(docPath) {
const relPath = path.relative(DOCS_DIR, docPath);
const withoutExt = relPath.replace(/\.md$/, '');
if (withoutExt === 'intro') return '/';
return '/' + withoutExt;
}
function getTitle(docPath, content) {
const fmMatch = content.match(/sidebar_label:\s*['"]?(.+?)['"]?\s*$/m);
if (fmMatch) return fmMatch[1].trim();
const headingMatch = content.match(/^#{1,6}\s+(.+)$/m);
if (headingMatch) return headingMatch[1].trim();
return path.basename(docPath, '.md');
}
function tokenize(text) {
const tokens = jieba.cut(text);
return [...new Set(tokens.filter(t => t.length > 1 && !/^[\s\d\W]+$/.test(t)))];
}
function processDoc(filePath) {
const content = fs.readFileSync(filePath, 'utf-8');
const text = stripMarkdown(content);
const title = getTitle(filePath, content);
const breadcrumb = getBreadcrumb(path.dirname(filePath));
const url = getUrl(filePath);
const tokens = tokenize(text);
const fullBreadcrumb = breadcrumb ? `${breadcrumb} > ${title}` : title;
return { title, text, tokens, breadcrumb: fullBreadcrumb, url };
}
function walkDocs(dir) {
const entries = fs.readdirSync(dir, { withFileTypes: true });
const mdFiles = [];
for (const entry of entries) {
const fullPath = path.join(dir, entry.name);
if (entry.name === 'superpowers') continue;
if (entry.isDirectory()) {
mdFiles.push(...walkDocs(fullPath));
} else if (entry.name.endsWith('.md')) {
mdFiles.push(fullPath);
}
}
return mdFiles;
}
function main() {
console.log('Generating document index...');
const mdFiles = walkDocs(DOCS_DIR);
console.log(`Found ${mdFiles.length} markdown files`);
const index = mdFiles.map(processDoc);
const totalTokens = index.reduce((sum, doc) => sum + doc.tokens.length, 0);
console.log(`Index: ${index.length} docs, ${totalTokens} total unique tokens`);
fs.writeFileSync(OUTPUT_PATH, JSON.stringify(index, null, 2), 'utf-8');
console.log(`Written to ${OUTPUT_PATH}`);
}
main();