Gitlink
/
gitlink_help_center
0
0
Fork 56
Code Issues Pull Requests 32 Projects Releases Wiki Activity

fix: fix Chinese token filter regex for BM25 search 

\W in regex matches Chinese characters, causing all Chinese tokens
to be filtered out. Replaced with \p{P} (Unicode punctuation) to
correctly preserve Chinese tokens while filtering punctuation.
This commit is contained in:
z2_cc 2026-05-13 11:51:03 +08:00
parent 423e40b3dc
commit 391a83a11d
2 changed files with 3 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
api/search/indexer.js
View File

@ -7,7 +7,7 @@ let jieba;
try {
jieba = require('@node-rs/jieba');
} catch {
jieba = { cut: (text) => [...new Set(text.replace(/[^一-龥a-zA-Z0-9]/g, ' ').split(/\s+/).filter(t => t.length > 1))] };
jieba = { cut: (text) => [...new Set(text.replace(/[^一-鿿a-zA-Z0-9]/g, ' ').split(/\s+/).filter(t => t.length > 1))] };
}
let index = null;
@ -24,7 +24,7 @@ function loadIndex() {
}
function tokenizeQuery(query) {
return jieba.cut(query).filter(t => t.length > 1 && !/^[\s\d\W]+$/.test(t));
return jieba.cut(query).filter(t => t.length > 1 && !/^[\s\d\p{P}]+$/u.test(t));
}
/**

2
scripts/generate-doc-index.js
View File

@ -59,7 +59,7 @@ function getTitle(docPath, content) {
function tokenize(text) {
const tokens = jieba.cut(text);
return [...new Set(tokens.filter(t => t.length > 1 && !/^[\s\d\W]+$/.test(t)))];
return [...new Set(tokens.filter(t => t.length > 1 && !/^[\s\d\p{P}]+$/u.test(t)))];
}
function processDoc(filePath) {