From 391a83a11d6deb2304bac25f52156259f468ef76 Mon Sep 17 00:00:00 2001
From: z2_cc <170238968@qq.com>
Date: Wed, 13 May 2026 11:51:03 +0800
Subject: [PATCH] fix: fix Chinese token filter regex for BM25 search

\W in regex matches Chinese characters, causing all Chinese tokens
to be filtered out. Replaced with \p{P} (Unicode punctuation) to
correctly preserve Chinese tokens while filtering punctuation.
---
 api/search/indexer.js         | 4 ++--
 scripts/generate-doc-index.js | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/api/search/indexer.js b/api/search/indexer.js
index b2a8453..70b7b13 100644
--- a/api/search/indexer.js
+++ b/api/search/indexer.js
@@ -7,7 +7,7 @@ let jieba;
 try {
   jieba = require('@node-rs/jieba');
 } catch {
-  jieba = { cut: (text) => [...new Set(text.replace(/[^一-龥a-zA-Z0-9]/g, ' ').split(/\s+/).filter(t => t.length > 1))] };
+  jieba = { cut: (text) => [...new Set(text.replace(/[^一-鿿a-zA-Z0-9]/g, ' ').split(/\s+/).filter(t => t.length > 1))] };
 }
 
 let index = null;
@@ -24,7 +24,7 @@ function loadIndex() {
 }
 
 function tokenizeQuery(query) {
-  return jieba.cut(query).filter(t => t.length > 1 && !/^[\s\d\W]+$/.test(t));
+  return jieba.cut(query).filter(t => t.length > 1 && !/^[\s\d\p{P}]+$/u.test(t));
 }
 
 /**
diff --git a/scripts/generate-doc-index.js b/scripts/generate-doc-index.js
index 063ea93..02da529 100644
--- a/scripts/generate-doc-index.js
+++ b/scripts/generate-doc-index.js
@@ -59,7 +59,7 @@ function getTitle(docPath, content) {
 
 function tokenize(text) {
   const tokens = jieba.cut(text);
-  return [...new Set(tokens.filter(t => t.length > 1 && !/^[\s\d\W]+$/.test(t)))];
+  return [...new Set(tokens.filter(t => t.length > 1 && !/^[\s\d\p{P}]+$/u.test(t)))];
 }
 
 function processDoc(filePath) {