refactor: 简化检索策略，对所有问题在所有数据库中进行向量化检索，构造一般化提示词
This commit is contained in:
linlin
parent
7005ff2c66
commit
f8a514b30d
|
|
@ -263,38 +263,17 @@ class RAGEngine:
|
|||
Response text chunks
|
||||
"""
|
||||
try:
|
||||
# 1. 使用小型模型分析问题，返回置信度和可能的filter
|
||||
logger.info("使用小型模型分析问题")
|
||||
analysis_result = await self.is_code_related(query, history)
|
||||
confidence = analysis_result['confidence']
|
||||
filters = analysis_result['filters']
|
||||
logger.info(f"分析结果: confidence={confidence}, filters={filters}")
|
||||
# 1. 不区分是否代码相关问题，直接在所有collection中检索
|
||||
logger.info("在所有collection中进行向量检索")
|
||||
|
||||
# 2. 根据置信度选择collection和检索策略
|
||||
threshold_low = settings.CODE_RELATED_THRESHOLD_LOW
|
||||
threshold_high = settings.CODE_RELATED_THRESHOLD_HIGH
|
||||
|
||||
if confidence < threshold_low:
|
||||
collection_key = 'non_code'
|
||||
use_advanced_prompt = False
|
||||
logger.info(f"置信度 {confidence} < {threshold_low}，判定为非代码问题，使用non_code collection")
|
||||
elif confidence > threshold_high:
|
||||
collection_key = 'code'
|
||||
use_advanced_prompt = True
|
||||
logger.info(f"置信度 {confidence} > {threshold_high}，判定为代码相关问题，使用code collection")
|
||||
else:
|
||||
collection_key = None
|
||||
use_advanced_prompt = False
|
||||
logger.info(f"置信度 {threshold_low} <= {confidence} <= {threshold_high}，通用问题，搜索所有collection")
|
||||
|
||||
# 3. 使用纯向量检索（替代融合检索）
|
||||
# 2. 使用纯向量检索
|
||||
logger.info("使用纯向量检索")
|
||||
k = top_k or settings.TOP_K
|
||||
|
||||
retriever = self.vector_store_manager.get_retriever(
|
||||
top_k=k * 2,
|
||||
filters=filters if filters else None,
|
||||
collection_key=collection_key
|
||||
filters=None,
|
||||
collection_key=None
|
||||
)
|
||||
|
||||
if isinstance(retriever, list):
|
||||
|
|
@ -357,26 +336,16 @@ class RAGEngine:
|
|||
|
||||
context_str = "\n\n".join(context_parts) if context_parts else "未找到相关参考信息"
|
||||
|
||||
# 6. 生成Prompt
|
||||
# 6. 生成Prompt - 根据是否有历史对话选择不同模板
|
||||
logger.info("生成Prompt")
|
||||
if use_advanced_prompt and confidence > threshold_high:
|
||||
filled_prompt = f"""基于以下参考信息回答用户问题。如果参考信息不足，请基于你的知识回答。
|
||||
|
||||
参考信息：
|
||||
{context_str}
|
||||
|
||||
用户问题：{query}
|
||||
|
||||
请直接回答："""
|
||||
if history:
|
||||
qa_prompt = QA_PROMPT_HISTORY
|
||||
filled_prompt = qa_prompt.format(history=history, context_str=context_str, query_str=query)
|
||||
else:
|
||||
if history:
|
||||
qa_prompt = QA_PROMPT_HISTORY
|
||||
filled_prompt = qa_prompt.format(history=history, context_str=context_str, query_str=query)
|
||||
else:
|
||||
qa_prompt = QA_PROMPT_NO_HISTORY
|
||||
filled_prompt = qa_prompt.format(context_str=context_str, query_str=query)
|
||||
qa_prompt = QA_PROMPT_NO_HISTORY
|
||||
filled_prompt = qa_prompt.format(context_str=context_str, query_str=query)
|
||||
|
||||
logger.info(f"使用Prompt类型: {'通用' if not use_advanced_prompt else '代码自由发挥'}")
|
||||
logger.info("根据历史对话选择prompt模板")
|
||||
|
||||
# 7. 流式生成回答
|
||||
stream_response = await self.llm.astream_complete(prompt=filled_prompt)
|
||||
|
|
@ -410,42 +379,18 @@ class RAGEngine:
|
|||
Complete response string
|
||||
"""
|
||||
try:
|
||||
# 1. 使用小型模型分析问题，返回置信度和可能的filter
|
||||
logger.info("使用小型模型分析问题")
|
||||
analysis_result = await self.is_code_related(query, history)
|
||||
confidence = analysis_result['confidence']
|
||||
filters = analysis_result['filters']
|
||||
logger.info(f"分析结果: confidence={confidence}, filters={filters}")
|
||||
# 1. 不区分是否代码相关问题，直接在所有collection中检索
|
||||
logger.info("在所有collection中进行向量检索")
|
||||
|
||||
# 2. 根据置信度选择collection和检索策略
|
||||
threshold_low = settings.CODE_RELATED_THRESHOLD_LOW
|
||||
threshold_high = settings.CODE_RELATED_THRESHOLD_HIGH
|
||||
|
||||
if confidence < threshold_low:
|
||||
# 置信度低：非代码问题，使用non_code collection
|
||||
collection_key = 'non_code'
|
||||
use_advanced_prompt = False
|
||||
logger.info(f"置信度 {confidence} < {threshold_low}，判定为非代码问题，使用non_code collection")
|
||||
elif confidence > threshold_high:
|
||||
# 置信度高：代码相关问题，使用code collection
|
||||
collection_key = 'code'
|
||||
use_advanced_prompt = True
|
||||
logger.info(f"置信度 {confidence} > {threshold_high}，判定为代码相关问题，使用code collection")
|
||||
else:
|
||||
# 中间区间：不区分，使用所有collection
|
||||
collection_key = None
|
||||
use_advanced_prompt = False
|
||||
logger.info(f"置信度 {threshold_low} <= {confidence} <= {threshold_high}，通用问题，搜索所有collection")
|
||||
|
||||
# 3. 使用纯向量检索（替代融合检索）
|
||||
# 2. 使用纯向量检索
|
||||
logger.info("使用纯向量检索")
|
||||
k = top_k or settings.TOP_K
|
||||
|
||||
# 获取retriever
|
||||
retriever = self.vector_store_manager.get_retriever(
|
||||
top_k=k * 2, # 获取更多结果用于去重
|
||||
filters=filters if filters else None,
|
||||
collection_key=collection_key
|
||||
filters=None,
|
||||
collection_key=None
|
||||
)
|
||||
|
||||
# 执行检索
|
||||
|
|
@ -512,28 +457,16 @@ class RAGEngine:
|
|||
|
||||
context_str = "\n\n".join(context_parts) if context_parts else "未找到相关参考信息"
|
||||
|
||||
# 6. 生成Prompt
|
||||
# 6. 生成Prompt - 根据是否有历史对话选择不同模板
|
||||
logger.info("生成Prompt")
|
||||
if use_advanced_prompt and confidence > threshold_high:
|
||||
# 高置信度代码问题：让LLM自由发挥，不预设模板
|
||||
filled_prompt = f"""基于以下参考信息回答用户问题。如果参考信息不足，请基于你的知识回答。
|
||||
|
||||
参考信息：
|
||||
{context_str}
|
||||
|
||||
用户问题：{query}
|
||||
|
||||
请直接回答："""
|
||||
if history:
|
||||
qa_prompt = QA_PROMPT_HISTORY
|
||||
filled_prompt = qa_prompt.format(history=history, context_str=context_str, query_str=query)
|
||||
else:
|
||||
# 非代码问题或中间区间：使用通用Prompt
|
||||
if history:
|
||||
qa_prompt = QA_PROMPT_HISTORY
|
||||
filled_prompt = qa_prompt.format(history=history, context_str=context_str, query_str=query)
|
||||
else:
|
||||
qa_prompt = QA_PROMPT_NO_HISTORY
|
||||
filled_prompt = qa_prompt.format(context_str=context_str, query_str=query)
|
||||
qa_prompt = QA_PROMPT_NO_HISTORY
|
||||
filled_prompt = qa_prompt.format(context_str=context_str, query_str=query)
|
||||
|
||||
logger.info(f"使用Prompt类型: {'通用' if not use_advanced_prompt else '代码自由发挥'}")
|
||||
logger.info("根据历史对话选择prompt模板")
|
||||
|
||||
# 7. 调用LLM生成回答
|
||||
response = await self.llm.acomplete(prompt=filled_prompt)
|
||||
|
|
|
|||
|
|
@ -20,6 +20,9 @@ def check_metadata():
|
|||
|
||||
# 获取所有文档的metadata
|
||||
print("正在获取所有文档的metadata...")
|
||||
results = vector_store_manager.collections['non_code'].get(include=['metadatas', 'documents'])
|
||||
print(results)
|
||||
input()
|
||||
# print(vector_store_manager)
|
||||
results = vector_store_manager.collections['code'].get(include=['metadatas', 'documents'])
|
||||
|
||||
|
|
@ -31,6 +34,15 @@ def check_metadata():
|
|||
processed_shas[file_path] = blob_sha
|
||||
print(f"已处理文件SHA: {file_path} -> {blob_sha}")
|
||||
# input("按任意键继续...")
|
||||
collection = vector_store_manager.collections['code']
|
||||
target_source = 'git_server_172_26_120_125'
|
||||
result = collection.get(where={"repo_id": {"$contains": target_source}})
|
||||
print(f"{target_source}查询结果: {str(result)[:50]}")
|
||||
target_source = 'git_server_172_26_120_125_testrepo'
|
||||
result = collection.get(where={"repo_id": {"$contains": target_source}})
|
||||
print(f"{target_source}查询结果: {str(result)[:50]}")
|
||||
result = collection.get(where={"repo_id": target_source})
|
||||
print(f"{target_source}查询结果: {str(result)[:50]}")
|
||||
exit(0)
|
||||
|
||||
# 提取数据
|
||||
|
|
|
|||
|
|
@ -96,7 +96,7 @@ class GitTool:
|
|||
# 检查是否在Windows系统上
|
||||
import platform
|
||||
system = platform.system()
|
||||
logger.info(f"系统类型：{system}")
|
||||
logger.info(f"本地系统类型：{system}")
|
||||
|
||||
# 使用 GIT_ASKPASS 机制，这是 Git 官方推荐的密码输入方式
|
||||
# 创建 askpass 脚本
|
||||
|
|
@ -174,7 +174,7 @@ else:
|
|||
"--branch", self.branch, url, self.local_repo_path
|
||||
]
|
||||
logger.info(f"执行Git克隆命令: {' '.join(cmd)}")
|
||||
# 捕获输出，避免密码提示
|
||||
# 捕获输出，避免密码提示 #BUG: 对于云服务器方式不能使用git pass密码验证，显示permission denied
|
||||
res = subprocess.run(cmd, capture_output=True, text=True, encoding='utf-8', errors='replace')
|
||||
|
||||
if res.returncode == 0:
|
||||
|
|
|
|||
Loading…
Reference in New Issue