feat: add data loader and fake test data
- JSONL loader with capability/scenario filtering and Pydantic validation - Sample-to-EvalSample conversion for eval scenario (strips Gold_Trace) - 6 fake samples across 5 capabilities with realistic research tasks - Each sample has complete 7-tuple structure with gold traces and negative cases Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
linlin
parent
8300e39063
commit
d03cc57bf7
Binary file not shown.
Binary file not shown.
Binary file not shown.
|
|
@ -0,0 +1,442 @@
|
|||
"""AIRBench fake data generator for testing.
|
||||
|
||||
Generates 2-3 realistic fake samples per capability and saves as JSONL files.
|
||||
Run with: python -m airbench.data.generator
|
||||
"""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from airbench.constants import Capability
|
||||
from airbench.schema import (
|
||||
EvalDimension,
|
||||
GoldTraceItem,
|
||||
LogicAnnotation,
|
||||
NegativeCase,
|
||||
RuleCriterion,
|
||||
Sample,
|
||||
SampleMeta,
|
||||
)
|
||||
|
||||
# ============================================================
|
||||
# C1: 知识理解与压缩
|
||||
# ============================================================
|
||||
|
||||
C1_SAMPLES = [
|
||||
{
|
||||
"meta": SampleMeta(
|
||||
sample_id="c1_001", capability=Capability.KNOWLEDGE_COMPRESSION,
|
||||
sub_task_type="paper_analysis", difficulty="medium", source="synthetic",
|
||||
tags=["推荐系统", "论文分析"],
|
||||
),
|
||||
"prompt": "对论文《Neural Collaborative Filtering》(He et al., 2017)的核心创新点进行抽取与批判性分析，要求覆盖方法论创新与实验验证两个维度。",
|
||||
"constraint": [
|
||||
"分析需覆盖方法论创新与实验验证两个维度",
|
||||
"需指出至少一个潜在局限性",
|
||||
"使用中文输出",
|
||||
"字数控制在500-800字",
|
||||
],
|
||||
"logic_annot": LogicAnnotation(
|
||||
prerequisites=["了解协同过滤基本原理", "了解神经网络基础架构"],
|
||||
causal_chain=["MF局限性分析→NN替代动机→NCF框架设计→实验验证"],
|
||||
assumptions=["读者具备推荐系统基础背景知识"],
|
||||
scope_boundary="仅分析该论文本身的贡献，不涉及后续改进工作",
|
||||
uncertainty="论文的实验仅覆盖MovieLens和Pinterest数据集，泛化性需进一步验证",
|
||||
),
|
||||
"gold_trace": [
|
||||
GoldTraceItem(
|
||||
approach_name="结构化分解法",
|
||||
thinking_process="首先明确论文的研究背景：传统矩阵分解用内积建模用户-物品交互，表达能力有限。然后定位核心创新：用多层神经网络替代内积，提出NCF框架，统一了GMF和MLP两种范式。接下来评估实验设计：在两个公开数据集上与多个基线对比，指标为Hit Ratio和NDCG。最后审视局限性：仅验证了隐式反馈场景，对显式评分场景的适用性未讨论。",
|
||||
final_output="【核心创新点】\n1. 方法论创新：提出NCF（Neural Collaborative Filtering）框架，用神经网络替代传统矩阵分解中的内积操作，增强了用户-物品交互建模的非线性表达能力。\n2. 架构设计：框架统一了GMF（广义矩阵分解）和MLP（多层感知机）两种建模范式，通过神经层进行特征交互而非手工设计交叉特征。\n3. 实验验证：在MovieLens和Pinterest数据集上，NCF在Hit Ratio和NDCG指标上均优于传统MF方法和item-popularity基线。\n\n【潜在局限】\n1. 仅验证了隐式反馈（点击/交互）场景，未涉及显式评分预测。\n2. 模型复杂度显著增加，但计算效率分析不足。\n3. 实验数据集偏向娱乐领域，对稀疏场景的泛化能力未验证。",
|
||||
quality_score=0.95,
|
||||
),
|
||||
GoldTraceItem(
|
||||
approach_name="对比批判法",
|
||||
thinking_process="我换一个视角：先列出论文要解决的问题，再对比现有解决方案，最后评估其增量贡献。论文目标是解决MF表达能力不足的问题。对比 NeuMF 和传统MF，核心差异是用可学习的非线性函数替代固定内积。但要注意，同期已有AutoRec等工作也在做类似的事情。",
|
||||
final_output="【问题定位】传统矩阵分解(MF)将用户和物品映射到共享隐空间，通过内积计算交互分数。内积操作本质上是线性加权求和，限制了模型捕获复杂交互模式的能力。\n\n【创新贡献】\n1. 提出用多层神经网络替代内积，理论上可拟合任意连续交互函数。\n2. 设计NeuMF模型，将GMF（保留MF线性特性）与MLP（引入非线性）通过拼接层融合，兼顾线性与非线性建模。\n3. 提供了预训练+微调的训练策略，解决深度模型收敛困难。\n\n【批判性分析】\n1. 与同期AutoRec(C Sedhain et al.)相比，创新增量有限——均是用NN增强CF。\n2. 负采样策略采用随机均匀采样，未探索更优的难负例采样方案。\n3. 消融实验对NeuMF的贡献度论证不够充分，GMF与MLP的融合增益未清晰量化。",
|
||||
quality_score=0.85,
|
||||
),
|
||||
],
|
||||
"neg_cases": [
|
||||
NegativeCase(
|
||||
error_type="hallucination",
|
||||
content="该论文在Amazon Reviews和Yelp数据集上也进行了广泛实验验证。",
|
||||
explanation="论文原始实验仅使用MovieLens和Pinterest数据集，未涉及Amazon和Yelp。",
|
||||
),
|
||||
NegativeCase(
|
||||
error_type="template_copy",
|
||||
content="本文提出了一种新颖的方法，在多个数据集上取得了优异的性能，具有重要的研究价值和应用前景。",
|
||||
explanation="空洞的模板式评价，没有具体指出创新点是什么、在什么数据集上、什么指标提升了多少。",
|
||||
),
|
||||
],
|
||||
"eval_dims": [
|
||||
EvalDimension(name="factual_accuracy", description="事实准确率：引用的论文内容是否准确", weight=0.25),
|
||||
EvalDimension(name="innovation_identification", description="创新点识别完整性", weight=0.25),
|
||||
EvalDimension(name="critical_depth", description="批判性分析深度", weight=0.25),
|
||||
EvalDimension(name="structure_clarity", description="输出结构清晰度", weight=0.25),
|
||||
],
|
||||
"rule_criteria": [
|
||||
RuleCriterion(rule_id="c1_r1", rule_name="创新点必须具体", rule_type="structural",
|
||||
check_description="创新点描述不能是泛泛的模板语句", keywords=["新颖的方法", "优异的性能", "重要价值"]),
|
||||
RuleCriterion(rule_id="c1_r2", rule_name="局限分析必选", rule_type="structural",
|
||||
check_description="必须包含至少一个局限性讨论", keywords=["局限", "不足", "未涉及", "未讨论"]),
|
||||
],
|
||||
},
|
||||
{
|
||||
"meta": SampleMeta(
|
||||
sample_id="c1_002", capability=Capability.KNOWLEDGE_COMPRESSION,
|
||||
sub_task_type="literature_review", difficulty="hard", source="synthetic",
|
||||
tags=["大语言模型", "文献综述", "RLHF"],
|
||||
),
|
||||
"prompt": "请对2023-2024年间关于RLHF(Reinforcement Learning from Human Feedback)的代表性工作进行综述归纳，重点比较不同RLHF变体在奖励模型训练、策略优化、对齐效果方面的差异。",
|
||||
"constraint": [
|
||||
"综述范围限定为2023-2024年公开发表的论文或技术报告",
|
||||
"需至少覆盖3个不同的RLHF改进方案",
|
||||
"需用对比表格呈现核心差异",
|
||||
"需讨论RLHF当前面临的主要挑战",
|
||||
],
|
||||
"logic_annot": LogicAnnotation(
|
||||
prerequisites=["了解强化学习基础(PPO等)", "了解大模型对齐基本概念"],
|
||||
causal_chain=["传统RLHF框架分析→各变体改进动机→技术路线对比→效果评估→挑战总结"],
|
||||
assumptions=["讨论的论文均为公开可获取的"],
|
||||
scope_boundary="仅覆盖RLHF改进方案，不涉及其他对齐方法(如DPO)的独立讨论",
|
||||
uncertainty="部分技术报告可能存在实验设置不一致的问题，对比结论需谨慎解读",
|
||||
),
|
||||
"gold_trace": [
|
||||
GoldTraceItem(
|
||||
approach_name="按技术维度对比法",
|
||||
thinking_process="先梳理标准RLHF流程：收集偏好数据→训练奖励模型→PPO优化策略。然后定位2023-2024年主要改进方向：(1)奖励模型改进(如RLAIF用AI反馈替代人类反馈)；(2)策略优化改进(如RRHF消除单独奖励模型的依赖)；(3)数据效率改进(如条件RLHF)。接下来构建对比表格，最后总结挑战。",
|
||||
final_output="【RLHF改进方案综述(2023-2024)】\n\n1. RLAIF (Lee et al., 2023): 用AI反馈(如GPT-4标注)替代人类反馈来训练奖励模型，降低了偏好数据获取成本。在Helpfulness和Harmlessness评测中接近人类标注效果。\n\n2. RRHF (Yuan et al., 2023): 将响应排序与策略优化统一到单一训练阶段，消除对独立奖励模型和PPO的依赖，简化了训练流程。\n\n3. Conditional RLHF (Yang et al., 2024): 引入条件奖励模型，根据不同对齐目标动态调整奖励函数，解决了单一奖励模型难以平衡多个对齐维度的问题。\n\n【对比表格】\n| 维度 | RLAIF | RRHF | Conditional RLHF |\n| 奖励模型 | AI标注替代人类标注 | 无需独立奖励模型 | 条件化奖励模型 |\n| 策略优化 | 标准PPO | 排序回归 | 条件PPO |\n| 数据成本 | 低(AI标注) | 中 | 中 |\n| 多目标支持 | 弱 | 弱 | 强 |\n\n【主要挑战】(1)奖励模型泛化性不足；(2)RL训练稳定性差；(3)人类偏好多样性建模困难。",
|
||||
quality_score=0.92,
|
||||
),
|
||||
],
|
||||
"neg_cases": [
|
||||
NegativeCase(
|
||||
error_type="logic_gap",
|
||||
content="RLHF是目前最好的对齐方法，所有大模型都使用RLHF进行对齐训练。",
|
||||
explanation="逻辑断层：(1)RLHF不是唯一方案，DPO、Constitutional AI等也是主流；(2)并非所有模型都使用RLHF，如Llama 2的部分版本。",
|
||||
),
|
||||
NegativeCase(
|
||||
error_type="false_innovation",
|
||||
content="本文提出了一种全新的RLHF方法，通过引入多智能体辩论来提升对齐质量。",
|
||||
explanation="虚假创新描述，将已有概念(多智能体辩论)简单套用到RLHF上但未给出具体技术细节和实验验证。",
|
||||
),
|
||||
],
|
||||
"eval_dims": [
|
||||
EvalDimension(name="coverage", description="文献覆盖完整性", weight=0.2),
|
||||
EvalDimension(name="comparison_depth", description="对比分析深度", weight=0.25),
|
||||
EvalDimension(name="factual_accuracy", description="引用准确性", weight=0.25),
|
||||
EvalDimension(name="challenge_insight", description="挑战洞察力", weight=0.15),
|
||||
EvalDimension(name="structure_quality", description="表格与结构质量", weight=0.15),
|
||||
],
|
||||
"rule_criteria": [
|
||||
RuleCriterion(rule_id="c1_r3", rule_name="对比表必须包含", rule_type="structural",
|
||||
check_description="综述必须包含对比表格", keywords=["|"]),
|
||||
RuleCriterion(rule_id="c1_r4", rule_name="至少3个方案", rule_type="structural",
|
||||
check_description="至少覆盖3个不同RLHF变体", keywords=["RLAIF", "RRHF", "Conditional", "DPO"]),
|
||||
RuleCriterion(rule_id="c1_r5", rule_name="挑战讨论必选", rule_type="structural",
|
||||
check_description="必须讨论主要挑战", keywords=["挑战", "问题", "困难", "不足"]),
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
# ============================================================
|
||||
# C2: 长程数理&因果推理
|
||||
# ============================================================
|
||||
|
||||
C2_SAMPLES = [
|
||||
{
|
||||
"meta": SampleMeta(
|
||||
sample_id="c2_001", capability=Capability.MATH_CAUSAL_REASONING,
|
||||
sub_task_type="mathematical_proof", difficulty="hard", source="synthetic",
|
||||
tags=["概率论", "贝叶斯推理", "数学推导"],
|
||||
),
|
||||
"prompt": "请完成以下推导：从贝叶斯定理出发，推导出高斯混合模型(GMM)的EM算法中E步的后验责任度γ(z_{nk})的完整表达式，并解释每一步的数学依据。",
|
||||
"constraint": [
|
||||
"推导过程必须逐步呈现，不能跳步",
|
||||
"每一步需标注使用的数学定理或性质",
|
||||
"需说明指数族分布的相关性质在推导中的作用",
|
||||
"最终表达式需与标准教科书一致",
|
||||
],
|
||||
"logic_annot": LogicAnnotation(
|
||||
prerequisites=["贝叶斯定理", "多元高斯分布", "指数族分布", "Jensen不等式"],
|
||||
causal_chain=["贝叶斯展开→联合分布分解→对数似然→E步推导→后验责任度"],
|
||||
assumptions=["读者了解概率论基本概念", "GMM模型设定已知"],
|
||||
scope_boundary="仅推导E步，不涉及M步参数更新公式的推导",
|
||||
uncertainty="EM算法只保证收敛到局部最优，推导过程不讨论全局最优性",
|
||||
),
|
||||
"gold_trace": [
|
||||
GoldTraceItem(
|
||||
approach_name="逐步展开法",
|
||||
thinking_process="先明确目标：求γ(z_{nk})=p(z_k=1|x_n)。从贝叶斯定理开始：p(z|x)=p(x|z)p(z)/p(x)。在GMM中，z是one-hot向量，先验p(z_k=1)=π_k，似然p(x|z_k=1)=N(x|μ_k,Σ_k)。分母p(x)=Σ_j π_j·N(x|μ_j,Σ_j)是全概率展开。关键点：分子分母的指数项可以展开和比较。",
|
||||
final_output="【推导过程】\n\n**Step 1: 贝叶斯定理展开**\n由贝叶斯定理：\nγ(z_{nk}) = p(z_{nk}=1|x_n) = p(x_n|z_{nk}=1)·p(z_{nk}=1) / p(x_n)\n依据：贝叶斯定理（条件概率定义）\n\n**Step 2: 代入GMM设定**\np(z_{nk}=1) = π_k （混合系数，即先验概率）\np(x_n|z_{nk}=1) = N(x_n|μ_k, Σ_k) = (2π)^{-D/2}|Σ_k|^{-1/2}exp{-½(x_n-μ_k)^T Σ_k^{-1}(x_n-μ_k)}\n依据：多元高斯分布定义\n\n**Step 3: 全概率公式展开分母**\np(x_n) = Σ_{j=1}^{K} π_j · N(x_n|μ_j, Σ_j)\n依据：全概率公式（Law of Total Probability）\n\n**Step 4: 合并表达式**\nγ(z_{nk}) = π_k · N(x_n|μ_k, Σ_k) / Σ_{j=1}^{K} π_j · N(x_n|μ_j, Σ_j)\n\n**指数族分布的作用**：高斯分布属于指数族分布，其概率密度可统一表示为exp{η^T·T(x)-A(η)}的形式，这一性质保证了EM算法的收敛性（E步计算后验时，指数项可线性叠加），且在M步推导中可利用充分统计量的性质简化求导。",
|
||||
quality_score=0.95,
|
||||
),
|
||||
],
|
||||
"neg_cases": [
|
||||
NegativeCase(
|
||||
error_type="logic_gap",
|
||||
content="γ(z_{nk}) = π_k · N(x_n|μ_k, Σ_k)，这就是后验责任度。",
|
||||
explanation="跳步严重：缺少分母的全概率展开，未说明推导依据，直接给出了不完整的公式。",
|
||||
),
|
||||
NegativeCase(
|
||||
error_type="hallucination",
|
||||
content="根据中心极限定理，当样本量足够大时，γ(z_{nk})趋近于均匀分布。",
|
||||
explanation="中心极限定理与GMM后验责任度计算无关，这是错误的数学引用。",
|
||||
),
|
||||
],
|
||||
"eval_dims": [
|
||||
EvalDimension(name="formula_correctness", description="公式推导正确性", weight=0.35),
|
||||
EvalDimension(name="step_completeness", description="推导步骤完整性", weight=0.25),
|
||||
EvalDimension(name="math_justification", description="每步数学依据是否标注", weight=0.25),
|
||||
EvalDimension(name="logic_consistency", description="前后逻辑一致性", weight=0.15),
|
||||
],
|
||||
"rule_criteria": [
|
||||
RuleCriterion(rule_id="c2_r1", rule_name="逐步推导", rule_type="structural",
|
||||
check_description="必须分步骤呈现推导", keywords=["Step", "步骤"]),
|
||||
RuleCriterion(rule_id="c2_r2", rule_name="数学依据标注", rule_type="logical",
|
||||
check_description="每步需标注使用的数学定理", keywords=["依据", "定理", "定义"]),
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
# ============================================================
|
||||
# C3: 任务规划与工具协同
|
||||
# ============================================================
|
||||
|
||||
C3_SAMPLES = [
|
||||
{
|
||||
"meta": SampleMeta(
|
||||
sample_id="c3_001", capability=Capability.TASK_PLANNING,
|
||||
sub_task_type="research_planning", difficulty="medium", source="synthetic",
|
||||
tags=["实验规划", "推荐系统", "A/B测试"],
|
||||
),
|
||||
"prompt": "设计一个完整的推荐系统算法改进实验方案：研究目标为「在保持点击率(CTR)的前提下，提升推荐多样性与长期用户满意度」。请规划完整的实验流程，包括离线评估方案、A/B测试设计、所需工具与资源。",
|
||||
"constraint": [
|
||||
"实验需基于PyTorch框架",
|
||||
"离线评估需包含至少3个多样性指标",
|
||||
"A/B测试需明确样本量计算方法和统计显著性标准",
|
||||
"算力限制为4张A100 GPU",
|
||||
"需考虑冷启动场景的特殊处理",
|
||||
],
|
||||
"logic_annot": LogicAnnotation(
|
||||
prerequisites=["了解推荐系统基本架构", "了解A/B测试统计原理"],
|
||||
causal_chain=["问题定义→基线选定→离线实验设计→多样性指标选取→A/B测试设计→工具资源规划→风险预案"],
|
||||
assumptions=["已有用户历史行为数据", "已有线上推荐服务基础设施"],
|
||||
scope_boundary="仅覆盖实验设计，不涉及具体算法实现细节",
|
||||
uncertainty="多样性指标与长期满意度之间的因果关系需要A/B测试验证，离线指标可能无法完全代表线上效果",
|
||||
),
|
||||
"gold_trace": [
|
||||
GoldTraceItem(
|
||||
approach_name="分阶段规划法",
|
||||
thinking_process="这是一个复杂的实验规划任务，需要拆解为：1.明确实验目标与约束；2.选择基线模型；3.设计离线实验方案；4.选取评估指标；5.设计A/B测试；6.规划工具和资源；7.制定风险预案。注意约束条件：4张A100、PyTorch框架、3个多样性指标。",
|
||||
final_output="【推荐多样性改进实验方案】\n\n**一、实验目标**\n主目标：提升推荐多样性（ILS指标提升≥15%）的同时保持CTR下降≤2%。\n次目标：7日用户留存率提升≥3%。\n\n**二、基线模型**\n- 离线基线：GRU4Rec + MLP排序模型\n- 线上基线：当前线上排序模型\n\n**三、离线实验设计**\n1. 数据集：内部用户行为日志（近90天），按时间切分train/val/test=8:1:1\n2. 待测方案：MMR(Marginal Diversity)、DPP、对比学习多样性正则化\n3. 多样性指标：\n - ILS(Intra-List Similarity)：推荐列表内物品相似度均值\n - Coverage：被推荐物品占总物品比例\n - Entropy：推荐分布的信息熵\n4. 准确性指标：HR@10, NDCG@10, CTR预估AUC\n\n**四、A/B测试设计**\n1. 样本量：基于效应量d=0.05, α=0.05, power=0.8，每组约需50万用户\n2. 实验分组：对照组(现有模型) + 3个实验组(3种多样性方案)\n3. 实验周期：14天（含3天预热期）\n4. 统计显著性：p<0.05，使用Bonferroni校正多重比较\n\n**五、工具与资源**\n- 训练：PyTorch + DeepSpeed，4×A100(80GB)\n- 特征工程：Spark + Hive\n- 离线评估：自建Python评测框架\n- A/B测试：公司内部实验平台\n\n**六、冷启动处理**\n- 新用户：基于热门物品+类别多样性采样的兜底策略\n- 新物品：基于内容的相似度计算接入推荐池\n\n**七、风险预案**\n- 离线多样性提升但CTR下降超阈值→调整多样性权重，重新离线验证\n- A/B测试期间线上指标异常→触发自动回滚机制",
|
||||
quality_score=0.90,
|
||||
),
|
||||
],
|
||||
"neg_cases": [
|
||||
NegativeCase(
|
||||
error_type="logic_gap",
|
||||
content="直接在推荐模型中加入随机采样来提升多样性即可，不需要复杂的实验设计。",
|
||||
explanation="逻辑断层：随机采样会严重影响推荐准确性，未考虑准确性与多样性的权衡，未满足约束条件中的指标要求。",
|
||||
),
|
||||
NegativeCase(
|
||||
error_type="template_copy",
|
||||
content="我们将在未来工作中进行A/B测试验证。",
|
||||
explanation="空洞模板，没有给出任何A/B测试的具体设计方案。",
|
||||
),
|
||||
],
|
||||
"eval_dims": [
|
||||
EvalDimension(name="task_decomposition", description="任务拆解完整度", weight=0.20),
|
||||
EvalDimension(name="feasibility", description="方案可行性", weight=0.20),
|
||||
EvalDimension(name="constraint_satisfaction", description="约束满足度", weight=0.20),
|
||||
EvalDimension(name="tool_rationality", description="工具选择合理性", weight=0.15),
|
||||
EvalDimension(name="risk_awareness", description="风险预案完备度", weight=0.15),
|
||||
EvalDimension(name="statistical_rigor", description="统计设计严谨性", weight=0.10),
|
||||
],
|
||||
"rule_criteria": [
|
||||
RuleCriterion(rule_id="c3_r1", rule_name="多样性指标≥3", rule_type="structural",
|
||||
check_description="离线评估需包含至少3个多样性指标", keywords=["ILS", "Coverage", "Entropy"]),
|
||||
RuleCriterion(rule_id="c3_r2", rule_name="A/B测试设计", rule_type="structural",
|
||||
check_description="A/B测试需包含样本量计算和显著性标准", keywords=["样本量", "显著性", "p<"]),
|
||||
RuleCriterion(rule_id="c3_r3", rule_name="资源约束", rule_type="structural",
|
||||
check_description="需说明算力和工具约束下的可行性", keywords=["A100", "GPU", "PyTorch"]),
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
# ============================================================
|
||||
# C4: 空白挖掘与跨域创新
|
||||
# ============================================================
|
||||
|
||||
C4_SAMPLES = [
|
||||
{
|
||||
"meta": SampleMeta(
|
||||
sample_id="c4_001", capability=Capability.GAP_MINING_INNOVATION,
|
||||
sub_task_type="research_gap_identification", difficulty="hard", source="synthetic",
|
||||
tags=["跨域创新", "知识蒸馏", "联邦学习"],
|
||||
),
|
||||
"prompt": "识别「联邦学习」与「知识蒸馏」两个领域的交叉研究空白，提出2-3个具有原创性的研究方向，要求每个方向包含：研究动机、核心研究问题、预期技术路线、潜在挑战。",
|
||||
"constraint": [
|
||||
"研究方向需区别于已有的FedKD类工作",
|
||||
"需从方法论层面而非纯应用层面提出创新",
|
||||
"每个方向需明确与现有工作的差异点",
|
||||
"需评估每个方向的可行性（高/中/低）",
|
||||
],
|
||||
"logic_annot": LogicAnnotation(
|
||||
prerequisites=["了解联邦学习基本框架", "了解知识蒸馏原理"],
|
||||
causal_chain=["两个领域现状分析→交叉点识别→空白挖掘→创新方向提出→可行性评估"],
|
||||
assumptions=["研究需面向学术前沿，非工程落地导向"],
|
||||
scope_boundary="仅提出研究方向框架，不涉及具体实现",
|
||||
uncertainty="创新方向的原创性需经文献调研确认，可能存在预印本已覆盖的情况",
|
||||
),
|
||||
"gold_trace": [
|
||||
GoldTraceItem(
|
||||
approach_name="双向交叉分析",
|
||||
thinking_process="先梳理两个领域的核心问题：联邦学习解决数据孤岛+隐私，知识蒸馏解决模型压缩+知识迁移。已有交叉工作FedKD主要是把蒸馏用作联邦通信压缩。还有哪些未覆盖的交叉点？(1)蒸馏过程中的隐私泄漏风险→研究联邦蒸馏中的隐私保护；(2)异构模型蒸馏→联邦环境中客户端模型架构不同时的蒸馏策略；(3)反事实蒸馏→用联邦数据增强蒸馏样本的多样性。",
|
||||
final_output="【联邦学习×知识蒸馏：交叉研究空白与方向】\n\n**方向一：联邦蒸馏中的隐私泄漏量化与防御**\n- 研究动机：知识蒸馏过程中，教师模型的logits/softmax输出可能泄露训练数据信息。在联邦场景下，全局模型向客户端进行蒸馏时，存在隐私放大风险，但现有FedKD工作未量化此风险。\n- 核心问题：蒸馏通信过程中，攻击者能从蒸馏信号中重构多少原始训练数据？\n- 技术路线：(1)建立蒸馏隐私泄漏的理论界(基于互信息)；(2)设计差分隐私蒸馏机制，在logits中注入可控噪声；(3)在CIFAR-10/100上验证隐私-效用权衡。\n- 潜在挑战：噪声注入对蒸馏效果的负面影响可能显著大于对普通训练的影响。\n- 与现有工作差异：FedKD关注蒸馏效率，本方向关注蒸馏安全。\n- 可行性：高\n\n**方向二：异构联邦环境下的自适应蒸馏架构搜索**\n- 研究动机：实际联邦场景中，客户端设备算力差异大（手机vs服务器），需不同大小的模型。传统蒸馏假设固定教师-学生架构对，不适应异构场景。\n- 核心问题：如何为异构客户端动态选择最优蒸馏策略（包括教师-学生配对、蒸馏层、温度参数）？\n- 技术路线：(1)建立客户端能力画像模型；(2)设计NAS-guided蒸馏架构搜索空间；(3)设计多教师蒸馏路由机制。\n- 潜在挑战：搜索空间过大，联邦通信开销可能抵消蒸馏收益。\n- 与现有工作差异：现有异构FL用模型切割/知识共享，本方向用自适应蒸馏搜索。\n- 可行性：中\n\n**方向三：基于因果推理的联邦反事实蒸馏**\n- 研究动机：标准蒸馏传递的是教师学到的相关性知识（correlation），而非因果知识（causation）。在联邦场景中，各客户端数据分布不同(Non-IID)，相关性知识迁移可能失败。\n- 核心问题：能否通过因果干预生成反事实蒸馏样本，使蒸馏知识具有分布不变的因果性？\n- 技术路线：(1)用结构因果模型(SCM)建模客户端数据生成过程；(2)通过do-calculus生成反事实特征；(3)用反事实特征进行跨域蒸馏。\n- 潜在挑战：SCM的准确建模需要领域先验，联邦场景下难以获取统一因果图。\n- 与现有工作差异：现有FedKD传递相关性知识，本方向传递因果性知识。\n- 可行性：低（但创新性最高）",
|
||||
quality_score=0.93,
|
||||
),
|
||||
],
|
||||
"neg_cases": [
|
||||
NegativeCase(
|
||||
error_type="false_innovation",
|
||||
content="将联邦学习和知识蒸馏结合起来，可以同时获得两个方法的优点。",
|
||||
explanation="空洞的模板式创新描述，未指出具体创新点、未与已有工作区分、无技术细节。",
|
||||
),
|
||||
NegativeCase(
|
||||
error_type="logic_gap",
|
||||
content="联邦蒸馏不需要考虑隐私问题，因为蒸馏已经是在传递知识而非数据。",
|
||||
explanation="逻辑错误：蒸馏信号本身可能泄露训练数据信息，已有研究表明logits可用于成员推断攻击。",
|
||||
),
|
||||
],
|
||||
"eval_dims": [
|
||||
EvalDimension(name="novelty", description="方向原创性", weight=0.30),
|
||||
EvalDimension(name="research_depth", description="研究问题深度", weight=0.20),
|
||||
EvalDimension(name="technical_feasibility", description="技术路线可行性", weight=0.20),
|
||||
EvalDimension(name="differentiation", description="与现有工作差异明确度", weight=0.15),
|
||||
EvalDimension(name="challenge_awareness", description="潜在挑战识别", weight=0.15),
|
||||
],
|
||||
"rule_criteria": [
|
||||
RuleCriterion(rule_id="c4_r1", rule_name="非FedKD类", rule_type="logical",
|
||||
check_description="研究方向需明确区别于FedKD", keywords=["区别", "差异", "不同于"]),
|
||||
RuleCriterion(rule_id="c4_r2", rule_name="方法论创新", rule_type="logical",
|
||||
check_description="创新需在方法论层面而非纯应用", keywords=["方法", "框架", "机制", "理论"]),
|
||||
RuleCriterion(rule_id="c4_r3", rule_name="可行性标注", rule_type="structural",
|
||||
check_description="每个方向需标注可行性等级", keywords=["高", "中", "低"]),
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
# ============================================================
|
||||
# C5: 工程实证与实验迭代
|
||||
# ============================================================
|
||||
|
||||
C5_SAMPLES = [
|
||||
{
|
||||
"meta": SampleMeta(
|
||||
sample_id="c5_001", capability=Capability.ENGINEERING_EXPERIMENT,
|
||||
sub_task_type="experiment_design", difficulty="medium", source="synthetic",
|
||||
tags=["实验设计", "消融实验", "模型调试"],
|
||||
),
|
||||
"prompt": "针对一个基于Transformer的文本分类模型，当发现模型在验证集上的loss在训练中期开始回升（出现过拟合趋势）时，请设计一套系统的实验方案来诊断原因并进行优化，要求包含消融实验设计。",
|
||||
"constraint": [
|
||||
"方案需覆盖至少4种可能的过拟合原因",
|
||||
"每种原因需设计对应的验证实验",
|
||||
"消融实验需包含控制变量说明",
|
||||
"需给出实验优先级排序及理由",
|
||||
"最终需给出推荐的解决方案组合",
|
||||
],
|
||||
"logic_annot": LogicAnnotation(
|
||||
prerequisites=["了解Transformer架构", "了解正则化方法", "了解过拟合诊断"],
|
||||
causal_chain=["现象观察→原因假设→实验设计→消融验证→方案推荐"],
|
||||
assumptions=["训练数据和验证数据划分正确", "模型架构基本合理"],
|
||||
scope_boundary="仅针对过拟合诊断与优化，不涉及模型架构重新设计",
|
||||
uncertainty="多种因素可能同时导致过拟合，单一消融实验可能无法完全隔离各因素贡献",
|
||||
),
|
||||
"gold_trace": [
|
||||
GoldTraceItem(
|
||||
approach_name="假设驱动诊断法",
|
||||
thinking_process="过拟合loss回升可能的原因：(1)训练数据不足/噪声大；(2)模型容量过大；(3)学习率过高导致跳出最优点；(4)缺乏正则化；(5)数据分布偏移。按可验证性和成本排序设计实验。",
|
||||
final_output="【Transformer文本分类模型过拟合诊断方案】\n\n**一、过拟合原因假设（按优先级排序）**\n\n| 优先级 | 原因假设 | 验证方法 | 预期判据 |\n| P0 | 学习率过高 | 降低lr至1/10，观察loss曲线 | 若loss回升消失则确认 |\n| P1 | 缺乏正则化 | 加入Dropout(0.1→0.3)、Weight Decay | 若val_loss改善则确认 |\n| P2 | 模型容量过大 | 减少Transformer层数/注意力头数 | 若过拟合延迟则确认 |\n| P3 | 数据量不足 | 用数据增强(back-translation等)扩充训练集 | 若过拟合缓解则确认 |\n\n**二、消融实验设计**\n\n实验基线：当前模型配置，固定seed\n控制变量原则：每次仅改变一个因素\n\n实验1：学习率消融\n- lr∈{1e-5, 3e-5, 5e-5, 1e-4}，其他固定\n- 监控：train_loss, val_loss, val_acc\n- 预期：找到loss回升前的最优lr\n\n实验2：正则化消融\n- Dropout∈{0.1, 0.2, 0.3, 0.5}\n- Weight Decay∈{0, 0.01, 0.1}\n- 交叉组合，共4×3=12组\n- 预期：找到最优正则化组合\n\n实验3：模型容量消融\n- Transformer层数∈{2, 4, 6, 12}\n- Attention头数∈{4, 8}\n- 共4×2=8组\n- 预期：找到容量与泛化的平衡点\n\n实验4：数据增强消融\n- 无增强 vs Back-translation vs EDA vs 混合\n- 监控：训练集大小变化对过拟合拐点的影响\n\n**三、推荐解决方案组合**\n1. 首选：lr=3e-5 + Dropout=0.2 + Weight Decay=0.01（低成本高收益）\n2. 次选：方案1 + 模型层数减半（如果首选取效不足）\n3. 兜底：方案2 + 数据增强（如果仍有过拟合）\n\n**四、实验执行注意事项**\n- 使用EarlyStopping(patience=5)防止无效训练\n- 每组实验重复3次取均值，报告标准差\n- 使用TensorBoard记录完整训练曲线",
|
||||
quality_score=0.92,
|
||||
),
|
||||
],
|
||||
"neg_cases": [
|
||||
NegativeCase(
|
||||
error_type="invalid_experiment",
|
||||
content="直接把模型换成BERT就行了，不需要做消融实验。",
|
||||
explanation="无效实验建议：未诊断具体原因就换模型，且BERT本身也是Transformer，同样可能过拟合。",
|
||||
),
|
||||
NegativeCase(
|
||||
error_type="logic_gap",
|
||||
content="过拟合是因为模型太好了，只需要增加训练数据就能解决。",
|
||||
explanation="逻辑过于简化：过拟合原因多样，增加数据只是方案之一，且未考虑数据获取成本和质量问题。",
|
||||
),
|
||||
],
|
||||
"eval_dims": [
|
||||
EvalDimension(name="cause_coverage", description="原因假设覆盖度", weight=0.20),
|
||||
EvalDimension(name="experiment_rigor", description="消融实验严谨性", weight=0.25),
|
||||
EvalDimension(name="priority_rationality", description="优先级排序合理性", weight=0.15),
|
||||
EvalDimension(name="reproducibility", description="实验可复现性", weight=0.20),
|
||||
EvalDimension(name="solution_quality", description="推荐方案质量", weight=0.20),
|
||||
],
|
||||
"rule_criteria": [
|
||||
RuleCriterion(rule_id="c5_r1", rule_name="原因≥4种", rule_type="structural",
|
||||
check_description="需覆盖至少4种过拟合原因", keywords=["学习率", "正则化", "模型容量", "数据"]),
|
||||
RuleCriterion(rule_id="c5_r2", rule_name="控制变量", rule_type="logical",
|
||||
check_description="消融实验需说明控制变量", keywords=["控制", "固定", "仅改变"]),
|
||||
RuleCriterion(rule_id="c5_r3", rule_name="可复现性要素", rule_type="structural",
|
||||
check_description="需包含seed、重复次数等复现要素", keywords=["seed", "重复", "均值", "标准差"]),
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
# ============================================================
|
||||
# Generator main
|
||||
# ============================================================
|
||||
|
||||
ALL_SAMPLES = {
|
||||
Capability.KNOWLEDGE_COMPRESSION: C1_SAMPLES,
|
||||
Capability.MATH_CAUSAL_REASONING: C2_SAMPLES,
|
||||
Capability.TASK_PLANNING: C3_SAMPLES,
|
||||
Capability.GAP_MINING_INNOVATION: C4_SAMPLES,
|
||||
Capability.ENGINEERING_EXPERIMENT: C5_SAMPLES,
|
||||
}
|
||||
|
||||
CAPABILITY_FILE_MAP = {
|
||||
Capability.KNOWLEDGE_COMPRESSION: "c1_knowledge_compression.jsonl",
|
||||
Capability.MATH_CAUSAL_REASONING: "c2_math_causal_reasoning.jsonl",
|
||||
Capability.TASK_PLANNING: "c3_task_planning.jsonl",
|
||||
Capability.GAP_MINING_INNOVATION: "c4_gap_mining_innovation.jsonl",
|
||||
Capability.ENGINEERING_EXPERIMENT: "c5_engineering_experiment.jsonl",
|
||||
}
|
||||
|
||||
|
||||
def generate_fake_data(output_dir: str = "data/fake") -> None:
|
||||
"""Generate all fake data JSONL files."""
|
||||
output_path = Path(output_dir)
|
||||
output_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
total = 0
|
||||
for capability, raw_samples in ALL_SAMPLES.items():
|
||||
filename = CAPABILITY_FILE_MAP[capability]
|
||||
filepath = output_path / filename
|
||||
|
||||
samples = []
|
||||
for raw in raw_samples:
|
||||
sample = Sample(**raw)
|
||||
samples.append(sample)
|
||||
|
||||
with filepath.open("w", encoding="utf-8") as fh:
|
||||
for sample in samples:
|
||||
fh.write(sample.model_dump_json() + "\n")
|
||||
|
||||
total += len(samples)
|
||||
print(f" Generated {len(samples)} samples -> {filepath}")
|
||||
|
||||
print(f"\nTotal: {total} fake samples generated in {output_path}/")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
generate_fake_data()
|
||||
|
|
@ -0,0 +1,147 @@
|
|||
"""AIRBench JSONL data loader.
|
||||
|
||||
Loads .jsonl files where each line is a Sample, validates against the
|
||||
Pydantic schema, and supports filtering by capability and scenario.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Union
|
||||
|
||||
from pydantic import ValidationError
|
||||
|
||||
from airbench.constants import Capability
|
||||
from airbench.schema import EvalSample, Sample
|
||||
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Conversion
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def sample_to_eval(sample: Sample) -> EvalSample:
|
||||
"""Strip gold_trace from a Sample to produce an EvalSample."""
|
||||
return EvalSample(
|
||||
meta=sample.meta,
|
||||
prompt=sample.prompt,
|
||||
constraint=sample.constraint,
|
||||
logic_annot=sample.logic_annot,
|
||||
neg_cases=sample.neg_cases,
|
||||
eval_dims=sample.eval_dims,
|
||||
rule_criteria=sample.rule_criteria,
|
||||
)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Core loader
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def load_jsonl(
|
||||
path: str | Path,
|
||||
*,
|
||||
scenario: str = "train",
|
||||
capability: Capability | None = None,
|
||||
) -> list[Union[Sample, EvalSample]]:
|
||||
"""Load a JSONL file and return a list of Sample or EvalSample objects.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
path : str | Path
|
||||
Path to the .jsonl file.
|
||||
scenario : str
|
||||
``"train"`` returns full Sample objects (with gold_trace).
|
||||
``"eval"`` returns EvalSample objects (gold_trace stripped).
|
||||
capability : Capability | None
|
||||
If set, only return samples whose ``meta.capability`` matches.
|
||||
|
||||
Returns
|
||||
-------
|
||||
list[Sample | EvalSample]
|
||||
Validated samples in the requested scenario view.
|
||||
|
||||
Raises
|
||||
------
|
||||
FileNotFoundError
|
||||
If *path* does not exist.
|
||||
ValueError
|
||||
If a line fails Pydantic validation.
|
||||
"""
|
||||
path = Path(path)
|
||||
if not path.exists():
|
||||
raise FileNotFoundError(f"Data file not found: {path}")
|
||||
|
||||
results: list[Union[Sample, EvalSample]] = []
|
||||
|
||||
with path.open("r", encoding="utf-8") as fh:
|
||||
for line_num, raw_line in enumerate(fh, start=1):
|
||||
raw_line = raw_line.strip()
|
||||
if not raw_line:
|
||||
continue
|
||||
|
||||
try:
|
||||
data = json.loads(raw_line)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise ValueError(
|
||||
f"Invalid JSON at {path.name}:{line_num}: {exc}"
|
||||
) from exc
|
||||
|
||||
try:
|
||||
sample = Sample.model_validate(data)
|
||||
except ValidationError as exc:
|
||||
raise ValueError(
|
||||
f"Schema validation failed at {path.name}:{line_num}: {exc}"
|
||||
) from exc
|
||||
|
||||
# Filter by capability
|
||||
if capability is not None and sample.meta.capability != capability:
|
||||
continue
|
||||
|
||||
if scenario == "eval":
|
||||
results.append(sample_to_eval(sample))
|
||||
else:
|
||||
results.append(sample)
|
||||
|
||||
return results
|
||||
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Convenience helpers
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def load_samples(
|
||||
path: str | Path,
|
||||
*,
|
||||
capability: Capability | None = None,
|
||||
) -> list[Sample]:
|
||||
"""Load a JSONL file returning full Sample objects (train scenario)."""
|
||||
return load_jsonl(path, scenario="train", capability=capability) # type: ignore[return-value]
|
||||
|
||||
|
||||
def load_eval_samples(
|
||||
path: str | Path,
|
||||
*,
|
||||
capability: Capability | None = None,
|
||||
) -> list[EvalSample]:
|
||||
"""Load a JSONL file returning EvalSample objects (eval scenario)."""
|
||||
return load_jsonl(path, scenario="eval", capability=capability) # type: ignore[return-value]
|
||||
|
||||
|
||||
def load_directory(
|
||||
directory: str | Path,
|
||||
*,
|
||||
scenario: str = "train",
|
||||
capability: Capability | None = None,
|
||||
) -> list[Union[Sample, EvalSample]]:
|
||||
"""Load all .jsonl files in a directory.
|
||||
|
||||
Files are processed in sorted order for deterministic results.
|
||||
"""
|
||||
directory = Path(directory)
|
||||
if not directory.is_dir():
|
||||
raise FileNotFoundError(f"Directory not found: {directory}")
|
||||
|
||||
results: list[Union[Sample, EvalSample]] = []
|
||||
for jsonl_file in sorted(directory.glob("*.jsonl")):
|
||||
results.extend(load_jsonl(jsonl_file, scenario=scenario, capability=capability))
|
||||
return results
|
||||
File diff suppressed because one or more lines are too long
|
|
@ -0,0 +1 @@
|
|||
{"meta":{"sample_id":"c2_001","capability":"math_causal_reasoning","sub_task_type":"mathematical_proof","difficulty":"hard","source":"synthetic","tags":["概率论","贝叶斯推理","数学推导"]},"prompt":"请完成以下推导：从贝叶斯定理出发，推导出高斯混合模型(GMM)的EM算法中E步的后验责任度γ(z_{nk})的完整表达式，并解释每一步的数学依据。","constraint":["推导过程必须逐步呈现，不能跳步","每一步需标注使用的数学定理或性质","需说明指数族分布的相关性质在推导中的作用","最终表达式需与标准教科书一致"],"logic_annot":{"prerequisites":["贝叶斯定理","多元高斯分布","指数族分布","Jensen不等式"],"causal_chain":["贝叶斯展开→联合分布分解→对数似然→E步推导→后验责任度"],"assumptions":["读者了解概率论基本概念","GMM模型设定已知"],"scope_boundary":"仅推导E步，不涉及M步参数更新公式的推导","uncertainty":"EM算法只保证收敛到局部最优，推导过程不讨论全局最优性"},"gold_trace":[{"approach_name":"逐步展开法","thinking_process":"先明确目标：求γ(z_{nk})=p(z_k=1|x_n)。从贝叶斯定理开始：p(z|x)=p(x|z)p(z)/p(x)。在GMM中，z是one-hot向量，先验p(z_k=1)=π_k，似然p(x|z_k=1)=N(x|μ_k,Σ_k)。分母p(x)=Σ_j π_j·N(x|μ_j,Σ_j)是全概率展开。关键点：分子分母的指数项可以展开和比较。","final_output":"【推导过程】\n\n**Step 1: 贝叶斯定理展开**\n由贝叶斯定理：\nγ(z_{nk}) = p(z_{nk}=1|x_n) = p(x_n|z_{nk}=1)·p(z_{nk}=1) / p(x_n)\n依据：贝叶斯定理（条件概率定义）\n\n**Step 2: 代入GMM设定**\np(z_{nk}=1) = π_k （混合系数，即先验概率）\np(x_n|z_{nk}=1) = N(x_n|μ_k, Σ_k) = (2π)^{-D/2}|Σ_k|^{-1/2}exp{-½(x_n-μ_k)^T Σ_k^{-1}(x_n-μ_k)}\n依据：多元高斯分布定义\n\n**Step 3: 全概率公式展开分母**\np(x_n) = Σ_{j=1}^{K} π_j · N(x_n|μ_j, Σ_j)\n依据：全概率公式（Law of Total Probability）\n\n**Step 4: 合并表达式**\nγ(z_{nk}) = π_k · N(x_n|μ_k, Σ_k) / Σ_{j=1}^{K} π_j · N(x_n|μ_j, Σ_j)\n\n**指数族分布的作用**：高斯分布属于指数族分布，其概率密度可统一表示为exp{η^T·T(x)-A(η)}的形式，这一性质保证了EM算法的收敛性（E步计算后验时，指数项可线性叠加），且在M步推导中可利用充分统计量的性质简化求导。","quality_score":0.95}],"neg_cases":[{"error_type":"logic_gap","content":"γ(z_{nk}) = π_k · N(x_n|μ_k, Σ_k)，这就是后验责任度。","explanation":"跳步严重：缺少分母的全概率展开，未说明推导依据，直接给出了不完整的公式。"},{"error_type":"hallucination","content":"根据中心极限定理，当样本量足够大时，γ(z_{nk})趋近于均匀分布。","explanation":"中心极限定理与GMM后验责任度计算无关，这是错误的数学引用。"}],"eval_dims":[{"name":"formula_correctness","description":"公式推导正确性","weight":0.35,"scoring_criteria":{}},{"name":"step_completeness","description":"推导步骤完整性","weight":0.25,"scoring_criteria":{}},{"name":"math_justification","description":"每步数学依据是否标注","weight":0.25,"scoring_criteria":{}},{"name":"logic_consistency","description":"前后逻辑一致性","weight":0.15,"scoring_criteria":{}}],"rule_criteria":[{"rule_id":"c2_r1","rule_name":"逐步推导","rule_type":"structural","check_description":"必须分步骤呈现推导","keywords":["Step","步骤"]},{"rule_id":"c2_r2","rule_name":"数学依据标注","rule_type":"logical","check_description":"每步需标注使用的数学定理","keywords":["依据","定理","定义"]}]}
|
||||
|
|
@ -0,0 +1 @@
|
|||
{"meta":{"sample_id":"c3_001","capability":"task_planning","sub_task_type":"research_planning","difficulty":"medium","source":"synthetic","tags":["实验规划","推荐系统","A/B测试"]},"prompt":"设计一个完整的推荐系统算法改进实验方案：研究目标为「在保持点击率(CTR)的前提下，提升推荐多样性与长期用户满意度」。请规划完整的实验流程，包括离线评估方案、A/B测试设计、所需工具与资源。","constraint":["实验需基于PyTorch框架","离线评估需包含至少3个多样性指标","A/B测试需明确样本量计算方法和统计显著性标准","算力限制为4张A100 GPU","需考虑冷启动场景的特殊处理"],"logic_annot":{"prerequisites":["了解推荐系统基本架构","了解A/B测试统计原理"],"causal_chain":["问题定义→基线选定→离线实验设计→多样性指标选取→A/B测试设计→工具资源规划→风险预案"],"assumptions":["已有用户历史行为数据","已有线上推荐服务基础设施"],"scope_boundary":"仅覆盖实验设计，不涉及具体算法实现细节","uncertainty":"多样性指标与长期满意度之间的因果关系需要A/B测试验证，离线指标可能无法完全代表线上效果"},"gold_trace":[{"approach_name":"分阶段规划法","thinking_process":"这是一个复杂的实验规划任务，需要拆解为：1.明确实验目标与约束；2.选择基线模型；3.设计离线实验方案；4.选取评估指标；5.设计A/B测试；6.规划工具和资源；7.制定风险预案。注意约束条件：4张A100、PyTorch框架、3个多样性指标。","final_output":"【推荐多样性改进实验方案】\n\n**一、实验目标**\n主目标：提升推荐多样性（ILS指标提升≥15%）的同时保持CTR下降≤2%。\n次目标：7日用户留存率提升≥3%。\n\n**二、基线模型**\n- 离线基线：GRU4Rec + MLP排序模型\n- 线上基线：当前线上排序模型\n\n**三、离线实验设计**\n1. 数据集：内部用户行为日志（近90天），按时间切分train/val/test=8:1:1\n2. 待测方案：MMR(Marginal Diversity)、DPP、对比学习多样性正则化\n3. 多样性指标：\n - ILS(Intra-List Similarity)：推荐列表内物品相似度均值\n - Coverage：被推荐物品占总物品比例\n - Entropy：推荐分布的信息熵\n4. 准确性指标：HR@10, NDCG@10, CTR预估AUC\n\n**四、A/B测试设计**\n1. 样本量：基于效应量d=0.05, α=0.05, power=0.8，每组约需50万用户\n2. 实验分组：对照组(现有模型) + 3个实验组(3种多样性方案)\n3. 实验周期：14天（含3天预热期）\n4. 统计显著性：p<0.05，使用Bonferroni校正多重比较\n\n**五、工具与资源**\n- 训练：PyTorch + DeepSpeed，4×A100(80GB)\n- 特征工程：Spark + Hive\n- 离线评估：自建Python评测框架\n- A/B测试：公司内部实验平台\n\n**六、冷启动处理**\n- 新用户：基于热门物品+类别多样性采样的兜底策略\n- 新物品：基于内容的相似度计算接入推荐池\n\n**七、风险预案**\n- 离线多样性提升但CTR下降超阈值→调整多样性权重，重新离线验证\n- A/B测试期间线上指标异常→触发自动回滚机制","quality_score":0.9}],"neg_cases":[{"error_type":"logic_gap","content":"直接在推荐模型中加入随机采样来提升多样性即可，不需要复杂的实验设计。","explanation":"逻辑断层：随机采样会严重影响推荐准确性，未考虑准确性与多样性的权衡，未满足约束条件中的指标要求。"},{"error_type":"template_copy","content":"我们将在未来工作中进行A/B测试验证。","explanation":"空洞模板，没有给出任何A/B测试的具体设计方案。"}],"eval_dims":[{"name":"task_decomposition","description":"任务拆解完整度","weight":0.2,"scoring_criteria":{}},{"name":"feasibility","description":"方案可行性","weight":0.2,"scoring_criteria":{}},{"name":"constraint_satisfaction","description":"约束满足度","weight":0.2,"scoring_criteria":{}},{"name":"tool_rationality","description":"工具选择合理性","weight":0.15,"scoring_criteria":{}},{"name":"risk_awareness","description":"风险预案完备度","weight":0.15,"scoring_criteria":{}},{"name":"statistical_rigor","description":"统计设计严谨性","weight":0.1,"scoring_criteria":{}}],"rule_criteria":[{"rule_id":"c3_r1","rule_name":"多样性指标≥3","rule_type":"structural","check_description":"离线评估需包含至少3个多样性指标","keywords":["ILS","Coverage","Entropy"]},{"rule_id":"c3_r2","rule_name":"A/B测试设计","rule_type":"structural","check_description":"A/B测试需包含样本量计算和显著性标准","keywords":["样本量","显著性","p<"]},{"rule_id":"c3_r3","rule_name":"资源约束","rule_type":"structural","check_description":"需说明算力和工具约束下的可行性","keywords":["A100","GPU","PyTorch"]}]}
|
||||
File diff suppressed because one or more lines are too long
|
|
@ -0,0 +1 @@
|
|||
{"meta":{"sample_id":"c5_001","capability":"engineering_experiment","sub_task_type":"experiment_design","difficulty":"medium","source":"synthetic","tags":["实验设计","消融实验","模型调试"]},"prompt":"针对一个基于Transformer的文本分类模型，当发现模型在验证集上的loss在训练中期开始回升（出现过拟合趋势）时，请设计一套系统的实验方案来诊断原因并进行优化，要求包含消融实验设计。","constraint":["方案需覆盖至少4种可能的过拟合原因","每种原因需设计对应的验证实验","消融实验需包含控制变量说明","需给出实验优先级排序及理由","最终需给出推荐的解决方案组合"],"logic_annot":{"prerequisites":["了解Transformer架构","了解正则化方法","了解过拟合诊断"],"causal_chain":["现象观察→原因假设→实验设计→消融验证→方案推荐"],"assumptions":["训练数据和验证数据划分正确","模型架构基本合理"],"scope_boundary":"仅针对过拟合诊断与优化，不涉及模型架构重新设计","uncertainty":"多种因素可能同时导致过拟合，单一消融实验可能无法完全隔离各因素贡献"},"gold_trace":[{"approach_name":"假设驱动诊断法","thinking_process":"过拟合loss回升可能的原因：(1)训练数据不足/噪声大；(2)模型容量过大；(3)学习率过高导致跳出最优点；(4)缺乏正则化；(5)数据分布偏移。按可验证性和成本排序设计实验。","final_output":"【Transformer文本分类模型过拟合诊断方案】\n\n**一、过拟合原因假设（按优先级排序）**\n\n| 优先级 | 原因假设 | 验证方法 | 预期判据 |\n| P0 | 学习率过高 | 降低lr至1/10，观察loss曲线 | 若loss回升消失则确认 |\n| P1 | 缺乏正则化 | 加入Dropout(0.1→0.3)、Weight Decay | 若val_loss改善则确认 |\n| P2 | 模型容量过大 | 减少Transformer层数/注意力头数 | 若过拟合延迟则确认 |\n| P3 | 数据量不足 | 用数据增强(back-translation等)扩充训练集 | 若过拟合缓解则确认 |\n\n**二、消融实验设计**\n\n实验基线：当前模型配置，固定seed\n控制变量原则：每次仅改变一个因素\n\n实验1：学习率消融\n- lr∈{1e-5, 3e-5, 5e-5, 1e-4}，其他固定\n- 监控：train_loss, val_loss, val_acc\n- 预期：找到loss回升前的最优lr\n\n实验2：正则化消融\n- Dropout∈{0.1, 0.2, 0.3, 0.5}\n- Weight Decay∈{0, 0.01, 0.1}\n- 交叉组合，共4×3=12组\n- 预期：找到最优正则化组合\n\n实验3：模型容量消融\n- Transformer层数∈{2, 4, 6, 12}\n- Attention头数∈{4, 8}\n- 共4×2=8组\n- 预期：找到容量与泛化的平衡点\n\n实验4：数据增强消融\n- 无增强 vs Back-translation vs EDA vs 混合\n- 监控：训练集大小变化对过拟合拐点的影响\n\n**三、推荐解决方案组合**\n1. 首选：lr=3e-5 + Dropout=0.2 + Weight Decay=0.01（低成本高收益）\n2. 次选：方案1 + 模型层数减半（如果首选取效不足）\n3. 兜底：方案2 + 数据增强（如果仍有过拟合）\n\n**四、实验执行注意事项**\n- 使用EarlyStopping(patience=5)防止无效训练\n- 每组实验重复3次取均值，报告标准差\n- 使用TensorBoard记录完整训练曲线","quality_score":0.92}],"neg_cases":[{"error_type":"invalid_experiment","content":"直接把模型换成BERT就行了，不需要做消融实验。","explanation":"无效实验建议：未诊断具体原因就换模型，且BERT本身也是Transformer，同样可能过拟合。"},{"error_type":"logic_gap","content":"过拟合是因为模型太好了，只需要增加训练数据就能解决。","explanation":"逻辑过于简化：过拟合原因多样，增加数据只是方案之一，且未考虑数据获取成本和质量问题。"}],"eval_dims":[{"name":"cause_coverage","description":"原因假设覆盖度","weight":0.2,"scoring_criteria":{}},{"name":"experiment_rigor","description":"消融实验严谨性","weight":0.25,"scoring_criteria":{}},{"name":"priority_rationality","description":"优先级排序合理性","weight":0.15,"scoring_criteria":{}},{"name":"reproducibility","description":"实验可复现性","weight":0.2,"scoring_criteria":{}},{"name":"solution_quality","description":"推荐方案质量","weight":0.2,"scoring_criteria":{}}],"rule_criteria":[{"rule_id":"c5_r1","rule_name":"原因≥4种","rule_type":"structural","check_description":"需覆盖至少4种过拟合原因","keywords":["学习率","正则化","模型容量","数据"]},{"rule_id":"c5_r2","rule_name":"控制变量","rule_type":"logical","check_description":"消融实验需说明控制变量","keywords":["控制","固定","仅改变"]},{"rule_id":"c5_r3","rule_name":"可复现性要素","rule_type":"structural","check_description":"需包含seed、重复次数等复现要素","keywords":["seed","重复","均值","标准差"]}]}
|
||||
Loading…
Reference in New Issue