diff --git a/基于AI Agent开发范式的国产GPU大模型推理算子库优化/baselines/flashinfer_baseline/summarize_results.py b/基于AI Agent开发范式的国产GPU大模型推理算子库优化/baselines/flashinfer_baseline/summarize_results.py new file mode 100644 index 0000000..7c2b8eb --- /dev/null +++ b/基于AI Agent开发范式的国产GPU大模型推理算子库优化/baselines/flashinfer_baseline/summarize_results.py @@ -0,0 +1,135 @@ +"""Summarize FlashInfer benchmark CSV files into a Markdown report.""" + +import argparse +import csv +from collections import Counter, defaultdict +from pathlib import Path + + +NUMERIC_COLUMNS = ("time_ms", "bandwidth_GB_s", "tflops") + + +def parse_float(value): + try: + return float(value) + except (TypeError, ValueError): + return None + + +def load_records(paths): + records = [] + for path in paths: + with path.open(newline="") as f: + reader = csv.DictReader(f) + for row in reader: + row["_source"] = str(path) + records.append(row) + return records + + +def group_by_api(records): + grouped = defaultdict(list) + for record in records: + grouped[record.get("api", "unknown")].append(record) + return dict(grouped) + + +def best_record(records, column, reverse=True): + candidates = [] + for record in records: + value = parse_float(record.get(column)) + if value is not None: + candidates.append((value, record)) + if not candidates: + return None + return sorted(candidates, key=lambda item: item[0], reverse=reverse)[0] + + +def describe_case(record): + case_keys = [ + "batch_size", + "seq_len", + "seq_len_q", + "seq_len_kv", + "num_heads", + "num_qo_heads", + "num_kv_heads", + "head_dim", + "head_dim_qk", + "head_dim_vo", + "head_dim_ckv", + "head_dim_kpe", + ] + parts = [f"{key}={record[key]}" for key in case_keys if record.get(key)] + return ", ".join(parts) if parts else "n/a" + + +def build_markdown(records): + lines = [ + "# FlashInfer Benchmark Summary", + "", + f"- CSV files: {len({record['_source'] for record in records})}", + f"- Total rows: {len(records)}", + "", + ] + + for api, api_records in sorted(group_by_api(records).items()): + statuses = Counter(record.get("status", "ok") or "ok" for record in api_records) + lines.extend( + [ + f"## {api}", + "", + f"- Rows: {len(api_records)}", + f"- Status: {', '.join(f'{key}={value}' for key, value in sorted(statuses.items()))}", + ] + ) + + for column in NUMERIC_COLUMNS: + best = best_record(api_records, column, reverse=(column != "time_ms")) + if best is None: + continue + value, record = best + lines.append(f"- Best {column}: {value:.6g} ({describe_case(record)})") + + failures = [ + record + for record in api_records + if record.get("status") and record.get("status") != "ok" + ] + for failure in failures[:5]: + lines.append( + f"- Failure: {describe_case(failure)}: " + f"{failure.get('error_type', 'error')} {failure.get('error', '')}".strip() + ) + if len(failures) > 5: + lines.append(f"- Additional failures omitted: {len(failures) - 5}") + lines.append("") + + return "\n".join(lines) + + +def parse_args(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("csv", nargs="+", type=Path, help="Benchmark CSV files.") + parser.add_argument( + "--output", + type=Path, + default=Path("flashinfer_benchmark_summary.md"), + help="Markdown summary output path.", + ) + return parser.parse_args() + + +def main(): + args = parse_args() + records = load_records(args.csv) + if not records: + raise ValueError("no benchmark rows found") + markdown = build_markdown(records) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(markdown, encoding="utf-8") + print(f"Summary saved to {args.output}") + + +if __name__ == "__main__": + main()