README.md

@@ -1,24 +0,0 @@
-# 背景介绍
-
-开源项目的成功与否很大程度上有赖于贡献者的持续参与，其中长期贡献者尤为重要。开发者流失可能导致项目进展缓慢，缺乏新功能或改进，以及安全漏洞的长时间悬而未决。同时，开源项目的成功也吸引了很多公司的参与，比如Linux内核已经有超过1400家公司参与贡献。公司的撤资或减少投入可能会影响项目长期可持续性。因此，识别潜在的流失风险对开源项目的持续发展至关重要。本次赛题聚焦于预测开源软件项目中的贡献者流失（包括个体志愿者与公司两种类型），特别关注如Linux内核和Rust系统编程语言等基础开源软件项目以及国内兴起的开源项目。
-
-## 数据集提供
-
-数据集以.csv文件格式提供，包括以下两个开源项目的commit数据：
-
-- **Linux内核**：时间范围为2001年9月17日至2023年11月22日。
-- **Rust**：时间范围为2010年6月23日至2021年12月6日。
-
-数据集涵盖以下字段（但不限于）：
-
-- `author_name`
-- `author_email`
-- `author_date`
-- `subject`
-- `message`
-
-
-此外，可以访问以下GitHub链接，获取两个项目的最新数据，自行构建和更新数据集：
-
-- [Linux内核](https://github.com/torvalds/linux)
-- [Rust](https://github.com/rust-lang/rust)

README_V2.md

@@ -0,0 +1,98 @@
+# 开源项目贡献者流失预测系统
+
+## 项目结构
+```
+contributor_churn_prediction/
+│
+├── data/
+│ ├── linux_commits.csv
+│ └── rust_commits.csv
+│
+├── src/
+│ ├── data_preprocessing.py
+│ ├── feature_engineering.py
+│ ├── model.py
+│ ├── train.py
+│ └── predict.py
+│
+├── requirements.txt
+├── README_V2.md
+└── main.py
+```
+
+## 环境配置
+
+1. 确保您的系统已安装 Python 3.8+ 和 CUDA 11.2。
+
+2. 创建并激活虚拟环境：
+
+   ```
+   python -m venv churn_env
+   source churn_env/bin/activate  # Linux/macOS
+   churn_env\Scripts\activate  # Windows
+   ```
+
+3. 安装依赖：
+
+   ```
+   pip install -r requirements.txt
+   ```
+
+4. 安装额外的系统依赖：
+
+   ```
+   sudo apt-get update
+   sudo apt-get install -y libpq-dev build-essential
+   ```
+
+5. 配置环境变量：
+
+   ```
+   export PYTHONPATH="${PYTHONPATH}:/path/to/contributor_churn_prediction"
+   export DATA_DIR="/path/to/data"
+   export MODEL_CACHE="/path/to/model_cache"
+   ```
+
+## 运行代码
+
+1. 数据预处理：
+
+   ```
+   python src/data_preprocessing.py --input-dir $DATA_DIR --output-dir $DATA_DIR/processed
+   ```
+
+2. 特征工程：
+
+   ```
+   python src/feature_engineering.py --input-dir $DATA_DIR/processed --output-dir $DATA_DIR/features
+   ```
+
+3. 模型训练：
+
+   ```
+   python src/train.py --data-dir $DATA_DIR/features --model-dir $MODEL_CACHE --epochs 200 --batch-size 64 --learning-rate 0.0005
+   ```
+
+4. 预测：
+
+   ```
+   python src/predict.py --model-dir $MODEL_CACHE --data-dir $DATA_DIR/features --output-dir results
+   ```
+
+5. 生成报告：
+
+   ```
+   jupyter nbconvert --to notebook --execute notebooks/exploratory_data_analysis.ipynb --output-dir reports
+   ```
+
+注意：为确保结果的可重现性，请设置随机种子：
+```
+python
+import numpy as np
+import tensorflow as tf
+import random
+SEED = 42
+random.seed(SEED)
+np.random.seed(SEED)
+tf.random.set_seed(SEED)
+```

main.py

@@ -0,0 +1,94 @@
+import os
+import argparse
+import logging
+from datetime import datetime
+
+from src.data_preprocessing import load_data, preprocess_data, create_time_series
+from src.feature_engineering import engineer_features
+from src.model import create_model
+from src.train import train_model, evaluate_model
+from src.predict import predict_churn, interpret_predictions
+
+logging.basicConfig(filename='churn_prediction.log', level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+def setup_argparse():
+    parser = argparse.ArgumentParser(description='开源项目贡献者流失预测系统')
+    parser.add_argument('--data-dir', type=str, default='data', help='数据目录路径')
+    parser.add_argument('--output-dir', type=str, default='results', help='输出目录路径')
+    parser.add_argument('--epochs', type=int, default=100, help='训练轮数')
+    parser.add_argument('--batch-size', type=int, default=32, help='批次大小')
+    parser.add_argument('--learning-rate', type=float, default=0.001, help='学习率')
+    return parser.parse_args()
+
+def main():
+    args = setup_argparse()
+    
+    logger.info(f"开始运行流失预测系统 - {datetime.now()}")
+    
+    try:
+        # 数据预处理
+        logger.info("开始数据预处理")
+        linux_df = load_data(os.path.join(args.data_dir, 'linux_commits.csv'))
+        rust_df = load_data(os.path.join(args.data_dir, 'rust_commits.csv'))
+        
+        linux_df = preprocess_data(linux_df)
+        rust_df = preprocess_data(rust_df)
+        
+        linux_X, linux_y = create_time_series(linux_df)
+        rust_X, rust_y = create_time_series(rust_df)
+        
+        # 特征工程
+        logger.info("开始特征工程")
+        linux_X_engineered = engineer_features(linux_X)
+        rust_X_engineered = engineer_features(rust_X)
+        
+        # 创建模型
+        logger.info("创建模型")
+        input_shape = (linux_X_engineered.shape[1], linux_X_engineered.shape[2])
+        model = create_model(input_shape)
+        
+        # 训练模型
+        logger.info("开始训练模型")
+        linux_model, _ = train_model(model, linux_X_engineered, linux_y, 
+                                     epochs=args.epochs, batch_size=args.batch_size)
+        rust_model, _ = train_model(model, rust_X_engineered, rust_y, 
+                                    epochs=args.epochs, batch_size=args.batch_size)
+        
+        # 评估模型
+        logger.info("评估模型")
+        linux_loss, linux_accuracy = evaluate_model(linux_model, linux_X_engineered, linux_y)
+        rust_loss, rust_accuracy = evaluate_model(rust_model, rust_X_engineered, rust_y)
+        
+        logger.info(f"Linux模型 - 损失: {linux_loss:.4f}, 准确率: {linux_accuracy:.4f}")
+        logger.info(f"Rust模型 - 损失: {rust_loss:.4f}, 准确率: {rust_accuracy:.4f}")
+        
+        # 预测
+        logger.info("开始预测")
+        linux_predictions = predict_churn(linux_model, linux_X_engineered)
+        rust_predictions = predict_churn(rust_model, rust_X_engineered)
+        
+        linux_churn_prob, linux_churn_status = interpret_predictions(linux_predictions)
+        rust_churn_prob, rust_churn_status = interpret_predictions(rust_predictions)
+        
+        # 保存结果
+        os.makedirs(args.output_dir, exist_ok=True)
+        with open(os.path.join(args.output_dir, 'linux_results.txt'), 'w') as f:
+            f.write(f"Linux项目贡献者流失预测:\n")
+            f.write(f"流失概率: {linux_churn_prob[:5]}\n")
+            f.write(f"流失状态: {linux_churn_status[:5]}\n")
+        
+        with open(os.path.join(args.output_dir, 'rust_results.txt'), 'w') as f:
+            f.write(f"Rust项目贡献者流失预测:\n")
+            f.write(f"流失概率: {rust_churn_prob[:5]}\n")
+            f.write(f"流失状态: {rust_churn_status[:5]}\n")
+        
+        logger.info(f"预测结果已保存到 {args.output_dir}")
+        logger.info(f"流失预测系统运行完成 - {datetime.now()}")
+        
+    except Exception as e:
+        logger.error(f"运行过程中发生错误: {str(e)}")
+        raise
+
+if __name__ == '__main__':
+    main()

requirements.txt

@@ -0,0 +1,51 @@
+numpy
+pandas
+scikit-learn
+tensorflow
+keras
+matplotlib
+seaborn
+plotly
+dash
+jupyter
+notebook
+ipywidgets
+tqdm
+joblib
+pytest
+flake8
+black
+isort
+mypy
+pylint
+torch
+torchvision
+transformers
+fastapi
+uvicorn
+sqlalchemy
+psycopg2-binary
+redis
+celery
+flower
+beautifulsoup4
+requests
+aiohttp
+pyyaml
+toml
+click
+typer
+rich
+pydantic
+fastapi-utils
+python-dotenv
+gunicorn
+docker
+kubernetes
+boto3
+google-cloud-storage
+azure-storage-blob
+pymongo
+elasticsearch
+neo4j
+graphene

src/data_processing.py

@@ -0,0 +1,65 @@
+import pandas as pd
+import numpy as np
+from sklearn.preprocessing import LabelEncoder
+
+def load_data(file_path):
+    return pd.read_csv(file_path, parse_dates=['author_date', 'committer_date'])
+
+def preprocess_data(df):
+    df['year_month'] = df['author_date'].dt.to_period('M')
+    df['author_name'] = df['author_name'].fillna('Unknown')
+    df['author_email'] = df['author_email'].fillna('unknown@example.com')
+    
+    label_encoder = LabelEncoder()
+    df['author_id'] = label_encoder.fit_transform(df['author_email'])
+    
+    df['message_length'] = df['message'].fillna('').apply(len)
+    df['subject_length'] = df['subject'].fillna('').apply(len)
+    
+    df['is_self_commit'] = (df['author_name'] == df['committer_name']).astype(int)
+    
+    df['files_changed'] = df['num_files'].fillna(0)
+    df['lines_added'] = df['added_lines'].fillna(0)
+    df['lines_deleted'] = df['deleted_lines'].fillna(0)
+    
+    return df
+
+def create_time_series(df, time_window=6):
+    df_grouped = df.groupby(['author_id', 'year_month']).agg({
+        'rev': 'count',
+        'message_length': 'mean',
+        'subject_length': 'mean',
+        'is_self_commit': 'mean',
+        'files_changed': 'sum',
+        'lines_added': 'sum',
+        'lines_deleted': 'sum'
+    }).reset_index()
+    
+    df_grouped = df_grouped.sort_values(['author_id', 'year_month'])
+    
+    authors = df_grouped['author_id'].unique()
+    X = []
+    y = []
+    
+    for author in authors:
+        author_data = df_grouped[df_grouped['author_id'] == author]
+        for i in range(len(author_data) - time_window):
+            X.append(author_data.iloc[i:i+time_window].drop(['author_id', 'year_month'], axis=1).values)
+            y.append(1 if i + time_window + 1 >= len(author_data) else 0)
+    
+    return np.array(X), np.array(y)
+
+def main():
+    linux_df = load_data('data/linux_commits.csv')
+    rust_df = load_data('data/rust_commits.csv')
+    
+    linux_df = preprocess_data(linux_df)
+    rust_df = preprocess_data(rust_df)
+    
+    linux_X, linux_y = create_time_series(linux_df)
+    rust_X, rust_y = create_time_series(rust_df)
+    
+    return linux_X, linux_y, rust_X, rust_y
+
+if __name__ == '__main__':
+    main()

src/feature_engineering.py

@@ -0,0 +1,36 @@
+import numpy as np
+from sklearn.preprocessing import StandardScaler
+
+def engineer_features(X):
+    n_samples, n_timesteps, n_features = X.shape
+    X_reshaped = X.reshape(n_samples * n_timesteps, n_features)
+    
+    scaler = StandardScaler()
+    X_scaled = scaler.fit_transform(X_reshaped)
+    
+    X_engineered = X_scaled.reshape(n_samples, n_timesteps, n_features)
+    
+    X_diff = np.diff(X_engineered, axis=1)
+    X_diff = np.pad(X_diff, ((0, 0), (1, 0), (0, 0)), mode='constant')
+    
+    X_rolling_mean = np.cumsum(X_engineered, axis=1) / np.arange(1, n_timesteps + 1)
+    
+    X_rolling_std = np.zeros_like(X_engineered)
+    for i in range(1, n_timesteps + 1):
+        X_rolling_std[:, i-1, :] = np.std(X_engineered[:, :i, :], axis=1)
+    
+    X_final = np.concatenate([X_engineered, X_diff, X_rolling_mean, X_rolling_std], axis=2)
+    
+    return X_final
+
+def main():
+    from data_preprocessing import main as preprocess_main
+    linux_X, linux_y, rust_X, rust_y = preprocess_main()
+    
+    linux_X_engineered = engineer_features(linux_X)
+    rust_X_engineered = engineer_features(rust_X)
+    
+    return linux_X_engineered, linux_y, rust_X_engineered, rust_y
+
+if __name__ == '__main__':
+    main()

src/model.py

@@ -0,0 +1,33 @@
+import tensorflow as tf
+from tensorflow.keras.models import Sequential
+from tensorflow.keras.layers import LSTM, Dense, Dropout
+from tensorflow.keras.optimizers import Adam
+
+def create_model(input_shape, lstm_units=64, dropout_rate=0.3):
+    model = Sequential([
+        LSTM(lstm_units, return_sequences=True, input_shape=input_shape),
+        Dropout(dropout_rate),
+        LSTM(lstm_units),
+        Dropout(dropout_rate),
+        Dense(32, activation='relu'),
+        Dropout(dropout_rate),
+        Dense(1, activation='sigmoid')
+    ])
+    
+    model.compile(optimizer=Adam(learning_rate=0.001),
+                  loss='binary_crossentropy',
+                  metrics=['accuracy'])
+    
+    return model
+
+def main():
+    from feature_engineering import main as feature_main
+    linux_X, linux_y, rust_X, rust_y = feature_main()
+    
+    input_shape = (linux_X.shape[1], linux_X.shape[2])
+    model = create_model(input_shape)
+    
+    return model, linux_X, linux_y, rust_X, rust_y
+
+if __name__ == '__main__':
+    main()

src/predict.py

@@ -0,0 +1,34 @@
+import numpy as np
+
+def predict_churn(model, X):
+    predictions = model.predict(X)
+    return predictions
+
+def interpret_predictions(predictions, threshold=0.5):
+    churn_prob = predictions.flatten()
+    churn_status = (churn_prob >= threshold).astype(int)
+    return churn_prob, churn_status
+
+def main():
+    from train import main as train_main
+    linux_model, rust_model = train_main()
+    
+    from feature_engineering import main as feature_main
+    linux_X, _, rust_X, _ = feature_main()
+    
+    linux_predictions = predict_churn(linux_model, linux_X)
+    rust_predictions = predict_churn(rust_model, rust_X)
+    
+    linux_churn_prob, linux_churn_status = interpret_predictions(linux_predictions)
+    rust_churn_prob, rust_churn_status = interpret_predictions(rust_predictions)
+    
+    print("Linux项目贡献者流失预测:")
+    print(f"流失概率: {linux_churn_prob[:5]}")
+    print(f"流失状态: {linux_churn_status[:5]}")
+    
+    print("\nRust项目贡献者流失预测:")
+    print(f"流失概率: {rust_churn_prob[:5]}")
+    print(f"流失状态: {rust_churn_status[:5]}")
+
+if __name__ == '__main__':
+    main()

src/train.py

@@ -0,0 +1,41 @@
+import numpy as np
+from sklearn.model_selection import train_test_split
+from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint
+
+def train_model(model, X, y, validation_split=0.2, epochs=100, batch_size=32):
+    X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=validation_split, random_state=42)
+    
+    early_stopping = EarlyStopping(monitor='val_loss', patience=10, restore_best_weights=True)
+    model_checkpoint = ModelCheckpoint('best_model.h5', save_best_only=True, monitor='val_loss')
+    
+    history = model.fit(
+        X_train, y_train,
+        validation_data=(X_val, y_val),
+        epochs=epochs,
+        batch_size=batch_size,
+        callbacks=[early_stopping, model_checkpoint]
+    )
+    
+    return model, history
+
+def evaluate_model(model, X_test, y_test):
+    loss, accuracy = model.evaluate(X_test, y_test)
+    return loss, accuracy
+
+def main():
+    from model import main as model_main
+    model, linux_X, linux_y, rust_X, rust_y = model_main()
+    
+    linux_model, linux_history = train_model(model, linux_X, linux_y)
+    rust_model, rust_history = train_model(model, rust_X, rust_y)
+    
+    linux_loss, linux_accuracy = evaluate_model(linux_model, linux_X, linux_y)
+    rust_loss, rust_accuracy = evaluate_model(rust_model, rust_X, rust_y)
+    
+    print(f"Linux model - Loss: {linux_loss:.4f}, Accuracy: {linux_accuracy:.4f}")
+    print(f"Rust model - Loss: {rust_loss:.4f}, Accuracy: {rust_accuracy:.4f}")
+    
+    return linux_model, rust_model
+
+if __name__ == '__main__':
+    main()