Dockerfile

@@ -0,0 +1,37 @@
+FROM nvidia/cuda:11.4.2-cudnn8-devel-ubuntu20.04
+
+ENV DEBIAN_FRONTEND=noninteractive
+ENV PATH="/root/miniconda3/bin:${PATH}"
+ARG PATH="/root/miniconda3/bin:${PATH}"
+
+RUN apt-get update && apt-get install -y \
+    wget \
+    git \
+    build-essential \
+    libpq-dev \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh \
+    && mkdir /root/.conda \
+    && bash Miniconda3-latest-Linux-x86_64.sh -b \
+    && rm -f Miniconda3-latest-Linux-x86_64.sh
+
+RUN conda create -n cvss_env python=3.8 -y
+
+SHELL ["conda", "run", "-n", "cvss_env", "/bin/bash", "-c"]
+
+WORKDIR /app
+
+COPY requirements.txt .
+
+RUN pip install --no-cache-dir -r requirements.txt
+
+RUN python -m spacy download en_core_web_sm
+
+COPY . .
+
+RUN chmod +x entrypoint.sh
+
+ENTRYPOINT ["./entrypoint.sh"]
+
+CMD ["python", "main.py"]

README.md

@@ -1,14 +0,0 @@
-## :rocket: 背景
-
-近年来，开源软件供应链遭受持续的软件投毒和恶意代码攻击，造成了无法估计的损失。例如，Apache Log4j2远程代码执行漏洞被认为是近10年最严重的漏洞之一，攻击者可以在目标服务器上执行任意代码和嗅探系统信息。网络安全专家认为Log4j 中的远程代码执行漏洞可能需要数月甚至数年时间才能得到妥善解决。受Log4J漏洞影响组件包括Apache的Struts2、Solr、Druid、Flink等，Github上60,644个开源项目发布321,094软件存在风险。因此，当前急需智能化技术辅助降低漏洞风险，提高漏洞工程能力，减少漏洞损失。
-
-
-## :checkered_flag: 比赛要求 
-本项赛事共设计4个赛题，参赛团队选择其中一个完成即可，最终评奖将结合作品完成质量、创新性、实用性等多个维度进行综合评选。
-
-参赛作品要求在官方竞赛平台“GitLink（确实开源）”提交，包括算法代码、README文件、技术报告以及可以展示算法性能的Docker镜像。具体要求参见赛事网站的“参赛指南”。
-
-注：推荐使用开源大模型。
-
-
-

README_V2.md

@@ -0,0 +1,56 @@
+# CVSS Assessment System
+## 项目结构
+```
+CVSS_Assessment_System/
+├── data_processing/
+│   ├── data_loader.py
+│   ├── data_cleaner.py
+│   └── feature_extractor.py
+├── model_architecture/
+│   ├── deep_learning_model.py
+│   ├── machine_learning_model.py
+│   └── ensemble_model.py
+├── training_pipeline/
+│   ├── model_trainer.py
+│   ├── hyperparameter_tuner.py
+│   └── cross_validator.py
+├── evaluation/
+│   ├── mse_calculator.py
+│   ├── rmse_calculator.py
+│   └── performance_visualizer.py
+├── utils/
+│   ├── config_manager.py
+│   ├── logger.py
+│   └── data_visualizer.py
+├── main.py
+├── requirements.txt
+├── Dockerfile
+├── config.yaml
+```
+
+## 环境配置
+```
+   pip install -r requirements.txt
+    export PYTHONPATH="${PYTHONPATH}:${PWD}"
+   export CVSS_CONFIG_PATH="${PWD}/config.yaml"
+```
+
+使用Docker(可选):
+```
+   docker build -t cvss-assessment:v1.0 .
+   docker run -it --rm -v ${PWD}/data:/app/data -v ${PWD}/output:/app/output cvss-assessment:v1.0
+```
+## 运行代码
+1. 准备数据：
+将数据集放在 ./data 目录下，确保有 SIR_train_set.json, SIR_validation_set.json 和 SIR_test_set.json 文件。
+2. 配置参数：
+编辑 config.yaml 文件，设置适当的参数。
+3. 运行代码：
+```
+   python main.py --config config.yaml --mode full
+```
+
+## 注意事项
+1. 确保使用兼容的 CUDA 版本（如果使用 GPU）
+2. 某些模型可能需要大量内存，请确保有足够的系统资源
+3. 完整的实验可能需要一段时间，请耐心等待

config.yaml

@@ -0,0 +1,117 @@
+# CVSS Assessment System Configuration
+
+# 数据处理配置
+data:
+  data_dir: ./data
+  train_file: SIR_train_set.json
+  validation_file: SIR_validation_set.json
+  test_file: SIR_test_set.json
+  feature_extraction:
+    text_max_features: 1000
+    use_tfidf: true
+    numeric_features:
+      - baseScore
+      - impactScore
+      - exploitabilityScore
+    categorical_features:
+      - severity
+
+# 模型配置
+models:
+  deep_learning:
+    hidden_layers: [256, 128, 64, 32]
+    dropout_rate: 0.3
+    activation: relu
+    optimizer: adam
+    learning_rate: 0.001
+    batch_size: 32
+    epochs: 100
+  random_forest:
+    n_estimators: 100
+    max_depth: null
+    min_samples_split: 2
+    min_samples_leaf: 1
+  svr:
+    kernel: rbf
+    C: 1.0
+    epsilon: 0.1
+  ensemble:
+    use_models: [deep_learning, random_forest, svr]
+    ensemble_method: voting
+
+# 训练配置
+training:
+  cross_validation:
+    n_splits: 5
+    shuffle: true
+    random_state: 42
+  hyperparameter_tuning:
+    method: random
+    n_iter: 50
+    cv: 3
+  early_stopping:
+    patience: 10
+    min_delta: 0.001
+
+# 评估配置
+evaluation:
+  metrics:
+    - mse
+    - rmse
+  visualizations:
+    - feature_importance
+    - learning_curve
+    - confusion_matrix
+    - roc_curve
+
+# 日志配置
+logging:
+  log_dir: ./logs
+  log_level: INFO
+  log_format: '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+
+# 输出配置
+output:
+  results_dir: ./results
+  save_models: true
+  generate_report: true
+
+# 系统配置
+system:
+  random_seed: 42
+  use_gpu: true
+  num_workers: 4
+  memory_limit: 8G
+
+# 实验配置
+experiments:
+  EXP001:
+    description: "Baseline experiment with default settings"
+  EXP002:
+    description: "Experiment with increased model complexity"
+    models:
+      deep_learning:
+        hidden_layers: [512, 256, 128, 64]
+      random_forest:
+        n_estimators: 200
+  EXP003:
+    description: "Experiment with different feature extraction"
+    data:
+      feature_extraction:
+        text_max_features: 2000
+        use_tfidf: false
+
+# 高级配置
+advanced:
+  use_mlflow: true
+  mlflow:
+    tracking_uri: http://localhost:5000
+    experiment_name: CVSS_Assessment
+  use_optuna: true
+  optuna:
+    study_name: CVSS_Optimization
+    n_trials: 100
+  use_ray: true
+  ray:
+    num_cpus: 8
+    num_gpus: 1

data/label_word_ids.json

@@ -0,0 +1,46 @@
+{
+    "AV": {
+        "network": 2897,
+        "adjacent": 5516,
+        "local": 2334,
+        "physical": 3558
+    },
+    "AC": {
+        "low": 2659,
+        "high": 2152
+    },
+    "PR": {
+        "none": 3904,
+        "low": 2659,
+        "high": 2152
+    },
+    "UI": {
+        "none": 3904,
+        "required": 3223
+    },
+    "S": {
+        "unchanged": 15704,
+        "changed": 2904
+    },
+    "C": {
+        "none": 3904,
+        "low": 2659,
+        "high": 2152
+    },
+    "I": {
+        "none": 3904,
+        "low": 2659,
+        "high": 2152
+    },
+    "A": {
+        "none": 3904,
+        "low": 2659,
+        "high": 2152
+    },
+    "severity": {
+        "low": 2659,
+        "medium": 5396,
+        "high": 2152,
+        "critical": 4187
+    }
+}

data/label_word_ids_CVSS2.json

@@ -0,0 +1,37 @@
+{
+    "AV": {
+        "network": 2897,
+        "adjacent": 5516,
+        "local": 2334
+    },
+    "AC": {
+        "low": 2659,
+        "medium": 5396,
+        "high": 2152
+    },
+    "Au": {
+        "none": 3904,
+        "single": 2309,
+        "multiple": 3674
+    },
+    "C": {
+        "none": 3904,
+        "partial": 7704,
+        "complete": 3143
+    },
+    "I": {
+        "none": 3904,
+        "partial": 7704,
+        "complete": 3143
+    },
+    "A": {
+        "none": 3904,
+        "partial": 7704,
+        "complete": 3143
+    },
+    "severity": {
+        "low": 2659,
+        "medium": 5396,
+        "high": 2152
+    }
+}

data_processing/__init__.py

@@ -0,0 +1,5 @@
+from .data_loader import DataLoader
+from .data_cleaner import DataCleaner
+from .feature_extractor import FeatureExtractor
+
+__all__ = ['DataLoader', 'DataCleaner', 'FeatureExtractor']

data_processing/data_cleaner.py

@@ -0,0 +1,23 @@
+import pandas as pd
+import re
+
+class DataCleaner:
+    def __init__(self):
+        pass
+
+    def clean_data(self, df):
+        df = df.copy()
+        df['description'] = df['description'].apply(self._clean_description)
+        df['vectorString'] = df['vectorString'].apply(self._clean_vector_string)
+        df['baseScore'] = pd.to_numeric(df['baseScore'], errors='coerce')
+        df['impactScore'] = pd.to_numeric(df['impactScore'], errors='coerce')
+        df['exploitabilityScore'] = pd.to_numeric(df['exploitabilityScore'], errors='coerce')
+        return df
+
+    def _clean_description(self, text):
+        text = re.sub(r'NUMBERTAG|APITAG|FILETAG|ERRORTAG', '', text)
+        text = re.sub(r'\s+', ' ', text)
+        return text.strip()
+
+    def _clean_vector_string(self, text):
+        return text.replace('CVSS:3.1/', '')

data_processing/data_loader.py

@@ -0,0 +1,30 @@
+import json
+import os
+import pandas as pd
+
+class DataLoader:
+    def __init__(self, data_dir):
+        self.data_dir = data_dir
+        self.train_data = None
+        self.validation_data = None
+        self.test_data = None
+
+    def load_data(self):
+        self.train_data = self._load_json_file('SIR_train_set.json')
+        self.validation_data = self._load_json_file('SIR_validation_set.json')
+        self.test_data = self._load_json_file('SIR_test_set.json')
+
+    def _load_json_file(self, filename):
+        file_path = os.path.join(self.data_dir, filename)
+        with open(file_path, 'r') as file:
+            data = json.load(file)
+        return pd.DataFrame(data)
+
+    def get_train_data(self):
+        return self.train_data
+
+    def get_validation_data(self):
+        return self.validation_data
+
+    def get_test_data(self):
+        return self.test_data

data_processing/feature_extractor.py

@@ -0,0 +1,48 @@
+from sklearn.feature_extraction.text import TfidfVectorizer
+from sklearn.preprocessing import OneHotEncoder
+import pandas as pd
+import numpy as np
+
+class FeatureExtractor:
+    def __init__(self):
+        self.tfidf = TfidfVectorizer(max_features=1000)
+        self.onehot = OneHotEncoder(sparse=False, handle_unknown='ignore')
+
+    def fit_transform(self, df):
+        description_features = self.tfidf.fit_transform(df['description'])
+        vector_features = self._extract_vector_features(df['vectorString'])
+        categorical_features = self.onehot.fit_transform(df[['severity']])
+        
+        numerical_features = df[['baseScore', 'impactScore', 'exploitabilityScore']].values
+        
+        all_features = np.hstack([
+            description_features.toarray(),
+            vector_features,
+            categorical_features,
+            numerical_features
+        ])
+        
+        return all_features
+
+    def transform(self, df):
+        description_features = self.tfidf.transform(df['description'])
+        vector_features = self._extract_vector_features(df['vectorString'])
+        categorical_features = self.onehot.transform(df[['severity']])
+        
+        numerical_features = df[['baseScore', 'impactScore', 'exploitabilityScore']].values
+        
+        all_features = np.hstack([
+            description_features.toarray(),
+            vector_features,
+            categorical_features,
+            numerical_features
+        ])
+        
+        return all_features
+
+    def _extract_vector_features(self, vector_strings):
+        features = []
+        for vector in vector_strings:
+            vector_dict = dict(item.split(':') for item in vector.split('/'))
+            features.append([vector_dict.get(key, '0') for key in ['AV', 'AC', 'PR', 'UI', 'S', 'C', 'I', 'A']])
+        return np.array(features)

evaluation/__init__.py

@@ -0,0 +1,5 @@
+from .mse_calculator import MSECalculator
+from .rmse_calculator import RMSECalculator
+from .performance_visualizer import PerformanceVisualizer
+
+__all__ = ['MSECalculator', 'RMSECalculator', 'PerformanceVisualizer']

evaluation/mse_calculator.py

@@ -0,0 +1,20 @@
+import numpy as np
+from sklearn.metrics import mean_squared_error
+
+class MSECalculator:
+    def __init__(self):
+        self.mse_scores = []
+
+    def calculate(self, y_true, y_pred):
+        mse = mean_squared_error(y_true, y_pred)
+        self.mse_scores.append(mse)
+        return mse
+
+    def get_average_mse(self):
+        return np.mean(self.mse_scores)
+
+    def get_mse_std(self):
+        return np.std(self.mse_scores)
+
+    def reset(self):
+        self.mse_scores = []

evaluation/performance_visualizer.py

@@ -0,0 +1,62 @@
+import matplotlib.pyplot as plt
+import seaborn as sns
+import pandas as pd
+import numpy as np
+
+class PerformanceVisualizer:
+    def __init__(self):
+        self.results = {}
+
+    def add_result(self, model_name, mse, rmse):
+        self.results[model_name] = {'MSE': mse, 'RMSE': rmse}
+
+    def plot_comparison(self, metric='MSE'):
+        plt.figure(figsize=(10, 6))
+        data = [(model, scores[metric]) for model, scores in self.results.items()]
+        df = pd.DataFrame(data, columns=['Model', metric])
+        sns.barplot(x='Model', y=metric, data=df)
+        plt.title(f'{metric} Comparison Across Models')
+        plt.ylabel(metric)
+        plt.xticks(rotation=45)
+        plt.tight_layout()
+        plt.show()
+
+    def plot_error_distribution(self, y_true, y_pred, model_name):
+        errors = y_true - y_pred
+        plt.figure(figsize=(10, 6))
+        sns.histplot(errors, kde=True)
+        plt.title(f'Error Distribution for {model_name}')
+        plt.xlabel('Error')
+        plt.ylabel('Frequency')
+        plt.tight_layout()
+        plt.show()
+
+    def plot_actual_vs_predicted(self, y_true, y_pred, model_name):
+        plt.figure(figsize=(10, 6))
+        plt.scatter(y_true, y_pred, alpha=0.5)
+        plt.plot([y_true.min(), y_true.max()], [y_true.min(), y_true.max()], 'r--', lw=2)
+        plt.title(f'Actual vs Predicted for {model_name}')
+        plt.xlabel('Actual')
+        plt.ylabel('Predicted')
+        plt.tight_layout()
+        plt.show()
+
+    def plot_residuals(self, y_true, y_pred, model_name):
+        residuals = y_true - y_pred
+        plt.figure(figsize=(10, 6))
+        plt.scatter(y_pred, residuals, alpha=0.5)
+        plt.hlines(y=0, xmin=y_pred.min(), xmax=y_pred.max(), colors='r', linestyles='--')
+        plt.title(f'Residual Plot for {model_name}')
+        plt.xlabel('Predicted')
+        plt.ylabel('Residuals')
+        plt.tight_layout()
+        plt.show()
+
+    def generate_summary_report(self):
+        report = "Performance Summary Report\n"
+        report += "==========================\n\n"
+        for model, scores in self.results.items():
+            report += f"Model: {model}\n"
+            report += f"  MSE: {scores['MSE']:.4f}\n"
+            report += f"  RMSE: {scores['RMSE']:.4f}\n\n"
+        return report

evaluation/rmse_calculator.py

@@ -0,0 +1,20 @@
+import numpy as np
+from sklearn.metrics import mean_squared_error
+
+class RMSECalculator:
+    def __init__(self):
+        self.rmse_scores = []
+
+    def calculate(self, y_true, y_pred):
+        rmse = np.sqrt(mean_squared_error(y_true, y_pred))
+        self.rmse_scores.append(rmse)
+        return rmse
+
+    def get_average_rmse(self):
+        return np.mean(self.rmse_scores)
+
+    def get_rmse_std(self):
+        return np.std(self.rmse_scores)
+
+    def reset(self):
+        self.rmse_scores = []

main.py

@@ -0,0 +1,90 @@
+import os
+import numpy as np
+from data_processing import DataLoader, DataCleaner, FeatureExtractor
+from model_architecture import DeepLearningModel, MachineLearningModel, EnsembleModel
+from training_pipeline import ModelTrainer, HyperparameterTuner, CrossValidator
+from evaluation import MSECalculator, RMSECalculator, PerformanceVisualizer
+from utils import ConfigManager, Logger, DataVisualizer
+
+def main():
+    # 初始化配置管理器和日志记录器
+    config = ConfigManager('config.yaml')
+    logger = Logger(config.get('log_dir', 'logs'))
+
+    logger.info("Starting CVSS Assessment System")
+
+    # 数据加载和预处理
+    data_loader = DataLoader(config.get('data_dir', 'data'))
+    data_loader.load_data()
+    
+    data_cleaner = DataCleaner()
+    feature_extractor = FeatureExtractor()
+
+    train_data = data_cleaner.clean_data(data_loader.get_train_data())
+    val_data = data_cleaner.clean_data(data_loader.get_validation_data())
+    test_data = data_cleaner.clean_data(data_loader.get_test_data())
+
+    X_train = feature_extractor.fit_transform(train_data)
+    y_train = train_data['baseScore'].values
+
+    X_val = feature_extractor.transform(val_data)
+    y_val = val_data['baseScore'].values
+
+    X_test = feature_extractor.transform(test_data)
+    y_test = test_data['baseScore'].values
+
+    # 数据可视化
+    data_visualizer = DataVisualizer()
+    data_visualizer.plot_distribution(train_data, 'baseScore', 'CVSS Base Score')
+    data_visualizer.plot_correlation_matrix(train_data)
+
+    # 模型训练和评估
+    models = ['deep_learning', 'random_forest', 'svr', 'ensemble']
+    performance_visualizer = PerformanceVisualizer()
+
+    for model_type in models:
+        logger.info(f"Training {model_type} model")
+
+        # 超参数调优
+        tuner = HyperparameterTuner(model_type, X_train.shape[1])
+        tuner.tune(X_train, y_train)
+        best_params = tuner.get_best_params()
+        logger.info(f"Best parameters for {model_type}: {best_params}")
+
+        # 模型训练
+        trainer = ModelTrainer(model_type, X_train.shape[1])
+        trainer.train(X_train, y_train, X_val, y_val)
+
+        # 模型评估
+        mse_calculator = MSECalculator()
+        rmse_calculator = RMSECalculator()
+
+        y_pred = trainer.model.predict(X_test)
+        mse = mse_calculator.calculate(y_test, y_pred)
+        rmse = rmse_calculator.calculate(y_test, y_pred)
+
+        logger.info(f"{model_type} model - MSE: {mse:.4f}, RMSE: {rmse:.4f}")
+        performance_visualizer.add_result(model_type, mse, rmse)
+
+        # 可视化模型性能
+        performance_visualizer.plot_actual_vs_predicted(y_test, y_pred, model_type)
+        performance_visualizer.plot_error_distribution(y_test, y_pred, model_type)
+        performance_visualizer.plot_residuals(y_test, y_pred, model_type)
+
+        # 交叉验证
+        cv = CrossValidator(model_type, X_train.shape[1])
+        cv_results = cv.cross_validate(X_train, y_train)
+        logger.info(f"Cross-validation results for {model_type}: {cv_results}")
+
+    # 比较所有模型性能
+    performance_visualizer.plot_comparison('MSE')
+    performance_visualizer.plot_comparison('RMSE')
+
+    # 生成性能总结报告
+    summary_report = performance_visualizer.generate_summary_report()
+    logger.info("Performance Summary Report:\n" + summary_report)
+
+    logger.info("CVSS Assessment System completed successfully")
+
+if __name__ == "__main__":
+    main()

model_architecture/__init__.py

@@ -0,0 +1,5 @@
+from .deep_learning_model import DeepLearningModel
+from .machine_learning_model import MachineLearningModel
+from .ensemble_model import EnsembleModel
+
+__all__ = ['DeepLearningModel', 'MachineLearningModel', 'EnsembleModel']

model_architecture/deep_learning_model.py

@@ -0,0 +1,45 @@
+import tensorflow as tf
+from tensorflow.keras.models import Sequential
+from tensorflow.keras.layers import Dense, Dropout, BatchNormalization
+from tensorflow.keras.optimizers import Adam
+
+class DeepLearningModel:
+    def __init__(self, input_dim):
+        self.model = self._build_model(input_dim)
+
+    def _build_model(self, input_dim):
+        model = Sequential([
+            Dense(256, activation='relu', input_shape=(input_dim,)),
+            BatchNormalization(),
+            Dropout(0.3),
+            Dense(128, activation='relu'),
+            BatchNormalization(),
+            Dropout(0.3),
+            Dense(64, activation='relu'),
+            BatchNormalization(),
+            Dropout(0.3),
+            Dense(32, activation='relu'),
+            BatchNormalization(),
+            Dense(1)
+        ])
+        model.compile(optimizer=Adam(learning_rate=0.001), loss='mse')
+        return model
+
+    def fit(self, X, y, validation_data=None, epochs=100, batch_size=32):
+        return self.model.fit(X, y, validation_data=validation_data, epochs=epochs, batch_size=batch_size)
+
+    def predict(self, X):
+        return self.model.predict(X)
+
+    def evaluate(self, X, y):
+        return self.model.evaluate(X, y)
+
+    def save(self, filepath):
+        self.model.save(filepath)
+
+    @classmethod
+    def load(cls, filepath):
+        model = tf.keras.models.load_model(filepath)
+        instance = cls(model.input_shape[1])
+        instance.model = model
+        return instance

model_architecture/ensemble_model.py

@@ -0,0 +1,50 @@
+import numpy as np
+from sklearn.ensemble import VotingRegressor
+from .deep_learning_model import DeepLearningModel
+from .machine_learning_model import MachineLearningModel
+
+class EnsembleModel:
+    def __init__(self, input_dim):
+        self.deep_learning_model = DeepLearningModel(input_dim)
+        self.rf_model = MachineLearningModel('rf')
+        self.svr_model = MachineLearningModel('svr')
+        self.ensemble = None
+
+    def fit(self, X, y, validation_data=None):
+        self.deep_learning_model.fit(X, y, validation_data=validation_data)
+        self.rf_model.fit(X, y)
+        self.svr_model.fit(X, y)
+
+        self.ensemble = VotingRegressor([
+            ('dl', self.deep_learning_model.model),
+            ('rf', self.rf_model.model),
+            ('svr', self.svr_model.model)
+        ])
+        self.ensemble.fit(X, y)
+
+    def predict(self, X):
+        return self.ensemble.predict(X)
+
+    def evaluate(self, X, y):
+        predictions = self.predict(X)
+        mse = np.mean((predictions - y) ** 2)
+        return mse
+
+    def save(self, filepath):
+        models = {
+            'deep_learning': self.deep_learning_model.model,
+            'random_forest': self.rf_model.model,
+            'svr': self.svr_model.model,
+            'ensemble': self.ensemble
+        }
+        joblib.dump(models, filepath)
+
+    @classmethod
+    def load(cls, filepath):
+        models = joblib.load(filepath)
+        instance = cls(models['deep_learning'].input_shape[1])
+        instance.deep_learning_model.model = models['deep_learning']
+        instance.rf_model.model = models['random_forest']
+        instance.svr_model.model = models['svr']
+        instance.ensemble = models['ensemble']
+        return instance

model_architecture/machine_learning_model.py

@@ -0,0 +1,51 @@
+from sklearn.ensemble import RandomForestRegressor
+from sklearn.svm import SVR
+from sklearn.model_selection import GridSearchCV
+import joblib
+
+class MachineLearningModel:
+    def __init__(self, model_type='rf'):
+        self.model_type = model_type
+        self.model = self._create_model()
+
+    def _create_model(self):
+        if self.model_type == 'rf':
+            return RandomForestRegressor(n_estimators=100, random_state=42)
+        elif self.model_type == 'svr':
+            return SVR(kernel='rbf')
+        else:
+            raise ValueError("Unsupported model type")
+
+    def fit(self, X, y):
+        if self.model_type == 'rf':
+            param_grid = {
+                'n_estimators': [100, 200, 300],
+                'max_depth': [None, 10, 20, 30],
+                'min_samples_split': [2, 5, 10],
+                'min_samples_leaf': [1, 2, 4]
+            }
+        elif self.model_type == 'svr':
+            param_grid = {
+                'C': [0.1, 1, 10],
+                'gamma': ['scale', 'auto', 0.1, 1],
+                'kernel': ['rbf', 'poly', 'sigmoid']
+            }
+
+        grid_search = GridSearchCV(self.model, param_grid, cv=5, scoring='neg_mean_squared_error', n_jobs=-1)
+        grid_search.fit(X, y)
+        self.model = grid_search.best_estimator_
+
+    def predict(self, X):
+        return self.model.predict(X)
+
+    def evaluate(self, X, y):
+        return self.model.score(X, y)
+
+    def save(self, filepath):
+        joblib.dump(self.model, filepath)
+
+    @classmethod
+    def load(cls, filepath):
+        instance = cls()
+        instance.model = joblib.load(filepath)
+        return instance

requirements.txt

@@ -0,0 +1,43 @@
+numpy==1.21.5
+pandas==1.3.5
+scikit-learn==0.24.2
+tensorflow==2.7.0
+torch==1.10.1
+transformers==4.15.0
+lightgbm==3.3.2
+xgboost==1.5.1
+catboost==1.0.3
+shap==0.40.0
+lime==0.2.0.1
+matplotlib==3.5.1
+seaborn==0.11.2
+plotly==5.5.0
+dash==2.0.0
+flask==2.0.2
+gunicorn==20.1.0
+pyyaml==6.0
+joblib==1.1.0
+tqdm==4.62.3
+pytest==6.2.5
+black==21.12b0
+isort==5.10.1
+mypy==0.930
+pylint==2.12.2
+scipy==1.7.3
+statsmodels==0.13.1
+networkx==2.6.3
+nltk==3.6.7
+gensim==4.1.2
+spacy==3.2.1
+en-core-web-sm @ https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.2.0/en_core_web_sm-3.2.0.tar.gz
+beautifulsoup4==4.10.0
+requests==2.26.0
+sqlalchemy==1.4.29
+psycopg2-binary==2.9.3
+redis==4.1.0
+celery==5.2.3
+docker==5.0.3
+kubernetes==21.7.0
+boto3==1.20.26
+google-cloud-storage==1.43.0
+azure-storage-blob==12.9.0

training_pipeline/__init__.py

@@ -0,0 +1,5 @@
+from .model_trainer import ModelTrainer
+from .hyperparameter_tuner import HyperparameterTuner
+from .cross_validator import CrossValidator
+
+__all__ = ['ModelTrainer', 'HyperparameterTuner', 'CrossValidator']

training_pipeline/cross_validator.py

@@ -0,0 +1,49 @@
+from sklearn.model_selection import KFold
+from sklearn.metrics import mean_squared_error
+import numpy as np
+from model_architecture import DeepLearningModel, MachineLearningModel, EnsembleModel
+
+class CrossValidator:
+    def __init__(self, model_type, input_dim=None, n_splits=5):
+        self.model_type = model_type
+        self.input_dim = input_dim
+        self.n_splits = n_splits
+        self.kf = KFold(n_splits=n_splits, shuffle=True, random_state=42)
+
+    def cross_validate(self, X, y):
+        mse_scores = []
+        rmse_scores = []
+
+        for train_index, val_index in self.kf.split(X):
+            X_train, X_val = X[train_index], X[val_index]
+            y_train, y_val = y[train_index], y[val_index]
+
+            if self.model_type == 'deep_learning':
+                model = DeepLearningModel(self.input_dim)
+                model.fit(X_train, y_train, validation_data=(X_val, y_val))
+            elif self.model_type in ['random_forest', 'svr']:
+                model = MachineLearningModel(self.model_type)
+                model.fit(X_train, y_train)
+            elif self.model_type == 'ensemble':
+                model = EnsembleModel(self.input_dim)
+                model.fit(X_train, y_train, validation_data=(X_val, y_val))
+            else:
+                raise ValueError("Unsupported model type")
+
+            predictions = model.predict(X_val)
+            mse = mean_squared_error(y_val, predictions)
+            rmse = np.sqrt(mse)
+
+            mse_scores.append(mse)
+            rmse_scores.append(rmse)
+
+        return {
+            'mse': {
+                'mean': np.mean(mse_scores),
+                'std': np.std(mse_scores)
+            },
+            'rmse': {
+                'mean': np.mean(rmse_scores),
+                'std': np.std(rmse_scores)
+            }
+        }

training_pipeline/hyperparameter_tuner.py

@@ -0,0 +1,49 @@
+from sklearn.model_selection import RandomizedSearchCV
+from scipy.stats import uniform, randint
+from model_architecture import DeepLearningModel, MachineLearningModel
+
+class HyperparameterTuner:
+    def __init__(self, model_type, input_dim=None):
+        self.model_type = model_type
+        self.input_dim = input_dim
+        self.best_params = None
+        self.best_model = None
+
+    def tune(self, X, y, n_iter=10, cv=3):
+        if self.model_type == 'deep_learning':
+            param_distributions = {
+                'learning_rate': uniform(1e-4, 1e-2),
+                'batch_size': randint(16, 128),
+                'epochs': randint(50, 200),
+                'dropout_rate': uniform(0.1, 0.5)
+            }
+            model = DeepLearningModel(self.input_dim)
+            search = RandomizedSearchCV(model, param_distributions, n_iter=n_iter, cv=cv, scoring='neg_mean_squared_error')
+        elif self.model_type in ['random_forest', 'svr']:
+            if self.model_type == 'random_forest':
+                param_distributions = {
+                    'n_estimators': randint(100, 500),
+                    'max_depth': randint(10, 100),
+                    'min_samples_split': randint(2, 20),
+                    'min_samples_leaf': randint(1, 10)
+                }
+            else:  # SVR
+                param_distributions = {
+                    'C': uniform(0.1, 10),
+                    'gamma': uniform(0.01, 1),
+                    'kernel': ['rbf', 'poly', 'sigmoid']
+                }
+            model = MachineLearningModel(self.model_type)
+            search = RandomizedSearchCV(model.model, param_distributions, n_iter=n_iter, cv=cv, scoring='neg_mean_squared_error')
+        else:
+            raise ValueError("Unsupported model type for hyperparameter tuning")
+
+        search.fit(X, y)
+        self.best_params = search.best_params_
+        self.best_model = search.best_estimator_
+
+    def get_best_params(self):
+        return self.best_params
+
+    def get_best_model(self):
+        return self.best_model

training_pipeline/model_trainer.py

@@ -0,0 +1,49 @@
+import numpy as np
+from sklearn.metrics import mean_squared_error
+from model_architecture import DeepLearningModel, MachineLearningModel, EnsembleModel
+
+class ModelTrainer:
+    def __init__(self, model_type, input_dim=None):
+        self.model_type = model_type
+        self.input_dim = input_dim
+        self.model = self._create_model()
+
+    def _create_model(self):
+        if self.model_type == 'deep_learning':
+            return DeepLearningModel(self.input_dim)
+        elif self.model_type == 'random_forest':
+            return MachineLearningModel('rf')
+        elif self.model_type == 'svr':
+            return MachineLearningModel('svr')
+        elif self.model_type == 'ensemble':
+            return EnsembleModel(self.input_dim)
+        else:
+            raise ValueError("Unsupported model type")
+
+    def train(self, X_train, y_train, X_val=None, y_val=None):
+        if self.model_type == 'deep_learning':
+            self.model.fit(X_train, y_train, validation_data=(X_val, y_val) if X_val is not None else None)
+        elif self.model_type in ['random_forest', 'svr']:
+            self.model.fit(X_train, y_train)
+        elif self.model_type == 'ensemble':
+            self.model.fit(X_train, y_train, validation_data=(X_val, y_val) if X_val is not None else None)
+
+    def evaluate(self, X_test, y_test):
+        predictions = self.model.predict(X_test)
+        mse = mean_squared_error(y_test, predictions)
+        rmse = np.sqrt(mse)
+        return {'mse': mse, 'rmse': rmse}
+
+    def save_model(self, filepath):
+        self.model.save(filepath)
+
+    @classmethod
+    def load_model(cls, model_type, filepath):
+        trainer = cls(model_type)
+        if model_type == 'deep_learning':
+            trainer.model = DeepLearningModel.load(filepath)
+        elif model_type in ['random_forest', 'svr']:
+            trainer.model = MachineLearningModel.load(filepath)
+        elif model_type == 'ensemble':
+            trainer.model = EnsembleModel.load(filepath)
+        return trainer

utils/__init__.py

@@ -0,0 +1,5 @@
+from .config_manager import ConfigManager
+from .logger import Logger
+from .data_visualizer import DataVisualizer
+
+__all__ = ['ConfigManager', 'Logger', 'DataVisualizer']

utils/config_manager.py

@@ -0,0 +1,28 @@
+import yaml
+import os
+
+class ConfigManager:
+    def __init__(self, config_path):
+        self.config_path = config_path
+        self.config = self.load_config()
+
+    def load_config(self):
+        if not os.path.exists(self.config_path):
+            raise FileNotFoundError(f"Config file not found: {self.config_path}")
+        
+        with open(self.config_path, 'r') as config_file:
+            return yaml.safe_load(config_file)
+
+    def get(self, key, default=None):
+        return self.config.get(key, default)
+
+    def set(self, key, value):
+        self.config[key] = value
+
+    def save(self):
+        with open(self.config_path, 'w') as config_file:
+            yaml.dump(self.config, config_file)
+
+    def update(self, new_config):
+        self.config.update(new_config)
+        self.save()

utils/data_visualizer.py

@@ -0,0 +1,61 @@
+import matplotlib.pyplot as plt
+import seaborn as sns
+import pandas as pd
+import numpy as np
+
+class DataVisualizer:
+    def __init__(self):
+        pass
+
+    def plot_feature_importance(self, feature_importance, feature_names, top_n=20):
+        fi_df = pd.DataFrame({'feature': feature_names, 'importance': feature_importance})
+        fi_df = fi_df.sort_values('importance', ascending=False).head(top_n)
+
+        plt.figure(figsize=(12, 8))
+        sns.barplot(x='importance', y='feature', data=fi_df)
+        plt.title(f'Top {top_n} Feature Importance')
+        plt.tight_layout()
+        plt.show()
+
+    def plot_correlation_matrix(self, df):
+        plt.figure(figsize=(12, 10))
+        sns.heatmap(df.corr(), annot=True, cmap='coolwarm', linewidths=0.5)
+        plt.title('Correlation Matrix')
+        plt.tight_layout()
+        plt.show()
+
+    def plot_distribution(self, data, column, title):
+        plt.figure(figsize=(10, 6))
+        sns.histplot(data[column], kde=True)
+        plt.title(f'Distribution of {title}')
+        plt.xlabel(column)
+        plt.ylabel('Frequency')
+        plt.tight_layout()
+        plt.show()
+
+    def plot_scatter(self, data, x, y, title):
+        plt.figure(figsize=(10, 6))
+        sns.scatterplot(data=data, x=x, y=y)
+        plt.title(title)
+        plt.tight_layout()
+        plt.show()
+
+    def plot_boxplot(self, data, x, y, title):
+        plt.figure(figsize=(12, 6))
+        sns.boxplot(data=data, x=x, y=y)
+        plt.title(title)
+        plt.xticks(rotation=45)
+        plt.tight_layout()
+        plt.show()
+
+    def plot_learning_curve(self, train_scores, val_scores, train_sizes):
+        plt.figure(figsize=(10, 6))
+        plt.plot(train_sizes, np.mean(train_scores, axis=1), label='Training score')
+        plt.plot(train_sizes, np.mean(val_scores, axis=1), label='Validation score')
+        plt.title('Learning Curve')
+        plt.xlabel('Training examples')
+        plt.ylabel('Score')
+        plt.legend(loc='best')
+        plt.grid(True)
+        plt.tight_layout()
+        plt.show()

utils/logger.py

@@ -0,0 +1,45 @@
+import logging
+import os
+from datetime import datetime
+
+class Logger:
+    def __init__(self, log_dir, log_level=logging.INFO):
+        self.log_dir = log_dir
+        self.log_level = log_level
+        self.logger = self._setup_logger()
+
+    def _setup_logger(self):
+        if not os.path.exists(self.log_dir):
+            os.makedirs(self.log_dir)
+
+        logger = logging.getLogger(__name__)
+        logger.setLevel(self.log_level)
+
+        formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+
+        file_handler = logging.FileHandler(
+            os.path.join(self.log_dir, f'log_{datetime.now().strftime("%Y%m%d_%H%M%S")}.txt')
+        )
+        file_handler.setFormatter(formatter)
+        logger.addHandler(file_handler)
+
+        console_handler = logging.StreamHandler()
+        console_handler.setFormatter(formatter)
+        logger.addHandler(console_handler)
+
+        return logger
+
+    def info(self, message):
+        self.logger.info(message)
+
+    def warning(self, message):
+        self.logger.warning(message)
+
+    def error(self, message):
+        self.logger.error(message)
+
+    def debug(self, message):
+        self.logger.debug(message)
+
+    def critical(self, message):
+        self.logger.critical(message)