ADD file via upload

ML_vul/dataProcess.py

@@ -0,0 +1,98 @@
+import pandas as pd
+import numpy as np
+from sklearn import preprocessing
+from sklearn.model_selection import GridSearchCV
+import matplotlib.pyplot as plt
+from sklearn.model_selection import train_test_split
+from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score,matthews_corrcoef
+from sklearn.preprocessing import OrdinalEncoder
+import seaborn as sns
+from scipy.stats import normaltest
+dir=".//OutputFiles//"
+CorrFile="spearman_correlation_target.csv"
+NormalResultFile="normality_test_results.csv"
+low_corr_features=[]
+#数据预处理
+def data_process():
+    data1=pd.read_csv(".//datasets//new_train_feature.csv")
+    data2=pd.read_csv(".//datasets//new_valid_feature.csv")
+    data=pd.concat([data1,data2])
+    df=drop_selected_columns(data)
+    normal_check(df)
+    GetSpearmanCorr(df)
+    data1=drop_selected_columns(data1)
+    data2=drop_selected_columns(data2)
+    data1 = data1.drop(columns=low_corr_features)
+    data2 = data2.drop(columns=low_corr_features)
+    return data1,data2
+
+#   缺失值处理，去掉dataframe的部分特征
+def drop_selected_columns(data):
+    data = data.drop(['function_id', 'cwe_id'], axis=1)
+    non_na_counts = data.notna().sum()
+    # 计算总行数的三分之一
+    threshold = len(data) * (2 / 3)
+    # 找出需要删除的列
+    cols_to_drop = non_na_counts[non_na_counts < threshold].index
+    # 删除这些列
+    data = data.drop(columns=cols_to_drop)
+    data = data.drop(columns='project')
+    return data
+
+def get_test_df():
+    data3 = pd.read_csv(".//datasets//new_test_feature.csv")
+    function_id=data3['function_id'].values
+    data3=drop_selected_columns(data3)
+    return function_id,data3
+
+#将正态分布检验结果输出到csv文件中
+def normal_to_csv(data):
+    results = []
+    for column in data.columns:
+        stat, pvalue = normaltest(data[column].dropna())  # 使用 normaltest 检验正态性
+        results.append({'Feature': column, 'Statistic': stat, 'P-Value': pvalue})
+    return results
+
+#正态分布检验
+def normal_check(data):
+    normality_results = pd.DataFrame(normal_to_csv(data))
+    normality_results.to_csv(dir+NormalResultFile, index=False)
+
+
+#绘制Spearman相关性的热力图
+def plotSpeatman(spearman_corr):
+    plt.figure(figsize=(20, 16))
+    sns.heatmap(spearman_corr, annot=True, fmt=".2f", cmap='coolwarm', cbar_kws={'label': 'Spearman Correlation'})
+    plt.title('Spearman correlation heatmap')
+    plt.xlabel('Features')
+    plt.ylabel('Features')
+
+#spearman相关性分析
+def GetSpearmanCorr(df):
+    spearman_corr = df.corr(method='spearman')
+    #绘制热力图
+    #plotSpeatman(spearman_corr)
+    corr_matrix = spearman_corr
+    print("所有特征和target标签的相关性：")
+    print(corr_matrix['target'])
+    print("与target标签相关性较低的特征：")
+    low_corr_features = corr_matrix['target'][corr_matrix['target'].abs() < 0.030].index.tolist()
+    print(low_corr_features)
+    #输出至CSV文件中
+    Corr_to_CSV(corr_matrix)
+
+#将相关性矩阵输出到csv文件中
+def Corr_to_CSV(corr_matrix):
+    # 指定CSV文件的路径
+    csv_file_path = dir+CorrFile
+    # 将Spearman相关系数矩阵保存为CSV文件
+    corr_matrix.to_csv(csv_file_path)
+
+def Result_to_File(function_id,target,filename):
+    df = pd.DataFrame({
+        'function_id': function_id,
+        'target': target
+    })
+    df_unique = df.drop_duplicates(subset='function_id', keep='first')
+    # 将处理后的DataFrame保存回CSV文件
+    df_unique.to_csv(filename, index=False)