98 lines
3.5 KiB
Python
98 lines
3.5 KiB
Python
import pandas as pd
|
||
import numpy as np
|
||
from sklearn import preprocessing
|
||
from sklearn.model_selection import GridSearchCV
|
||
import matplotlib.pyplot as plt
|
||
from sklearn.model_selection import train_test_split
|
||
from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score,matthews_corrcoef
|
||
from sklearn.preprocessing import OrdinalEncoder
|
||
import seaborn as sns
|
||
from scipy.stats import normaltest
|
||
dir=".//OutputFiles//"
|
||
CorrFile="spearman_correlation_target.csv"
|
||
NormalResultFile="normality_test_results.csv"
|
||
low_corr_features=[]
|
||
#数据预处理
|
||
def data_process():
|
||
data1=pd.read_csv(".//datasets//new_train_feature.csv")
|
||
data2=pd.read_csv(".//datasets//new_valid_feature.csv")
|
||
data=pd.concat([data1,data2])
|
||
df=drop_selected_columns(data)
|
||
normal_check(df)
|
||
GetSpearmanCorr(df)
|
||
data1=drop_selected_columns(data1)
|
||
data2=drop_selected_columns(data2)
|
||
data1 = data1.drop(columns=low_corr_features)
|
||
data2 = data2.drop(columns=low_corr_features)
|
||
return data1,data2
|
||
|
||
# 缺失值处理，去掉dataframe的部分特征
|
||
def drop_selected_columns(data):
|
||
data = data.drop(['function_id', 'cwe_id'], axis=1)
|
||
non_na_counts = data.notna().sum()
|
||
# 计算总行数的三分之一
|
||
threshold = len(data) * (2 / 3)
|
||
# 找出需要删除的列
|
||
cols_to_drop = non_na_counts[non_na_counts < threshold].index
|
||
# 删除这些列
|
||
data = data.drop(columns=cols_to_drop)
|
||
data = data.drop(columns='project')
|
||
return data
|
||
|
||
def get_test_df():
|
||
data3 = pd.read_csv(".//datasets//new_test_feature.csv")
|
||
function_id=data3['function_id'].values
|
||
data3=drop_selected_columns(data3)
|
||
return function_id,data3
|
||
|
||
#将正态分布检验结果输出到csv文件中
|
||
def normal_to_csv(data):
|
||
results = []
|
||
for column in data.columns:
|
||
stat, pvalue = normaltest(data[column].dropna()) # 使用 normaltest 检验正态性
|
||
results.append({'Feature': column, 'Statistic': stat, 'P-Value': pvalue})
|
||
return results
|
||
|
||
#正态分布检验
|
||
def normal_check(data):
|
||
normality_results = pd.DataFrame(normal_to_csv(data))
|
||
normality_results.to_csv(dir+NormalResultFile, index=False)
|
||
|
||
|
||
#绘制Spearman相关性的热力图
|
||
def plotSpeatman(spearman_corr):
|
||
plt.figure(figsize=(20, 16))
|
||
sns.heatmap(spearman_corr, annot=True, fmt=".2f", cmap='coolwarm', cbar_kws={'label': 'Spearman Correlation'})
|
||
plt.title('Spearman correlation heatmap')
|
||
plt.xlabel('Features')
|
||
plt.ylabel('Features')
|
||
|
||
#spearman相关性分析
|
||
def GetSpearmanCorr(df):
|
||
spearman_corr = df.corr(method='spearman')
|
||
#绘制热力图
|
||
#plotSpeatman(spearman_corr)
|
||
corr_matrix = spearman_corr
|
||
print("所有特征和target标签的相关性：")
|
||
print(corr_matrix['target'])
|
||
print("与target标签相关性较低的特征：")
|
||
low_corr_features = corr_matrix['target'][corr_matrix['target'].abs() < 0.030].index.tolist()
|
||
print(low_corr_features)
|
||
#输出至CSV文件中
|
||
Corr_to_CSV(corr_matrix)
|
||
|
||
#将相关性矩阵输出到csv文件中
|
||
def Corr_to_CSV(corr_matrix):
|
||
# 指定CSV文件的路径
|
||
csv_file_path = dir+CorrFile
|
||
# 将Spearman相关系数矩阵保存为CSV文件
|
||
corr_matrix.to_csv(csv_file_path)
|
||
|
||
def Result_to_File(function_id,target,filename):
|
||
df = pd.DataFrame({
|
||
'function_id': function_id,
|
||
'target': target
|
||
})
|
||
df_unique = df.drop_duplicates(subset='function_id', keep='first')
|
||
# 将处理后的DataFrame保存回CSV文件
|
||
df_unique.to_csv(filename, index=False) |