121 lines
3.7 KiB
Python
121 lines
3.7 KiB
Python
# 分析post中text block和code block数据的相关性
|
||
# 一元线性回归
|
||
import pandas as pd
|
||
import numpy as np
|
||
import matplotlib.pyplot as plt
|
||
from sklearn import datasets, linear_model
|
||
from sklearn.metrics import mean_squared_error, r2_score
|
||
from sklearn.model_selection import train_test_split
|
||
|
||
plt.rcParams['font.sans-serif'] = ['SimHei']
|
||
plt.rcParams['axes.unicode_minus'] = False
|
||
|
||
|
||
versioncnt = pd.read_csv('E:/学习相关资料/SOTorrent数据分析/posts_versioncount.csv/posts_versioncount.csv') # 读取version数据
|
||
commentblkcnt = pd.read_csv('E:/学习相关资料/SOTorrent数据分析/posts_metadata.csv/posts_metadata.csv') # 读取comment数据
|
||
|
||
aa = versioncnt.iloc[10000:20000, 2:3] # 39905357
|
||
print(aa.shape)
|
||
bb = commentblkcnt.iloc[10000:20000, 3:4]
|
||
print(bb.shape)
|
||
aa = np.array(aa)
|
||
bb = np.array(bb)
|
||
# cc = np.array([aa, bb])
|
||
|
||
aa_mean = np.mean(aa, axis=0) # axis=0,表示按列求均值
|
||
aa_std = np.std(aa, axis=0)
|
||
aa_zscore = (aa - aa_mean) / aa_std # 标准化
|
||
|
||
bb_mean = np.mean(bb, axis=0) # axis=0,表示按列求均值
|
||
bb_std = np.std(bb, axis=0)
|
||
bb_zscore = (bb - bb_mean) / bb_std # 标准化
|
||
|
||
cc_zscore_corr = np.corrcoef(aa_zscore.reshape(1, 10000), bb_zscore.reshape(1, 10000)) # 相关系数矩阵
|
||
print(cc_zscore_corr[0, 1])
|
||
|
||
# d = pd.read_csv('Salary_Data. csv')
|
||
# d.head()
|
||
# # #看一下数据的分布情況
|
||
# # %matplotlib inline
|
||
# # d.plot.scatter(x='x1', y='y')
|
||
# #处理数据，将其变成特征矩阵X和结果矩阵Y
|
||
# df=pd.Dataframe(data=d, columns=['yearsexperlence', 'Salary'])
|
||
# df.insert(0, 'x0', [1]*df.shape[0])
|
||
# df =df. rename(columns={'Yearsexperience': 'x1', 'Salary': 'y'})
|
||
# x=df[['x0','x1']].values
|
||
# y=df[['y']].values
|
||
# # 画线性方程和数据的散点图
|
||
# def plotline(x, y, theta):
|
||
# px = np.linspace(x[0][1],x[-1][1],10000)
|
||
# py = theta[0]+theta[1]*px #方程式
|
||
# plt.figure(num=1)
|
||
# #获取图片并命名
|
||
# plt.plot(px, py, color='blue', linewidth=1.0, label='blue')
|
||
# plt.plot(x[: ,1],y, 'bo')
|
||
# plt.show()
|
||
# 梯度下降
|
||
def Optimization(x, y, theta, learning_rate):
|
||
for i in range(iter):
|
||
theta = Updata(x, y, theta, learning_rate)
|
||
return theta
|
||
|
||
|
||
def Updata(x, y, theta, learning_rate):
|
||
m = len(x)
|
||
sum = 0.0
|
||
sum1 = 0.0
|
||
alpha = learning_rate
|
||
h = 0
|
||
for i in range(m):
|
||
h = theta[0] + theta[1] * x[i]
|
||
sum += (h - y[i])
|
||
sum1 += (h - y[i]) * x[i]
|
||
theta[0] -= alpha * sum / m
|
||
theta[1] -= alpha * sum1 / m
|
||
return theta
|
||
|
||
|
||
# 数据初始化
|
||
learning_rate = 0.001
|
||
theta = [0, 0]
|
||
iter = 1000
|
||
aa_zscore = np.array(aa_zscore)
|
||
bb_zscore = np.array(bb_zscore)
|
||
|
||
# %pylab inline
|
||
plt.scatter(aa_zscore, bb_zscore, color='black')
|
||
plt.xlabel('version')
|
||
plt.ylabel('comment')
|
||
plt.show()
|
||
plt.savefig("Version与Comment关系散点图.jpg")
|
||
|
||
regr = linear_model.LinearRegression()
|
||
regr.fit(aa_zscore, bb_zscore)
|
||
regr.fit(aa_zscore, bb_zscore)
|
||
|
||
print('Intercept:{}'.format(regr.intercept_))
|
||
print('Coefficient:{}'.format(regr.coef_))
|
||
|
||
plt.scatter(aa_zscore, bb_zscore, color='black')
|
||
plt.plot(aa_zscore, regr.predict(aa_zscore), linewidth=3, color='blue')
|
||
plt.xlabel('version')
|
||
plt.ylabel('comment')
|
||
plt.title('VersionCount与CommentCount关系')
|
||
plt.show()
|
||
|
||
plt.savefig("Version与Comment线性关系图.jpg")
|
||
|
||
theta = Optimization(aa_zscore, bb_zscore, theta, learning_rate)
|
||
|
||
# 可视化
|
||
plt.figure(figsize=(35, 35))
|
||
plt.scatter(aa_zscore, bb_zscore, marker='o')
|
||
plt.plot(aa_zscore, bb_zscore)
|
||
plt.xticks(fontsize=40)
|
||
plt.yticks(fontsize=40)
|
||
plt.xlabel('Comment Count', fontsize=40)
|
||
plt.ylabel('Version Count', fontsize=40)
|
||
plt.title('Version与Comment线性关系', fontsize=40)
|
||
plt.savefig("Version与Comment线性关系.jpg")
|
||
plt.show()
|