删除文件
This commit is contained in:
crystal20277
parent
e44fce6e3e
commit
080579d069
|
|
@ -2,7 +2,7 @@
|
|||
<module type="PYTHON_MODULE" version="4">
|
||||
<component name="NewModuleRootManager">
|
||||
<content url="file://$MODULE_DIR$" />
|
||||
<orderEntry type="jdk" jdkName="Python 3.8 (aiforge)" jdkType="Python SDK" />
|
||||
<orderEntry type="inheritedJdk" />
|
||||
<orderEntry type="sourceFolder" forTests="false" />
|
||||
</component>
|
||||
<component name="PyDocumentationSettings">
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="ProjectRootManager" version="2" project-jdk-name="Python 3.8 (aiforge)" project-jdk-type="Python SDK" />
|
||||
<component name="ProjectRootManager" version="2" languageLevel="JDK_17" project-jdk-name="Python 3.8 (aiforge)" project-jdk-type="Python SDK" />
|
||||
</project>
|
||||
|
|
@ -1,36 +0,0 @@
|
|||
"""
|
||||
企业行业地位、企业业务方向
|
||||
"""
|
||||
import warnings
|
||||
import pandas as pd
|
||||
from pandas.io import sql
|
||||
from datetime import datetime
|
||||
from sqlalchemy import create_engine
|
||||
from config import host, port, user, passwd, database
|
||||
warnings.filterwarnings("ignore")
|
||||
|
||||
|
||||
def generate_org_industry_status_data():
|
||||
engine = create_engine(f'postgresql+psycopg2://{user}:{passwd}@{host}:{port}/{database}', pool_recycle=3600)
|
||||
conn = engine.connect()
|
||||
|
||||
truncate_sql = f"truncate ai_repo_topic_detail"
|
||||
sql.execute(truncate_sql, conn)
|
||||
|
||||
sql_str = f"""
|
||||
select
|
||||
t1.id repo_id, t1.num_watches, t1.num_stars, t1.num_forks, t1.clone_cnt,
|
||||
t1.num_watches + t1.num_stars + t1.num_forks + t1.clone_cnt num_hots,
|
||||
t3.id topic_id, t3.name topic_name
|
||||
from repository t1
|
||||
left join repo_topic t2 on t1.id = t2.repo_id
|
||||
left join topic t3 on t2.topic_id = t3."id"
|
||||
where t3.name is not null
|
||||
"""
|
||||
all_df = pd.read_sql(sql_str, conn)
|
||||
all_df.insert(8, 'created_unix', int(datetime.now().timestamp()))
|
||||
all_df.to_sql('ai_repo_topic_detail', conn, index=False, if_exists='append')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
generate_org_industry_status_data()
|
||||
|
|
@ -1,47 +0,0 @@
|
|||
"""
|
||||
企业开源实力排名
|
||||
"""
|
||||
import warnings
|
||||
import pandas as pd
|
||||
from pandas.io import sql
|
||||
from datetime import datetime
|
||||
from sqlalchemy import create_engine
|
||||
from config import host, port, user, passwd, database
|
||||
warnings.filterwarnings("ignore")
|
||||
|
||||
|
||||
def generate_org_open_source_rank_data():
|
||||
engine = create_engine(f'postgresql+psycopg2://{user}:{passwd}@{host}:{port}/{database}', pool_recycle=3600)
|
||||
conn = engine.connect()
|
||||
|
||||
truncate_sql = f"truncate ai_org_open_source_rank_statistics"
|
||||
sql.execute(truncate_sql, conn)
|
||||
|
||||
sql_str = f"""
|
||||
select
|
||||
org_id, num_repos, num_forks, num_issues, num_pulls,
|
||||
num_commit, num_all, rank() over(order by num_all desc) num_rank
|
||||
from(
|
||||
select
|
||||
org_id, count(repo_id) num_repos, sum(num_forks) num_forks,
|
||||
sum(num_issues) num_issues, sum(num_pulls) num_pulls, sum(num_commit) num_commit,
|
||||
count(repo_id) + sum(num_forks) + sum(num_issues) + sum(num_pulls) + sum(num_commit) num_all
|
||||
from(
|
||||
select
|
||||
t2.org_id, t2.repo_id, t3.num_forks, t3.num_issues, t3.num_pulls, t3.num_commit
|
||||
from
|
||||
opendata.public.user t1
|
||||
left join team_repo t2 on t1.id = t2.org_id
|
||||
left join repository t3 on t2.repo_id = t3.id
|
||||
where t1.type = 1 and t2.org_id is not null
|
||||
) tt1 GROUP BY tt1.org_id
|
||||
) ttt1
|
||||
"""
|
||||
all_df = pd.read_sql(sql_str, conn)
|
||||
all_df.insert(8, 'num_all_org', all_df.shape[0])
|
||||
all_df.insert(9, 'created_unix', int(datetime.now().timestamp()))
|
||||
all_df.to_sql('ai_org_open_source_rank_statistics', conn, index=False, if_exists='append')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
generate_org_open_source_rank_data()
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
## 这里面存放数据
|
||||
|
||||
|
|
@ -1,113 +0,0 @@
|
|||
import datetime
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from pandas.io import sql
|
||||
from config import logger
|
||||
from utils import get_before_date
|
||||
from config import data_before_days
|
||||
from sqlalchemy import create_engine
|
||||
from config import activity_analysis_path
|
||||
from config import host, port, user, passwd, database
|
||||
|
||||
|
||||
def get_data_from_pgsql(latest_data_date):
|
||||
"""
|
||||
从pgsql数据库获取原始数据
|
||||
"""
|
||||
start = datetime.datetime.now()
|
||||
logger.info("开始获取用户活跃度数据...")
|
||||
|
||||
engine = create_engine(f'postgresql+psycopg2://{user}:{passwd}@{host}:{port}/{database}', pool_recycle=3600)
|
||||
conn = engine.connect()
|
||||
|
||||
sql_text = f"""
|
||||
SELECT
|
||||
u_id user_id,
|
||||
'login' action_type,
|
||||
to_char(to_timestamp(created_unix), 'YYYY-MM-DD') created_at
|
||||
FROM
|
||||
user_login_log
|
||||
WHERE to_char(to_timestamp(created_unix), 'YYYY-MM-DD') >= '{latest_data_date}'
|
||||
UNION
|
||||
SELECT
|
||||
t1.user_id,
|
||||
CASE WHEN t1.status = 2 THEN 'clickNotice' ELSE' closeNotice' END action_type,
|
||||
to_char(to_timestamp(created_unix), 'YYYY-MM-DD') created_at
|
||||
FROM
|
||||
notification t1
|
||||
WHERE
|
||||
t1.status IN ( 2, 4 )
|
||||
AND to_char(to_timestamp(created_unix), 'YYYY-MM-DD') >= '{latest_data_date}'
|
||||
"""
|
||||
data_df = pd.read_sql(sql_text, conn)
|
||||
data_df.to_csv(activity_analysis_path + 'data/user_actions.csv', index=False, header=True, sep='\t')
|
||||
logger.info("用户活跃度数据下载完毕，共" + str(data_df.shape[0]) + "条数据，总耗时" + str((datetime.datetime.now() - start).seconds) + "秒")
|
||||
|
||||
|
||||
def read_action_data():
|
||||
"""
|
||||
处理用户活跃度数据
|
||||
"""
|
||||
data_user_action = pd.read_csv(activity_analysis_path + 'data/user_actions.csv', sep='\t')
|
||||
|
||||
data_user_action = data_user_action[['user_id', 'action_type', 'created_at']]
|
||||
# 提取出三种最常用的操作
|
||||
some_action = ['login', 'closeNotice', 'clickNotice']
|
||||
data_user_action = data_user_action.loc[data_user_action['action_type'].isin(some_action)]
|
||||
# 将日期格式进行转换
|
||||
data_user_action['created_at'] = pd.to_datetime(data_user_action['created_at'], dayfirst=True)
|
||||
|
||||
data_user_action['created_at'] = data_user_action['created_at'].dt.date
|
||||
return data_user_action
|
||||
|
||||
|
||||
def get_rfm_data():
|
||||
"""
|
||||
生成RFM聚类模型训练数据
|
||||
"""
|
||||
data = read_action_data()
|
||||
|
||||
df_rfm = data[['user_id', 'action_type', 'created_at']]
|
||||
class_dict = {
|
||||
'login': 1,
|
||||
'clickNotice': 2,
|
||||
'closeNotice': 3,
|
||||
}
|
||||
# 给不同操作赋值，来区分不同操作的重要度
|
||||
df_rfm['action_values'] = df_rfm['action_type'].map(lambda x: x)
|
||||
df_rfm['action_values'] = df_rfm['action_values'].map(class_dict)
|
||||
# 计算 R,F,M 值
|
||||
df_rfm = df_rfm.groupby("user_id").agg({'created_at': 'max', 'user_id': 'count', 'action_values': 'sum'})
|
||||
df_rfm = df_rfm.rename(columns={'created_at': 'Recentdate', 'user_id': 'F', 'action_values': 'M'})
|
||||
|
||||
df_rfm["R"] = (df_rfm['Recentdate'].max() - df_rfm['Recentdate']) / np.timedelta64(1, 'D')
|
||||
df_rfm["R"] = df_rfm["R"].astype('str').str.split(" ", expand=True)
|
||||
df_rfm["R"] = df_rfm["R"].astype("float").astype("int")
|
||||
df_rfm.drop(columns='Recentdate', inplace=True)
|
||||
|
||||
return df_rfm
|
||||
|
||||
|
||||
def save_train_result_to_pgsql(df):
|
||||
start = datetime.datetime.now()
|
||||
logger.info("开始将用户活跃度数据保存到数据库...")
|
||||
|
||||
engine = create_engine(f'postgresql+psycopg2://{user}:{passwd}@{host}:{port}/{database}', pool_recycle=3600)
|
||||
conn = engine.connect()
|
||||
|
||||
type_str = 'activity'
|
||||
|
||||
truncate_sql = f"DELETE FROM ai_user_portraint_detail WHERE type = '{type_str}'"
|
||||
sql.execute(truncate_sql, conn)
|
||||
|
||||
df = df.rename(columns={'activity': 'value'})
|
||||
df['type'] = type_str
|
||||
df['created_unix'] = int(datetime.datetime.now().timestamp())
|
||||
df.to_sql('ai_user_portraint_detail', conn, index=False, if_exists='append')
|
||||
logger.info("用户活跃度数据保存到数据库完毕，共" + str(df.shape[0]) + "条数据，总耗时" + str((datetime.datetime.now() - start).seconds) + "秒")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
# 取之前多少天登录的数据
|
||||
latest_data_date = get_before_date(data_before_days)
|
||||
get_data_from_pgsql(latest_data_date)
|
||||
Binary file not shown.
Binary file not shown.
|
|
@ -1,24 +0,0 @@
|
|||
import pickle
|
||||
from config import logger
|
||||
from config import test_user_id
|
||||
from config import activity_analysis_path
|
||||
|
||||
logger.info('加载用户活跃度字典')
|
||||
user_activity_dict = pickle.load(open(activity_analysis_path + 'results/user_activity_dict.pkl', 'rb'))
|
||||
|
||||
|
||||
def user_activity_predict(user_id):
|
||||
"""
|
||||
用户活跃度预测
|
||||
"""
|
||||
if user_id not in user_activity_dict:
|
||||
result = -1
|
||||
else:
|
||||
result = user_activity_dict[user_id]
|
||||
|
||||
return result
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
result = user_activity_predict(user_id=test_user_id)
|
||||
print('用户ID：', test_user_id, '贡献度：', result)
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
## 这里存放模型的训练结果
|
||||
|
||||
|
|
@ -1,85 +0,0 @@
|
|||
import pickle
|
||||
import warnings
|
||||
import pandas as pd
|
||||
from sklearn.cluster import KMeans
|
||||
from config import logger
|
||||
from config import RANDOM_SEED
|
||||
from data_process import get_rfm_data, save_train_result_to_pgsql
|
||||
from config import activity_analysis_path
|
||||
warnings.filterwarnings('ignore')
|
||||
|
||||
|
||||
def iflabel(x):
|
||||
if x == "高高高":
|
||||
return 5
|
||||
elif x == "高低高":
|
||||
return 4
|
||||
elif x == "高高低":
|
||||
return 4
|
||||
elif x == "高低低":
|
||||
return 3
|
||||
elif x == "低高高":
|
||||
return 4
|
||||
elif x == "低低高":
|
||||
return 2
|
||||
elif x == "低高低":
|
||||
return 2
|
||||
elif x == "低低低":
|
||||
return 1
|
||||
|
||||
|
||||
def train():
|
||||
"""
|
||||
用户活跃度分析模型训练
|
||||
"""
|
||||
logger.info('开始训练用户活跃度分析模型...')
|
||||
|
||||
df_rfm = get_rfm_data()
|
||||
rfm_cluster = df_rfm.copy()
|
||||
# 根据 R,F,M 值将所有数据分为 8 类
|
||||
Cluster = df_rfm[["R", "F", "M"]].values
|
||||
kmeans_model = KMeans(n_clusters=8, random_state=RANDOM_SEED).fit(Cluster)
|
||||
|
||||
rfm_cluster["label"] = kmeans_model.labels_
|
||||
|
||||
# 此处计算均值 RFM 模型，每个用户的R、F、M值同总体均值进行比较，比均值高的就为“高”，低的就标明“低”，然后通过三个指标的高低来打标签。
|
||||
df_rfm_mean = rfm_cluster[["label", "R", "F", "M"]].groupby("label").agg("mean")
|
||||
df_rfm_mean["IFR"] = df_rfm_mean["R"].apply(lambda x: "高" if x > df_rfm_mean["R"].mean() else "低")
|
||||
df_rfm_mean["IFF"] = df_rfm_mean["F"].apply(lambda x: "高" if x > df_rfm_mean["F"].mean() else "低")
|
||||
df_rfm_mean["IFM"] = df_rfm_mean["M"].apply(lambda x: "高" if x > df_rfm_mean["M"].mean() else "低")
|
||||
|
||||
# 方法和上面一样，#将三个指标合并起来，生成临时列temp，表示RFM综合指标
|
||||
df_rfm_mean["temp"] = df_rfm_mean["IFR"] + df_rfm_mean["IFF"] + df_rfm_mean["IFM"]
|
||||
# 对指标进行综合判断
|
||||
df_rfm_mean["label"] = df_rfm_mean["temp"].apply(lambda x: iflabel(x))
|
||||
# 此处在使用聚类RFM模型，比较的时候使用总体均值进行比较
|
||||
rfm_cluster["IFR"] = rfm_cluster["R"].apply(lambda x: "高" if x > df_rfm_mean["R"].mean() else "低")
|
||||
rfm_cluster["IFF"] = rfm_cluster["F"].apply(lambda x: "高" if x > df_rfm_mean["F"].mean() else "低")
|
||||
rfm_cluster["IFM"] = rfm_cluster["M"].apply(lambda x: "高" if x > df_rfm_mean["M"].mean() else "低")
|
||||
# 方法和上面一样，将三个指标合并起来，生成临时列temp，表示RFM综合指标
|
||||
rfm_cluster["temp"] = rfm_cluster["IFR"] + rfm_cluster["IFF"] + rfm_cluster["IFM"]
|
||||
# 根据定义的打分规则给每个用户进行打分，1-5分
|
||||
rfm_cluster["label"] = rfm_cluster["temp"].apply(lambda x: iflabel(x))
|
||||
|
||||
result = rfm_cluster[['label']]
|
||||
|
||||
result_csv = pd.DataFrame(list(result.index), index=range(len(list(result.index))), columns=['user_id'])
|
||||
result_csv['activity'] = list(result.values.reshape(1, len(list(result.index)))[0])
|
||||
# 将用户 id 和活跃度分值提取出来并创新建立一个表
|
||||
result_csv['user_id'] = result_csv['user_id'].astype(int)
|
||||
result_csv['activity'] = result_csv['activity'].astype(int)
|
||||
result_dict = dict(zip(result_csv['user_id'], result_csv['activity']))
|
||||
|
||||
# 将训练结果保存到数据库
|
||||
save_train_result_to_pgsql(result_csv)
|
||||
|
||||
# 保存模型和结果
|
||||
pickle.dump(kmeans_model, open(activity_analysis_path + 'results/user_activity_model.pkl', 'wb'))
|
||||
pickle.dump(result_dict, open(activity_analysis_path + 'results/user_activity_dict.pkl', 'wb'))
|
||||
|
||||
logger.info('用户活跃度分析模型训练完成')
|
||||
return result_dict
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
train()
|
||||
|
|
@ -1,36 +0,0 @@
|
|||
import os
|
||||
from datetime import datetime
|
||||
from utils import create_logger
|
||||
|
||||
|
||||
# 项目根目录
|
||||
root_path = os.path.abspath(os.path.dirname(__file__))
|
||||
|
||||
# 各子模型目录
|
||||
activity_analysis_path = root_path + '/activity_analysis/'
|
||||
contribution_analysis_path = root_path + '/contribution_analysis/'
|
||||
interests_analysis_path = root_path + '/interests_analysis/'
|
||||
learning_ability_analysis_path = root_path + '/learning_ability_analysis/'
|
||||
programming_ability_analysis_path = root_path + '/programming_ability_analysis/'
|
||||
professional_ability_analysis_path = root_path + '/professional_ability_analysis/'
|
||||
user_label_analysis_path = root_path + '/user_label_analysis/'
|
||||
|
||||
# 随机数种子
|
||||
RANDOM_SEED = 42
|
||||
|
||||
# 创建日志对象
|
||||
logger = create_logger(root_path + '/logs/' + str(datetime.date(datetime.now())) + '.log')
|
||||
|
||||
# mysql连接配置
|
||||
mysql_host = "rm-bp13v5020p7828r5rso.mysql.rds.aliyuncs.com"
|
||||
mysql_user = "testeducoder"
|
||||
mysql_passwd = "TEST@123"
|
||||
mysql_port = 3306
|
||||
mysql_database = "preeducoderweb"
|
||||
|
||||
# 之前登录的天数
|
||||
before_login_days = 30
|
||||
|
||||
test_user_id = 201
|
||||
|
||||
data_before_days = 10
|
||||
|
|
@ -1,36 +0,0 @@
|
|||
user_id value created_on
|
||||
9 4 2022-07-26
|
||||
10 4 2022-08-11
|
||||
5 4 2022-07-21
|
||||
4 4 2022-07-13
|
||||
18 4 2022-08-05
|
||||
20 4 2022-08-11
|
||||
11 4 2022-08-29
|
||||
1 4 2022-07-14
|
||||
1 4 2022-06-30
|
||||
1 4 2022-07-21
|
||||
4 4 2022-08-10
|
||||
1 4 2022-06-30
|
||||
25 4 2022-08-30
|
||||
1 4 2022-07-05
|
||||
18 4 2022-08-12
|
||||
4 4 2022-08-10
|
||||
1 4 2022-07-08
|
||||
4 4 2022-08-11
|
||||
1 4 2022-07-08
|
||||
1 4 2022-07-13
|
||||
6 4 2022-08-08
|
||||
1 4 2022-06-30
|
||||
2 4 2022-08-08
|
||||
8 4 2022-07-22
|
||||
6 4 2022-08-08
|
||||
4 4 2022-08-17
|
||||
25 4 2022-08-30
|
||||
9 4 2022-08-22
|
||||
22 4 2022-08-30
|
||||
9 4 2022-08-22
|
||||
28 4 2022-09-07
|
||||
4 4 2022-08-22
|
||||
29 4 2022-09-07
|
||||
25 4 2022-08-30
|
||||
4 4 2022-08-26
|
||||
|
|
|
@ -1,4 +0,0 @@
|
|||
user_id value created_on
|
||||
10 1 2022-08-12
|
||||
10 1 2022-08-22
|
||||
10 1 2022-09-08
|
||||
|
|
|
@ -1,109 +0,0 @@
|
|||
import datetime
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from pandas.io import sql
|
||||
from config import logger
|
||||
from utils import get_before_date
|
||||
from config import data_before_days
|
||||
from sqlalchemy import create_engine
|
||||
from config import contribution_analysis_path
|
||||
from config import host, port, user, passwd, database
|
||||
|
||||
|
||||
def get_data_from_pgsql(latest_data_date):
|
||||
"""
|
||||
从mysql数据库获取原始数据
|
||||
"""
|
||||
start = datetime.datetime.now()
|
||||
logger.info("开始获取用户贡献度数据...")
|
||||
|
||||
engine = create_engine(f'postgresql+psycopg2://{user}:{passwd}@{host}:{port}/{database}', pool_recycle=3600)
|
||||
sql_conn = engine.connect()
|
||||
|
||||
sql_text1 = f"""
|
||||
select
|
||||
t.owner_id user_id, 4 as value, to_char(to_timestamp(created_unix), 'YYYY-MM-DD') created_on
|
||||
from repository t
|
||||
WHERE
|
||||
to_char(to_timestamp(created_unix), 'YYYY-MM-DD') >= '{latest_data_date}'
|
||||
"""
|
||||
|
||||
data_journals = pd.read_sql(sql_text1, con=sql_conn)
|
||||
data_journals.to_csv(contribution_analysis_path + 'data/journals.csv', index=False, header=True, sep='\t')
|
||||
|
||||
logger.info("用户贡献度数据1数据下载完毕，共" + str(data_journals.shape[0]) + "条数据，总耗时" + str((datetime.datetime.now() - start).seconds) + "秒")
|
||||
|
||||
sql_text2 = f"""
|
||||
select
|
||||
t.uid user_id, 1 as value, to_char(to_timestamp(created_unix), 'YYYY-MM-DD') created_on
|
||||
from star t
|
||||
WHERE
|
||||
to_char(to_timestamp(created_unix), 'YYYY-MM-DD') >= '{latest_data_date}'
|
||||
"""
|
||||
|
||||
data_journals_for_messages = pd.read_sql(sql_text2, con=sql_conn)
|
||||
data_journals_for_messages.to_csv(contribution_analysis_path + 'data/journals_for_messages.csv', index=False, header=True, sep='\t')
|
||||
|
||||
logger.info(
|
||||
"用户贡献度数据2下载完毕，共" + str(data_journals_for_messages.shape[0]) + "条数据，总耗时" + str((datetime.datetime.now() - start).seconds) + "秒")
|
||||
|
||||
|
||||
def read_action_data():
|
||||
"""
|
||||
处理用户贡献度度数据
|
||||
"""
|
||||
data_journals = pd.read_csv(contribution_analysis_path + 'data/journals.csv', sep='\t')
|
||||
data_journals = data_journals[['user_id', 'value', 'created_on']]
|
||||
|
||||
data_journals_for_message = pd.read_csv(contribution_analysis_path + 'data/journals_for_messages.csv', sep='\t')
|
||||
data_journals_for_message = data_journals_for_message[['user_id', 'value', 'created_on']]
|
||||
# 连接两个表
|
||||
data_concat = pd.concat([data_journals, data_journals_for_message], ignore_index=True)
|
||||
# 转换日期格式
|
||||
data_concat['created_on'] = pd.to_datetime(data_concat['created_on'], dayfirst=True)
|
||||
data_concat['created_on'] = data_concat['created_on'].dt.date
|
||||
data_concat = data_concat.dropna()
|
||||
|
||||
return data_concat
|
||||
|
||||
|
||||
def get_rfm_data():
|
||||
"""
|
||||
生成RFM聚类模型训练数据
|
||||
"""
|
||||
data_all = read_action_data()
|
||||
# 计算 R,F,M 值
|
||||
df_rfm = data_all[['user_id', 'value', 'created_on']]
|
||||
df_rfm = df_rfm.groupby("user_id").agg({'created_on': 'max', 'user_id': 'count', 'value': 'sum'})
|
||||
df_rfm = df_rfm.rename(columns={'created_on': 'Recentdate', 'user_id': 'F', 'value': 'M'})
|
||||
|
||||
df_rfm["R"] = (df_rfm['Recentdate'].max() - df_rfm['Recentdate']) / np.timedelta64(1, 'D')
|
||||
df_rfm["R"] = df_rfm["R"].astype('str').str.split(" ", expand=True)
|
||||
df_rfm["R"] = df_rfm["R"].astype("float").astype("int")
|
||||
df_rfm.drop(columns='Recentdate', inplace=True)
|
||||
|
||||
return df_rfm
|
||||
|
||||
|
||||
def save_train_result_to_pgsql(df):
|
||||
start = datetime.datetime.now()
|
||||
logger.info("开始将用户贡献度数据保存到数据库...")
|
||||
|
||||
engine = create_engine(f'postgresql+psycopg2://{user}:{passwd}@{host}:{port}/{database}', pool_recycle=3600)
|
||||
conn = engine.connect()
|
||||
type_str = 'contribution'
|
||||
|
||||
truncate_sql = f"DELETE FROM ai_user_portraint_detail WHERE type = '{type_str}'"
|
||||
sql.execute(truncate_sql, conn)
|
||||
|
||||
df = df.rename(columns={'contribution': 'value'})
|
||||
df['type'] = type_str
|
||||
df['created_unix'] = int(datetime.datetime.now().timestamp())
|
||||
df.to_sql('ai_user_portraint_detail', conn, index=False, if_exists='append')
|
||||
logger.info("用户贡献度数据保存到数据库完毕，共" + str(df.shape[0]) + "条数据，总耗时" + str((datetime.datetime.now() - start).seconds) + "秒")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
# 取数据
|
||||
latest_data_date = get_before_date(data_before_days)
|
||||
get_data_from_pgsql(latest_data_date)
|
||||
Binary file not shown.
Binary file not shown.
|
|
@ -1,24 +0,0 @@
|
|||
import pickle
|
||||
from config import logger
|
||||
from config import test_user_id
|
||||
from config import contribution_analysis_path
|
||||
|
||||
logger.info('加载用户贡献度度字典')
|
||||
user_contribution_dict = pickle.load(open(contribution_analysis_path + 'results/user_contribution_dict.pkl', 'rb'))
|
||||
|
||||
|
||||
def user_contribution_predict(user_id):
|
||||
"""
|
||||
用户贡献度预测
|
||||
"""
|
||||
if user_id not in user_contribution_dict:
|
||||
result = -1
|
||||
else:
|
||||
result = user_contribution_dict[user_id]
|
||||
|
||||
return result
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
result = user_contribution_predict(user_id=test_user_id)
|
||||
print('用户ID：', test_user_id, '活跃度：', result)
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
## 这里存放模型的训练结果
|
||||
|
||||
|
|
@ -1,84 +0,0 @@
|
|||
import pickle
|
||||
import warnings
|
||||
import pandas as pd
|
||||
from config import logger
|
||||
from config import RANDOM_SEED
|
||||
from sklearn.cluster import KMeans
|
||||
from data_process import get_rfm_data, save_train_result_to_pgsql
|
||||
from config import contribution_analysis_path
|
||||
warnings.filterwarnings('ignore')
|
||||
|
||||
|
||||
def iflabel(x):
|
||||
# 设置用户贡献度打分规则
|
||||
if x == "高高高":
|
||||
return 5
|
||||
elif x == "高低高":
|
||||
return 4
|
||||
elif x == "高高低":
|
||||
return 4
|
||||
elif x == "高低低":
|
||||
return 3
|
||||
elif x == "低高高":
|
||||
return 4
|
||||
elif x == "低低高":
|
||||
return 2
|
||||
elif x == "低高低":
|
||||
return 2
|
||||
elif x == "低低低":
|
||||
return 1
|
||||
|
||||
|
||||
def train():
|
||||
"""
|
||||
用户贡献度分析模型训练
|
||||
"""
|
||||
logger.info('开始训练用户贡献度分析模型...')
|
||||
df_rfm = get_rfm_data()
|
||||
rfm_cluster = df_rfm.copy()
|
||||
|
||||
# 根据R,F,M值进行聚类，分为8类
|
||||
Cluster = df_rfm[["R", "F", "M"]].values
|
||||
kmeans_model = KMeans(n_clusters=8, random_state=RANDOM_SEED).fit(Cluster)
|
||||
rfm_cluster["label"] = kmeans_model.labels_
|
||||
|
||||
# 先求每一类的R、F、M均值 # 此处计算均值 RFM 模型，每个用户的R、F、M值同总体均值进行比较，比均值高的就为“高”，低的就标明“低”，然后通过三个指标的高低来打标签。
|
||||
df_rfm_mean = rfm_cluster[["label", "R", "F", "M"]].groupby("label").agg("mean")
|
||||
df_rfm_mean["IFR"] = df_rfm_mean["R"].apply(lambda x: "高" if x > df_rfm_mean["R"].mean() else "低")
|
||||
df_rfm_mean["IFF"] = df_rfm_mean["F"].apply(lambda x: "高" if x > df_rfm_mean["F"].mean() else "低")
|
||||
df_rfm_mean["IFM"] = df_rfm_mean["M"].apply(lambda x: "高" if x > df_rfm_mean["M"].mean() else "低")
|
||||
|
||||
# 方法和上面一样，#将三个指标合并起来，生成临时列temp，表示RFM综合指标
|
||||
df_rfm_mean["temp"] = df_rfm_mean["IFR"] + df_rfm_mean["IFF"] + df_rfm_mean["IFM"]
|
||||
#将指标进行综合判断
|
||||
df_rfm_mean["label"] = df_rfm_mean["temp"].apply(lambda x: iflabel(x))
|
||||
# 此处在使用聚类RFM模型，比较的时候使用总体均值进行比较
|
||||
rfm_cluster["IFR"] = rfm_cluster["R"].apply(lambda x: "高" if x > df_rfm_mean["R"].mean() else "低")
|
||||
rfm_cluster["IFF"] = rfm_cluster["F"].apply(lambda x: "高" if x > df_rfm_mean["F"].mean() else "低")
|
||||
rfm_cluster["IFM"] = rfm_cluster["M"].apply(lambda x: "高" if x > df_rfm_mean["M"].mean() else "低")
|
||||
## 方法和上面一样，将三个指标合并起来，生成临时列temp，表示RFM综合指标
|
||||
rfm_cluster["temp"] = rfm_cluster["IFR"] + rfm_cluster["IFF"] + rfm_cluster["IFM"]
|
||||
rfm_cluster["label"] = rfm_cluster["temp"].apply(lambda x: iflabel(x))
|
||||
# 根据打分规则函数给用户进行打分
|
||||
result = rfm_cluster[['label']]
|
||||
|
||||
# 将用户 id 和贡献度提取出来保存在字典中
|
||||
result_csv = pd.DataFrame(list(result.index),index=range(len(list(result.index))),columns = ['user_id'])
|
||||
result_csv['contribution'] = list(result.values.reshape(1,len(list(result.index)))[0])
|
||||
result_csv['user_id'] = result_csv['user_id'].astype(int)
|
||||
result_csv['contribution'] = result_csv['contribution'].astype(int)
|
||||
results_dict = dict(zip(result_csv['user_id'], result_csv['contribution']))
|
||||
|
||||
# 将训练结果保存到数据库
|
||||
save_train_result_to_pgsql(result_csv)
|
||||
|
||||
# 保存模型和结果
|
||||
pickle.dump(kmeans_model, open(contribution_analysis_path + 'results/user_contribution_model.pkl', 'wb'))
|
||||
pickle.dump(results_dict, open(contribution_analysis_path + 'results/user_contribution_dict.pkl', 'wb'))
|
||||
|
||||
logger.info('用户贡献度度分析模型训练完成')
|
||||
return results_dict
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
train()
|
||||
|
|
@ -1,193 +0,0 @@
|
|||
'''
|
||||
@Compony: EduCoder
|
||||
@Author: dengzaiyong
|
||||
@Date: 2022-03-29 15:16:08
|
||||
@LastEditTime: 2022-03-29 19:37:08
|
||||
@LastEditors: dengzaiyong
|
||||
@Description: 用户画像分析模型预测接口
|
||||
@FilePath: /user_portrait_analysis/flask_app.py
|
||||
'''
|
||||
|
||||
import json
|
||||
from flask import Flask
|
||||
from flask_cors import CORS
|
||||
from flask import request
|
||||
from activity_analysis.predict import user_activity_predict
|
||||
from contribution_analysis.predict import user_contribution_predict
|
||||
from interests_analysis.predict import user_interests_predict
|
||||
from professional_ability_analysis.predict import user_professional_ability_predict
|
||||
from programming_ability_analysis.predict import user_programming_ability_predict
|
||||
from user_label_analysis.predict import user_label_predict
|
||||
|
||||
app = Flask(__name__)
|
||||
CORS(app, resources=r'/*')
|
||||
|
||||
@app.route('/user_activity', methods=["POST"])
|
||||
def get_user_activity():
|
||||
'''
|
||||
以RESTful的方式获取用户活跃度分析模型结果
|
||||
:param user_id: 用户ID
|
||||
|
||||
:return 以json格式返回
|
||||
'''
|
||||
result = {}
|
||||
|
||||
user_id = request.form.get('user_id', type=str, default='')
|
||||
|
||||
if user_id.strip() == '':
|
||||
result = {
|
||||
"status_code": str('False'),
|
||||
"error_msg": str('参数错误: 缺少user_id')
|
||||
}
|
||||
return json.dumps(result, ensure_ascii=False)
|
||||
|
||||
results = user_activity_predict(int(user_id))
|
||||
result = {
|
||||
"status_code": str('True'),
|
||||
"user_id": str(user_id),
|
||||
"results": str(results),
|
||||
}
|
||||
return json.dumps(result, ensure_ascii=False)
|
||||
|
||||
@app.route('/user_contribution', methods=["POST"])
|
||||
def get_user_contribution():
|
||||
'''
|
||||
以RESTful的方式获取用户活跃度分析模型结果
|
||||
:param user_id: 用户ID
|
||||
|
||||
:return 以json格式返回
|
||||
'''
|
||||
result = {}
|
||||
|
||||
user_id = request.form.get('user_id', type=str, default='')
|
||||
|
||||
if user_id.strip() == '':
|
||||
result = {
|
||||
"status_code": str('False'),
|
||||
"error_msg": str('参数错误: 缺少user_id')
|
||||
}
|
||||
return json.dumps(result, ensure_ascii=False)
|
||||
|
||||
results = user_contribution_predict(int(user_id))
|
||||
result = {
|
||||
"status_code": str('True'),
|
||||
"user_id": str(user_id),
|
||||
"results": str(results),
|
||||
}
|
||||
return json.dumps(result, ensure_ascii=False)
|
||||
|
||||
@app.route('/user_interests', methods=["POST"])
|
||||
def get_user_interests():
|
||||
'''
|
||||
以RESTful的方式获取用户活跃度分析模型结果
|
||||
:param user_id: 用户ID
|
||||
|
||||
:return 以json格式返回
|
||||
'''
|
||||
result = {}
|
||||
|
||||
user_id = request.form.get('user_id', type=str, default='')
|
||||
|
||||
if user_id.strip() == '':
|
||||
result = {
|
||||
"status_code": str('False'),
|
||||
"error_msg": str('参数错误: 缺少user_id')
|
||||
}
|
||||
return json.dumps(result, ensure_ascii=False)
|
||||
|
||||
results1,results2 = user_interests_predict(int(user_id))
|
||||
result = {
|
||||
"status_code": str('True'),
|
||||
"user_id": str(user_id),
|
||||
"results1": str(results1),
|
||||
"results2": str(results2),
|
||||
}
|
||||
return json.dumps(result, ensure_ascii=False)
|
||||
|
||||
@app.route('/user_professional_ability', methods=["POST"])
|
||||
def get_user_professional_ability():
|
||||
'''
|
||||
以RESTful的方式获取用户活跃度分析模型结果
|
||||
:param user_id: 用户ID
|
||||
|
||||
:return 以json格式返回
|
||||
'''
|
||||
result = {}
|
||||
|
||||
user_id = request.form.get('user_id', type=str, default='')
|
||||
|
||||
if user_id.strip() == '':
|
||||
result = {
|
||||
"status_code": str('False'),
|
||||
"error_msg": str('参数错误: 缺少user_id')
|
||||
}
|
||||
return json.dumps(result, ensure_ascii=False)
|
||||
|
||||
results = user_professional_ability_predict(int(user_id))
|
||||
result = {
|
||||
"status_code": str('True'),
|
||||
"user_id": str(user_id),
|
||||
"results": str(results),
|
||||
}
|
||||
return json.dumps(result, ensure_ascii=False)
|
||||
|
||||
@app.route('/user_programming_ability', methods=["POST"])
|
||||
def get_user_programming_ability():
|
||||
'''
|
||||
以RESTful的方式获取用户活跃度分析模型结果
|
||||
:param user_id: 用户ID
|
||||
|
||||
:return 以json格式返回
|
||||
'''
|
||||
result = {}
|
||||
|
||||
user_id = request.form.get('user_id', type=str, default='')
|
||||
|
||||
if user_id.strip() == '':
|
||||
result = {
|
||||
"status_code": str('False'),
|
||||
"error_msg": str('参数错误: 缺少user_id')
|
||||
}
|
||||
return json.dumps(result, ensure_ascii=False)
|
||||
|
||||
results = user_programming_ability_predict(int(user_id))
|
||||
result = {
|
||||
"status_code": str('True'),
|
||||
"user_id": str(user_id),
|
||||
"results": str(results),
|
||||
}
|
||||
return json.dumps(result, ensure_ascii=False)
|
||||
|
||||
@app.route('/user_labels', methods=["POST"])
|
||||
def get_user_labels():
|
||||
'''
|
||||
以RESTful的方式获取用户活跃度分析模型结果
|
||||
:param user_id: 用户ID
|
||||
|
||||
:return 以json格式返回
|
||||
'''
|
||||
result = {}
|
||||
|
||||
user_id = request.form.get('user_id', type=str, default='')
|
||||
|
||||
if user_id.strip() == '':
|
||||
result = {
|
||||
"status_code": str('False'),
|
||||
"error_msg": str('参数错误: 缺少user_id')
|
||||
}
|
||||
return json.dumps(result, ensure_ascii=False)
|
||||
|
||||
r1,r2,r3,r4 = user_label_predict(int(user_id))
|
||||
result = {
|
||||
"status_code": str('True'),
|
||||
"user_id": str(user_id),
|
||||
"results1": str(r1),
|
||||
"results2": str(r2),
|
||||
"results3": str(r3),
|
||||
"results4": str(r4),
|
||||
}
|
||||
return json.dumps(result, ensure_ascii=False)
|
||||
|
||||
# python -m flask run
|
||||
if __name__ == '__main__':
|
||||
app.run(host='0.0.0.0', port=8088, debug=True, use_reloader=False)
|
||||
|
|
@ -1 +0,0 @@
|
|||
## 这里面存放数据
|
||||
|
|
@ -1,157 +0,0 @@
|
|||
import os
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from config import logger
|
||||
import datetime
|
||||
import pymysql
|
||||
from utils import get_before_date
|
||||
from config import mysql_database, mysql_passwd
|
||||
from config import mysql_host, mysql_port, mysql_user
|
||||
from config import interests_analysis_path
|
||||
from config import data_before_days
|
||||
|
||||
|
||||
def get_data_from_mysql(latest_data_date):
|
||||
"""
|
||||
从mysql数据库获取原始数据
|
||||
"""
|
||||
start = datetime.datetime.now()
|
||||
logger.info("开始获取用户爱好数据...")
|
||||
|
||||
sql_conn = pymysql.connect(host=mysql_host,
|
||||
user=mysql_user,
|
||||
passwd=mysql_passwd,
|
||||
port=mysql_port,
|
||||
db=mysql_database)
|
||||
|
||||
sql_text1 = f"""
|
||||
SELECT user_id,repertoire_id
|
||||
FROM user_interests
|
||||
"""
|
||||
|
||||
data_shixun = pd.read_sql(sql_text1, con=sql_conn)
|
||||
data_shixun.to_csv(interests_analysis_path + 'data/user_interests.csv', index=False, header=True, sep='\t')
|
||||
|
||||
logger.info("用户爱好数据1数据下载完毕，共" + str(data_shixun.shape[0]) + "条数据，总耗时" + str((datetime.datetime.now() - start).seconds) + "秒")
|
||||
|
||||
sql_text2 = f"""
|
||||
SELECT id,name
|
||||
FROM repertoires
|
||||
"""
|
||||
|
||||
data_repertoirse = pd.read_sql(sql_text2, con=sql_conn)
|
||||
data_repertoirse.to_csv(interests_analysis_path + 'data/repertoires.csv', index=False, header=True, sep='\t')
|
||||
|
||||
logger.info(
|
||||
"用户贡献度数据2下载完毕，共" + str(data_repertoirse.shape[0]) + "条数据，总耗时" + str((datetime.datetime.now() - start).seconds) + "秒")
|
||||
|
||||
sql_text3 = f"""
|
||||
SELECT id,user_id,visits
|
||||
FROM shixuns
|
||||
"""
|
||||
|
||||
data_shixuns = pd.read_sql(sql_text3, con=sql_conn)
|
||||
data_shixuns.to_csv(interests_analysis_path + 'data/shixuns.csv', index=False, header=True, sep='\t')
|
||||
|
||||
logger.info(
|
||||
"用户贡献度数据3下载完毕，共" + str(data_shixuns.shape[0]) + "条数据，总耗时" + str((datetime.datetime.now() - start).seconds) + "秒")
|
||||
|
||||
sql_text4 = f"""
|
||||
SELECT shixun_id,tag_repertoire_id
|
||||
FROM shixun_tag_repertoires
|
||||
"""
|
||||
|
||||
data_shixun_tag_repertoires = pd.read_sql(sql_text4, con=sql_conn)
|
||||
data_shixun_tag_repertoires.to_csv(interests_analysis_path + 'data/shixun_tag_repertoires.csv', index=False, header=True, sep='\t')
|
||||
|
||||
logger.info(
|
||||
"用户贡献度数据4下载完毕，共" + str(data_shixun_tag_repertoires.shape[0]) + "条数据，总耗时" + str((datetime.datetime.now() - start).seconds) + "秒")
|
||||
|
||||
sql_text5 = f"""
|
||||
SELECT id,sub_repertoire_id,name
|
||||
FROM tag_repertoires
|
||||
"""
|
||||
|
||||
data_tag_repertoires = pd.read_sql(sql_text5, con=sql_conn)
|
||||
data_tag_repertoires.to_csv(interests_analysis_path + 'data/tag_repertoires.csv', index=False, header=True, sep='\t')
|
||||
|
||||
logger.info(
|
||||
"用户贡献度数据5下载完毕，共" + str(data_tag_repertoires.shape[0]) + "条数据，总耗时" + str((datetime.datetime.now() - start).seconds) + "秒")
|
||||
|
||||
sql_text6 = f"""
|
||||
SELECT id,name,repertoire_id
|
||||
FROM sub_repertoires
|
||||
"""
|
||||
|
||||
data_sub_repertoires = pd.read_sql(sql_text6, con=sql_conn)
|
||||
data_sub_repertoires.to_csv(interests_analysis_path + 'data/sub_repertoires.csv', index=False, header=True, sep='\t')
|
||||
|
||||
logger.info(
|
||||
"用户贡献度数据6下载完毕，共" + str(data_sub_repertoires.shape[0]) + "条数据，总耗时" + str((datetime.datetime.now() - start).seconds) + "秒")
|
||||
|
||||
|
||||
def read_action_data():
|
||||
"""
|
||||
处理爱好数据
|
||||
"""
|
||||
# 读取所有与用户兴趣表格相关的表格
|
||||
data_user_interests = pd.read_csv(interests_analysis_path + 'data/user_interests.csv', sep='\t')
|
||||
data_user_interests= data_user_interests[['user_id','repertoire_id']]
|
||||
|
||||
data_repertosires = pd.read_csv(interests_analysis_path + 'data/repertoires.csv', sep='\t')
|
||||
data_repertosires = data_repertosires[['id','name']]
|
||||
data_repertosires = data_repertosires.rename(columns={'id': 'repertoire_id'})
|
||||
|
||||
data_shixuns = pd.read_csv(interests_analysis_path + 'data/shixuns.csv', sep='\t')
|
||||
data_shixuns = data_shixuns[['id', 'user_id', 'visits']]
|
||||
data_shixuns = data_shixuns.rename(columns={'id': 'shixun_id'})
|
||||
data_shixuns = data_shixuns.dropna()
|
||||
data_shixuns['user_id'] = data_shixuns['user_id'].astype("float").astype("int")
|
||||
data_shixuns = data_shixuns[data_shixuns.visits > data_shixuns.visits.mean()]
|
||||
|
||||
data_shixun_tag_repertoires = pd.read_csv(interests_analysis_path + 'data/shixun_tag_repertoires.csv', sep='\t')
|
||||
data_shixun_tag_repertoires = data_shixun_tag_repertoires[['shixun_id','tag_repertoire_id']]
|
||||
data_shixun_tag_repertoires = data_shixun_tag_repertoires.dropna()
|
||||
|
||||
data_tag_repertoires = pd.read_csv(interests_analysis_path + 'data/tag_repertoires.csv', sep='\t')
|
||||
data_tag_repertoires = data_tag_repertoires[['id','sub_repertoire_id','name']]
|
||||
data_tag_repertoires = data_tag_repertoires.rename(columns={'id': 'tag_repertoire_id'})
|
||||
|
||||
data_sub_repertoires = pd.read_csv(interests_analysis_path + 'data/sub_repertoires.csv', sep='\t')
|
||||
data_sub_repertoires = data_sub_repertoires[['id', 'name', 'repertoire_id']]
|
||||
data_sub_repertoires = data_sub_repertoires.rename(columns={'id': 'sub_repertoire_id'})
|
||||
|
||||
return data_user_interests,data_repertosires,data_shixuns,data_shixun_tag_repertoires,data_tag_repertoires,data_sub_repertoires
|
||||
|
||||
|
||||
def get_hob_data():
|
||||
"""
|
||||
生成用户爱好数据
|
||||
"""
|
||||
# 进行树形搜索，将用户感兴趣学科的所有相关的词都给搜索出来作为用户的爱好
|
||||
df_hob_1, df_hob_2, df_hob_3, df_hob_4, df_hob_5, df_hob_6 = read_action_data()
|
||||
df_merge_1 = pd.merge(df_hob_4, df_hob_5, on='tag_repertoire_id')
|
||||
df_merge_2 = df_merge_1[['shixun_id', 'sub_repertoire_id', 'name']]
|
||||
|
||||
df_merge_3 = pd.merge(df_merge_2, df_hob_6, on='sub_repertoire_id')
|
||||
df_merge_3 = df_merge_3[['shixun_id', 'name_x', 'name_y', 'repertoire_id']]
|
||||
|
||||
df_merge_4 = pd.merge(df_hob_2, df_merge_3, on='repertoire_id')
|
||||
df_merge_4 = df_merge_4[['shixun_id', 'name', 'name_x', 'name_y']]
|
||||
|
||||
df_merge_5 = df_hob_3[['user_id', 'shixun_id']]
|
||||
|
||||
df_merge_6 = pd.merge(df_merge_4, df_merge_5, on='shixun_id')
|
||||
df_merge_6 = df_merge_6[['user_id', 'name', 'name_x', 'name_y']]
|
||||
|
||||
# 将所有表格连接后将所有关键词加起来变为一个整个字符串
|
||||
df_merge_6['hobbies'] = df_merge_6['name'] + ' ' + df_merge_6['name_x'] + ' ' + df_merge_6['name_y']
|
||||
data = df_merge_6[['user_id', 'hobbies']]
|
||||
|
||||
return data, df_hob_1, df_hob_2
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
# 取数据
|
||||
latest_data_date = get_before_date(data_before_days)
|
||||
get_data_from_mysql(latest_data_date)
|
||||
|
|
@ -1,157 +0,0 @@
|
|||
import os
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from config import logger
|
||||
import datetime
|
||||
import pymysql
|
||||
from utils import get_before_date
|
||||
from config import mysql_database, mysql_passwd
|
||||
from config import mysql_host, mysql_port, mysql_user
|
||||
from config import interests_analysis_path
|
||||
from config import data_before_days
|
||||
|
||||
|
||||
def get_data_from_mysql(latest_data_date):
|
||||
"""
|
||||
从mysql数据库获取原始数据
|
||||
"""
|
||||
start = datetime.datetime.now()
|
||||
logger.info("开始获取用户爱好数据...")
|
||||
|
||||
sql_conn = pymysql.connect(host=mysql_host,
|
||||
user=mysql_user,
|
||||
passwd=mysql_passwd,
|
||||
port=mysql_port,
|
||||
db=mysql_database)
|
||||
|
||||
sql_text1 = f"""
|
||||
SELECT user_id,repertoire_id
|
||||
FROM user_interests
|
||||
"""
|
||||
|
||||
data_shixun = pd.read_sql(sql_text1, con=sql_conn)
|
||||
data_shixun.to_csv(interests_analysis_path + 'data/user_interests.csv', index=False, header=True, sep='\t')
|
||||
|
||||
logger.info("用户爱好数据1数据下载完毕，共" + str(data_shixun.shape[0]) + "条数据，总耗时" + str((datetime.datetime.now() - start).seconds) + "秒")
|
||||
|
||||
sql_text2 = f"""
|
||||
SELECT id,name
|
||||
FROM repertoires
|
||||
"""
|
||||
|
||||
data_repertoirse = pd.read_sql(sql_text2, con=sql_conn)
|
||||
data_repertoirse.to_csv(interests_analysis_path + 'data/repertoires.csv', index=False, header=True, sep='\t')
|
||||
|
||||
logger.info(
|
||||
"用户贡献度数据2下载完毕，共" + str(data_repertoirse.shape[0]) + "条数据，总耗时" + str((datetime.datetime.now() - start).seconds) + "秒")
|
||||
|
||||
sql_text3 = f"""
|
||||
SELECT id,user_id,visits
|
||||
FROM shixuns
|
||||
"""
|
||||
|
||||
data_shixuns = pd.read_sql(sql_text3, con=sql_conn)
|
||||
data_shixuns.to_csv(interests_analysis_path + 'data/shixuns.csv', index=False, header=True, sep='\t')
|
||||
|
||||
logger.info(
|
||||
"用户贡献度数据3下载完毕，共" + str(data_shixuns.shape[0]) + "条数据，总耗时" + str((datetime.datetime.now() - start).seconds) + "秒")
|
||||
|
||||
sql_text4 = f"""
|
||||
SELECT shixun_id,tag_repertoire_id
|
||||
FROM shixun_tag_repertoires
|
||||
"""
|
||||
|
||||
data_shixun_tag_repertoires = pd.read_sql(sql_text4, con=sql_conn)
|
||||
data_shixun_tag_repertoires.to_csv(interests_analysis_path + 'data/shixun_tag_repertoires.csv', index=False, header=True, sep='\t')
|
||||
|
||||
logger.info(
|
||||
"用户贡献度数据4下载完毕，共" + str(data_shixun_tag_repertoires.shape[0]) + "条数据，总耗时" + str((datetime.datetime.now() - start).seconds) + "秒")
|
||||
|
||||
sql_text5 = f"""
|
||||
SELECT id,sub_repertoire_id,name
|
||||
FROM tag_repertoires
|
||||
"""
|
||||
|
||||
data_tag_repertoires = pd.read_sql(sql_text5, con=sql_conn)
|
||||
data_tag_repertoires.to_csv(interests_analysis_path + 'data/tag_repertoires.csv', index=False, header=True, sep='\t')
|
||||
|
||||
logger.info(
|
||||
"用户贡献度数据5下载完毕，共" + str(data_tag_repertoires.shape[0]) + "条数据，总耗时" + str((datetime.datetime.now() - start).seconds) + "秒")
|
||||
|
||||
sql_text6 = f"""
|
||||
SELECT id,name,repertoire_id
|
||||
FROM sub_repertoires
|
||||
"""
|
||||
|
||||
data_sub_repertoires = pd.read_sql(sql_text6, con=sql_conn)
|
||||
data_sub_repertoires.to_csv(interests_analysis_path + 'data/sub_repertoires.csv', index=False, header=True, sep='\t')
|
||||
|
||||
logger.info(
|
||||
"用户贡献度数据6下载完毕，共" + str(data_sub_repertoires.shape[0]) + "条数据，总耗时" + str((datetime.datetime.now() - start).seconds) + "秒")
|
||||
|
||||
|
||||
def read_action_data():
|
||||
"""
|
||||
处理爱好数据
|
||||
"""
|
||||
# 读取所有与用户兴趣表格相关的表格
|
||||
data_user_interests = pd.read_csv(interests_analysis_path + 'data/user_interests.csv', sep='\t')
|
||||
data_user_interests= data_user_interests[['user_id','repertoire_id']]
|
||||
|
||||
data_repertosires = pd.read_csv(interests_analysis_path + 'data/repertoires.csv', sep='\t')
|
||||
data_repertosires = data_repertosires[['id','name']]
|
||||
data_repertosires = data_repertosires.rename(columns={'id': 'repertoire_id'})
|
||||
|
||||
data_shixuns = pd.read_csv(interests_analysis_path + 'data/shixuns.csv', sep='\t')
|
||||
data_shixuns = data_shixuns[['id', 'user_id', 'visits']]
|
||||
data_shixuns = data_shixuns.rename(columns={'id': 'shixun_id'})
|
||||
data_shixuns = data_shixuns.dropna()
|
||||
data_shixuns['user_id'] = data_shixuns['user_id'].astype("float").astype("int")
|
||||
data_shixuns = data_shixuns[data_shixuns.visits > data_shixuns.visits.mean()]
|
||||
|
||||
data_shixun_tag_repertoires = pd.read_csv(interests_analysis_path + 'data/shixun_tag_repertoires.csv', sep='\t')
|
||||
data_shixun_tag_repertoires = data_shixun_tag_repertoires[['shixun_id','tag_repertoire_id']]
|
||||
data_shixun_tag_repertoires = data_shixun_tag_repertoires.dropna()
|
||||
|
||||
data_tag_repertoires = pd.read_csv(interests_analysis_path + 'data/tag_repertoires.csv', sep='\t')
|
||||
data_tag_repertoires = data_tag_repertoires[['id','sub_repertoire_id','name']]
|
||||
data_tag_repertoires = data_tag_repertoires.rename(columns={'id': 'tag_repertoire_id'})
|
||||
|
||||
data_sub_repertoires = pd.read_csv(interests_analysis_path + 'data/sub_repertoires.csv', sep='\t')
|
||||
data_sub_repertoires = data_sub_repertoires[['id', 'name', 'repertoire_id']]
|
||||
data_sub_repertoires = data_sub_repertoires.rename(columns={'id': 'sub_repertoire_id'})
|
||||
|
||||
return data_user_interests,data_repertosires,data_shixuns,data_shixun_tag_repertoires,data_tag_repertoires,data_sub_repertoires
|
||||
|
||||
|
||||
def get_hob_data():
|
||||
"""
|
||||
生成用户爱好数据
|
||||
"""
|
||||
# 进行树形搜索，将用户感兴趣学科的所有相关的词都给搜索出来作为用户的爱好
|
||||
df_hob_1, df_hob_2, df_hob_3, df_hob_4, df_hob_5, df_hob_6 = read_action_data()
|
||||
df_merge_1 = pd.merge(df_hob_4, df_hob_5, on='tag_repertoire_id')
|
||||
df_merge_2 = df_merge_1[['shixun_id', 'sub_repertoire_id', 'name']]
|
||||
|
||||
df_merge_3 = pd.merge(df_merge_2, df_hob_6, on='sub_repertoire_id')
|
||||
df_merge_3 = df_merge_3[['shixun_id', 'name_x', 'name_y', 'repertoire_id']]
|
||||
|
||||
df_merge_4 = pd.merge(df_hob_2, df_merge_3, on='repertoire_id')
|
||||
df_merge_4 = df_merge_4[['shixun_id', 'name', 'name_x', 'name_y']]
|
||||
|
||||
df_merge_5 = df_hob_3[['user_id', 'shixun_id']]
|
||||
|
||||
df_merge_6 = pd.merge(df_merge_4, df_merge_5, on='shixun_id')
|
||||
df_merge_6 = df_merge_6[['user_id', 'name', 'name_x', 'name_y']]
|
||||
|
||||
# 将所有表格连接后将所有关键词加起来变为一个整个字符串
|
||||
df_merge_6['hobbies'] = df_merge_6['name'] + ' ' + df_merge_6['name_x'] + ' ' + df_merge_6['name_y']
|
||||
data = df_merge_6[['user_id', 'hobbies']]
|
||||
|
||||
return data, df_hob_1, df_hob_2
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
# 取数据
|
||||
latest_data_date = get_before_date(data_before_days)
|
||||
get_data_from_mysql(latest_data_date)
|
||||
Binary file not shown.
Binary file not shown.
|
|
@ -1,33 +0,0 @@
|
|||
import pickle
|
||||
from config import logger
|
||||
from config import test_user_id
|
||||
from config import interests_analysis_path
|
||||
|
||||
logger.info('加载用户爱好字典')
|
||||
user_interests_dict = pickle.load(open(interests_analysis_path + 'results/user_interests_dict.pkl', 'rb'))
|
||||
user_possible_hobby_dict = pickle.load(open(interests_analysis_path + 'results/user_possible_hobby_dict.pkl', 'rb'))
|
||||
def output_hobbies(hobbies):
|
||||
str = ''
|
||||
for i in range(len(hobbies)):
|
||||
str += hobbies[i]+' '
|
||||
return str
|
||||
def user_interests_predict(user_id):
|
||||
"""
|
||||
用户爱好预测
|
||||
"""
|
||||
if user_id not in user_interests_dict:
|
||||
result1 = list('')
|
||||
else:
|
||||
result1 = user_interests_dict[user_id]
|
||||
|
||||
if user_id not in user_possible_hobby_dict:
|
||||
result2 = list('')
|
||||
else:
|
||||
result2 = user_possible_hobby_dict[user_id]
|
||||
|
||||
return result1,result2
|
||||
|
||||
if __name__ == '__main__':
|
||||
result1,result2 = user_interests_predict(user_id=test_user_id)
|
||||
print('用户ID：', test_user_id, '的爱好有：', output_hobbies(result1))
|
||||
print('用户ID：', test_user_id, '可能的爱好有：', result2)
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
## 这里面存放模型训练结果
|
||||
|
||||
|
|
@ -1,185 +0,0 @@
|
|||
import os
|
||||
import pandas as pd
|
||||
import pickle
|
||||
from sklearn.cluster import KMeans
|
||||
from config import logger
|
||||
from config import RANDOM_SEED
|
||||
from data_process import get_hob_data
|
||||
from config import interests_analysis_path
|
||||
|
||||
"""
|
||||
协同过滤算法
|
||||
"""
|
||||
from abc import ABCMeta, abstractmethod
|
||||
import numpy as np
|
||||
from collections import defaultdict
|
||||
|
||||
class CF_base(metaclass=ABCMeta):
|
||||
def __init__(self, k=3):
|
||||
self.k = k
|
||||
self.n_user = None
|
||||
self.n_item = None
|
||||
|
||||
@abstractmethod
|
||||
def init_param(self, data):
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def cal_prediction(self, *args):
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def cal_recommendation(self, user_id, data):
|
||||
pass
|
||||
|
||||
def fit(self, data):
|
||||
# 计算所有用户的推荐物品
|
||||
self.init_param(data)
|
||||
all_users = []
|
||||
for i in range(self.n_user):
|
||||
all_users.append(self.cal_recommendation(i, data))
|
||||
return all_users
|
||||
|
||||
|
||||
class CF_knearest(CF_base):
|
||||
"""
|
||||
基于物品的K近邻协同过滤推荐算法
|
||||
"""
|
||||
|
||||
def __init__(self, k, criterion='cosine'):
|
||||
super(CF_knearest, self).__init__(k)
|
||||
self.criterion = criterion
|
||||
self.simi_mat = None
|
||||
return
|
||||
|
||||
def init_param(self, data):
|
||||
# 初始化参数
|
||||
self.n_user = data.shape[0]
|
||||
self.n_item = data.shape[1]
|
||||
self.simi_mat = self.cal_simi_mat(data)
|
||||
return
|
||||
|
||||
def cal_similarity(self, i, j, data):
|
||||
# 计算物品i和物品j的相似度
|
||||
items = data[:, [i, j]]
|
||||
del_inds = np.where(items == 0)[0]
|
||||
items = np.delete(items, del_inds, axis=0)
|
||||
if items.size == 0:
|
||||
similarity = 0
|
||||
else:
|
||||
v1 = items[:, 0]
|
||||
v2 = items[:, 1]
|
||||
if self.criterion == 'cosine':
|
||||
if np.std(v1) > 1e-3: # 方差过大，表明用户间评价尺度差别大需要进行调整
|
||||
v1 = v1 - v1.mean()
|
||||
if np.std(v2) > 1e-3:
|
||||
v2 = v2 - v2.mean()
|
||||
similarity = (v1 @ v2) / np.linalg.norm(v1, 2) / np.linalg.norm(v2, 2)
|
||||
elif self.criterion == 'pearson':
|
||||
similarity = np.corrcoef(v1, v2)[0, 1]
|
||||
else:
|
||||
raise ValueError('the method is not supported now')
|
||||
return similarity
|
||||
|
||||
def cal_simi_mat(self, data):
|
||||
# 计算物品间的相似度矩阵
|
||||
simi_mat = np.ones((self.n_item, self.n_item))
|
||||
for i in range(self.n_item):
|
||||
for j in range(i + 1, self.n_item):
|
||||
simi_mat[i, j] = self.cal_similarity(i, j, data)
|
||||
simi_mat[j, i] = simi_mat[i, j]
|
||||
return simi_mat
|
||||
|
||||
def cal_prediction(self, user_row, item_ind):
|
||||
# 计算预推荐物品i对目标活跃用户u的吸引力
|
||||
purchase_item_inds = np.where(user_row > 0)[0]
|
||||
rates = user_row[purchase_item_inds]
|
||||
simi = self.simi_mat[item_ind][purchase_item_inds]
|
||||
return np.sum(rates * simi) / np.linalg.norm(simi, 1)
|
||||
|
||||
def cal_recommendation(self, user_ind, data):
|
||||
# 计算目标用户的最具吸引力的k个物品list
|
||||
item_prediction = defaultdict(float)
|
||||
user_row = data[user_ind]
|
||||
un_purchase_item_inds = np.where(user_row == 0)[0]
|
||||
for item_ind in un_purchase_item_inds:
|
||||
item_prediction[item_ind] = self.cal_prediction(user_row, item_ind)
|
||||
res = sorted(item_prediction, key=item_prediction.get, reverse=True)
|
||||
return res[:self.k]
|
||||
|
||||
|
||||
def str2list(x):
|
||||
x = x.split(' ')
|
||||
x = list(set(x))
|
||||
return x
|
||||
|
||||
def train():
|
||||
"""
|
||||
用户爱好分析模型训练
|
||||
"""
|
||||
logger.info('开始训练爱好分析模型...')
|
||||
|
||||
# 将用户爱好数据进行整合
|
||||
data, df_hob_1, df_hob_2 = get_hob_data()
|
||||
result = data.groupby('user_id').sum()
|
||||
|
||||
# 挖掘用户现有的所有喜好
|
||||
# 将用户数据中重复的进行删除
|
||||
result['hobbies'] = result['hobbies'].apply(lambda x: str2list(x))
|
||||
|
||||
# 挖掘用户可能的喜好
|
||||
# 将用户在每个大类的使用情况统计出来
|
||||
df_hob_1_copy = df_hob_1.copy()
|
||||
df_hob_1_copy['values'] = True
|
||||
df_hob_cal = df_hob_1_copy.pivot_table(index='user_id', columns='repertoire_id', values='values',
|
||||
aggfunc='count').fillna(0)
|
||||
df_hob_cal = df_hob_cal.astype('int')
|
||||
|
||||
# 使用协同过滤算法预测用户最有可能对哪个科目感兴趣
|
||||
user_hob = np.array(df_hob_cal.values)
|
||||
cf_model = CF_knearest(k=1)
|
||||
df_hob_cf = cf_model.fit(user_hob)
|
||||
|
||||
# 将预测出来的结果转换为 repertoire_id
|
||||
y_pred = []
|
||||
for i in range(len(df_hob_cf)):
|
||||
if len(df_hob_cf[i]) == 0:
|
||||
y_pred.append(0)
|
||||
else:
|
||||
y_pred.append(df_hob_cf[i][0] + 1)
|
||||
|
||||
# 重塑表格将爱好 id 提取出来
|
||||
df_hob_result = pd.DataFrame(list(df_hob_cal.index), index=range(len(list(df_hob_cal.index))), columns=['user_id'])
|
||||
df_hob_result['repertoire_id'] = y_pred
|
||||
|
||||
# 将 repertoire_id 转化为爱好名称，并删除无效用户 id
|
||||
df_hob = pd.merge(df_hob_result, df_hob_2, on='repertoire_id', how='outer')
|
||||
df_hob = df_hob.fillna({'name': ' '})
|
||||
df_hob = df_hob.dropna()
|
||||
df_hob = df_hob[['user_id', 'name']]
|
||||
df_hob = df_hob.rename(columns={'name': 'possible_hobby'})
|
||||
df_hob['user_id'] = df_hob['user_id'].astype(int)
|
||||
|
||||
# 将用户 id 和爱好提取出来并保存
|
||||
result_csv1 = pd.DataFrame(list(result.index), index=range(len(list(result.index))), columns=['user_id'])
|
||||
result_csv1['hobbies'] = list(result.hobbies)
|
||||
result_csv1['user_id'] = result_csv1['user_id'].astype(int)
|
||||
results_dict1 = dict(zip(result_csv1['user_id'], result_csv1['hobbies']))
|
||||
|
||||
# 将用户 id 和爱好提取出来并保存
|
||||
result_csv2 = pd.DataFrame(list(df_hob.index), index=range(len(list(df_hob.index))), columns=['user_id'])
|
||||
result_csv2['user_id'] = result_csv2['user_id'].astype(int)
|
||||
result_csv2['possible_hobby'] = df_hob['possible_hobby']
|
||||
results_dict2 = dict(zip(result_csv2['user_id'], result_csv2['possible_hobby']))
|
||||
|
||||
# 保存模型和结果
|
||||
pickle.dump(results_dict1, open(interests_analysis_path + 'results/user_interests_dict.pkl', 'wb'))
|
||||
pickle.dump(results_dict2, open(interests_analysis_path + 'results/user_possible_hobby_dict.pkl', 'wb'))
|
||||
pickle.dump(cf_model, open(interests_analysis_path + 'results/user_possible_hobby_model.pkl', 'wb'))
|
||||
logger.info('用户爱好分析模型训练完成')
|
||||
|
||||
return results_dict1, results_dict2
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
train()
|
||||
|
|
@ -1 +0,0 @@
|
|||
## 这里面存放数据
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
user_id score created_at
|
||||
455865 0.0 2022-04-13 15:39:29
|
||||
455868 0.0 2022-04-13 15:39:29
|
||||
455865 0.0 2022-04-13 15:41:57
|
||||
455868 0.0 2022-04-13 15:41:57
|
||||
455864 0.0 2022-04-13 15:42:20
|
||||
455864 0.0 2022-04-13 15:42:20
|
||||
|
|
|
@ -1,40 +0,0 @@
|
|||
user_id final_score compelete_status created_at
|
||||
455864 0 2022-04-13 15:42:20
|
||||
455864 0 2022-04-13 15:42:20
|
||||
455864 0 2022-04-13 15:42:20
|
||||
455864 0 2022-04-13 15:42:20
|
||||
455864 0 2022-04-13 15:42:20
|
||||
455864 0 2022-04-13 15:42:20
|
||||
455864 0 2022-04-13 15:42:20
|
||||
455864 0 2022-04-13 15:42:20
|
||||
455864 0 2022-04-13 15:42:20
|
||||
455864 0 2022-04-13 15:42:20
|
||||
455864 0 2022-04-13 15:42:20
|
||||
455864 0 2022-04-13 15:42:20
|
||||
455864 0 2022-04-13 15:42:20
|
||||
455864 0 2022-04-13 15:42:20
|
||||
455864 0 2022-04-13 15:42:20
|
||||
455864 0 2022-04-13 15:42:20
|
||||
455864 0 2022-04-13 15:42:20
|
||||
455864 0 2022-04-13 15:42:20
|
||||
455864 0 2022-04-13 15:42:20
|
||||
455864 0 2022-04-13 15:42:20
|
||||
455864 0 2022-04-13 15:42:20
|
||||
455864 0 2022-04-13 15:42:20
|
||||
455864 0 2022-04-13 15:42:20
|
||||
455864 0 2022-04-13 15:42:20
|
||||
455864 0 2022-04-13 15:42:20
|
||||
455864 0 2022-04-13 15:42:20
|
||||
455864 0 2022-04-13 15:42:20
|
||||
455864 0 2022-04-13 15:42:20
|
||||
455864 0 2022-04-13 15:42:20
|
||||
455864 0 2022-04-13 15:42:20
|
||||
455864 0 2022-04-13 15:42:20
|
||||
455864 0 2022-04-13 15:42:20
|
||||
455864 0 2022-04-13 15:42:20
|
||||
455864 0 2022-04-13 15:42:20
|
||||
455864 0 2022-04-13 15:42:20
|
||||
455864 0 2022-04-13 15:42:20
|
||||
455864 0 2022-04-13 15:42:20
|
||||
455864 0 2022-04-13 15:42:20
|
||||
455864 0 2022-04-13 15:42:20
|
||||
|
|
|
@ -1 +0,0 @@
|
|||
user_id is_finished watch_duration
|
||||
|
|
|
@ -1,128 +0,0 @@
|
|||
import os
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from config import logger
|
||||
import datetime
|
||||
import pymysql
|
||||
from config import mysql_database, mysql_passwd
|
||||
from config import mysql_host, mysql_port, mysql_user
|
||||
from utils import get_before_date
|
||||
from config import learning_ability_analysis_path
|
||||
from config import data_before_days
|
||||
|
||||
def get_data_from_mysql(latest_data_date):
|
||||
"""
|
||||
从mysql数据库获取原始数据
|
||||
"""
|
||||
start = datetime.datetime.now()
|
||||
logger.info("开始获取用户学习数据...")
|
||||
|
||||
sql_conn = pymysql.connect(host=mysql_host,
|
||||
user=mysql_user,
|
||||
passwd=mysql_passwd,
|
||||
port=mysql_port,
|
||||
db=mysql_database)
|
||||
|
||||
sql_text1 = f"""
|
||||
SELECT user_id,is_finished,watch_duration
|
||||
FROM watch_course_videos
|
||||
"""
|
||||
|
||||
data_watch_course_videos = pd.read_sql(sql_text1, con=sql_conn)
|
||||
data_watch_course_videos.to_csv(learning_ability_analysis_path + 'data/watch_course_videos.csv', index=False, header=True, sep='\t')
|
||||
|
||||
logger.info("用户学习数据1下载完毕，共" + str(data_watch_course_videos.shape[0]) + "条数据，总耗时" + str((datetime.datetime.now() - start).seconds) + "秒")
|
||||
|
||||
sql_text2 = f"""
|
||||
SELECT user_id,final_score,compelete_status,created_at
|
||||
FROM student_works
|
||||
"""
|
||||
|
||||
data_student_works = pd.read_sql(sql_text2, con=sql_conn)
|
||||
data_student_works.to_csv(learning_ability_analysis_path + 'data/student_works.csv', index=False, header=True, sep='\t')
|
||||
|
||||
logger.info(
|
||||
"用户学习数据2下载完毕，共" + str(data_student_works.shape[0]) + "条数据，总耗时" + str((datetime.datetime.now() - start).seconds) + "秒")
|
||||
|
||||
sql_text3 = f"""
|
||||
SELECT user_id,score,created_at
|
||||
FROM exercise_users
|
||||
"""
|
||||
|
||||
data_exercise = pd.read_sql(sql_text3, con=sql_conn)
|
||||
data_exercise.to_csv(learning_ability_analysis_path + 'data/exercise_users.csv', index=False, header=True, sep='\t')
|
||||
|
||||
logger.info(
|
||||
"用户学习数据3下载完毕，共" + str(data_exercise.shape[0]) + "条数据，总耗时" + str((datetime.datetime.now() - start).seconds) + "秒")
|
||||
|
||||
def read_action_data():
|
||||
"""
|
||||
处理爱好数据
|
||||
"""
|
||||
data_watch_course_videos = pd.read_csv(learning_ability_analysis_path+ 'data/watch_course_videos.csv', sep='\t')
|
||||
data_watch_course_videos = data_watch_course_videos[['user_id','is_finished','watch_duration']]
|
||||
|
||||
data_student_works = pd.read_csv(learning_ability_analysis_path + 'data/student_works.csv', sep='\t')
|
||||
data_student_works = data_student_works[['user_id', 'final_score', 'compelete_status']]
|
||||
data_student_works = data_student_works.fillna(0)
|
||||
|
||||
data_exercise = pd.read_csv(learning_ability_analysis_path + 'data/exercise_users.csv', sep='\t')
|
||||
data_exercise = data_exercise[['user_id','score']]
|
||||
# 将所有没成绩的 NaN 值填充为 0
|
||||
data_exercise= data_exercise.fillna(0)
|
||||
data_exercise['user_id'] = data_exercise['user_id'].astype("float").astype("int")
|
||||
|
||||
return data_watch_course_videos,data_student_works,data_exercise
|
||||
|
||||
def get_learn_data():
|
||||
"""
|
||||
生成学习数据
|
||||
"""
|
||||
df_watch, df_work, df_exercise = read_action_data()
|
||||
|
||||
# 计算用户观看视频个数，完成观看视频任务个数，观看时常
|
||||
df_watch_cal = df_watch.groupby('user_id').agg({'is_finished': 'sum', 'user_id': 'count', 'watch_duration': 'sum'})
|
||||
df_watch_cal = df_watch_cal.rename(columns={'user_id': 'total'})
|
||||
|
||||
# 计算用户完成率，并将上述数据重建一个表保存
|
||||
df_watch_cal['is_finished'] = df_watch_cal['is_finished'] / df_watch_cal['total']
|
||||
df_watch_result = pd.DataFrame(list(df_watch_cal.index), index=range(len(list(df_watch_cal.index))),
|
||||
columns=['user_id'])
|
||||
df_watch_result['is_finished'] = list(df_watch_cal.is_finished.values.reshape(1, len(list(df_watch_cal.index)))[0])
|
||||
df_watch_result['total'] = list(df_watch_cal.total.values.reshape(1, len(list(df_watch_cal.index)))[0])
|
||||
df_watch_result['watch_duration'] = list(df_watch_cal.watch_duration.values.reshape(1, len(list(df_watch_cal.index)))[0])
|
||||
|
||||
# 计算用户作业个数，作业分数和实际完成作业个数
|
||||
df_work_cal = df_work.groupby('user_id').agg({'user_id': 'count', 'final_score': 'sum', 'compelete_status': 'sum'})
|
||||
df_work_cal = df_work_cal.rename(columns={'user_id': 'total_work'})
|
||||
|
||||
# 计算用户作业完成率，并将上述数据重建一个表进行保存
|
||||
df_work_cal['compelete_status'] = df_work_cal['compelete_status'] / df_work_cal['total_work']
|
||||
df_work_result = pd.DataFrame(list(df_work_cal.index), index=range(len(list(df_work_cal.index))),
|
||||
columns=['user_id'])
|
||||
df_work_result['compelete_status'] = list(
|
||||
df_work_cal.compelete_status.values.reshape(1, len(list(df_work_cal.index)))[0])
|
||||
df_work_result['total_work'] = list(df_work_cal.total_work.values.reshape(1, len(list(df_work_cal.index)))[0])
|
||||
df_work_result['final_score'] = list(df_work_cal.final_score.values.reshape(1, len(list(df_work_cal.index)))[0])
|
||||
|
||||
# 计算用户考试次数和得分，并将数据重建表保存
|
||||
df_exercise_cal = df_exercise.groupby('user_id').agg({'user_id': 'count', 'score': 'sum'})
|
||||
df_exercise_result = pd.DataFrame(list(df_exercise_cal.index), index=range(len(list(df_exercise_cal.index))),
|
||||
columns=['user_id'])
|
||||
df_exercise_result['score'] = list(df_exercise_cal.score.values.reshape(1, len(list(df_exercise_cal.index)))[0])
|
||||
df_exercise_result['user_id'] = df_exercise_result['user_id'].astype("float").astype("int")
|
||||
|
||||
# 将上述数据合并，计算实际完成的视频和作业数
|
||||
df_merge_1 = pd.merge(df_watch_result, df_work_result)
|
||||
df_merge_2 = pd.merge(df_exercise_result, df_merge_1)
|
||||
df_merge_2['watch'] = df_merge_2['is_finished'] * df_merge_2['total']
|
||||
df_merge_2['work'] = df_merge_2['compelete_status'] * df_merge_2['total_work']
|
||||
|
||||
data = df_merge_2[['user_id', 'watch', 'work', 'final_score', 'score']]
|
||||
return data
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
# 取数据
|
||||
latest_data_date = get_before_date(data_before_days)
|
||||
get_data_from_mysql(latest_data_date)
|
||||
|
|
@ -1,153 +0,0 @@
|
|||
import os
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from config import logger
|
||||
import datetime
|
||||
import pymysql
|
||||
from config import mysql_database, mysql_passwd
|
||||
from config import mysql_host, mysql_port, mysql_user
|
||||
from utils import get_before_date
|
||||
from config import learning_ability_analysis_path
|
||||
from config import data_before_days
|
||||
|
||||
def get_data_from_mysql(latest_data_date):
|
||||
"""
|
||||
从mysql数据库获取原始数据
|
||||
"""
|
||||
start = datetime.datetime.now()
|
||||
logger.info("开始获取用户学习数据...")
|
||||
|
||||
sql_conn = pymysql.connect(host=mysql_host,
|
||||
user=mysql_user,
|
||||
passwd=mysql_passwd,
|
||||
port=mysql_port,
|
||||
db=mysql_database)
|
||||
|
||||
sql_text1 = f"""
|
||||
SELECT
|
||||
t1.user_id,
|
||||
t1.is_finished,
|
||||
t1.watch_duration
|
||||
FROM
|
||||
watch_course_videos t1
|
||||
LEFT JOIN users t2 ON t1.user_id = t2.id
|
||||
where
|
||||
DATE_FORMAT(t2.last_login_on, '%Y-%m-%d') >= '{latest_data_date}'
|
||||
and DATE_FORMAT(t1.created_at, '%Y-%m-%d') >= '{latest_data_date}'
|
||||
"""
|
||||
|
||||
data_watch_course_videos = pd.read_sql(sql_text1, con=sql_conn)
|
||||
data_watch_course_videos.to_csv(learning_ability_analysis_path + 'data/watch_course_videos.csv', index=False, header=True, sep='\t')
|
||||
|
||||
logger.info("用户学习数据1下载完毕，共" + str(data_watch_course_videos.shape[0]) + "条数据，总耗时" + str((datetime.datetime.now() - start).seconds) + "秒")
|
||||
|
||||
sql_text2 = f"""
|
||||
SELECT
|
||||
t1.user_id,
|
||||
t1.final_score,
|
||||
t1.compelete_status,
|
||||
t1.created_at
|
||||
FROM
|
||||
student_works t1
|
||||
LEFT JOIN users t2 ON t1.user_id = t2.id
|
||||
where
|
||||
DATE_FORMAT(t2.last_login_on, '%Y-%m-%d') >= '{latest_data_date}'
|
||||
and DATE_FORMAT(t1.created_at, '%Y-%m-%d') >= '{latest_data_date}'
|
||||
"""
|
||||
|
||||
data_student_works = pd.read_sql(sql_text2, con=sql_conn)
|
||||
data_student_works.to_csv(learning_ability_analysis_path + 'data/student_works.csv', index=False, header=True, sep='\t')
|
||||
|
||||
logger.info(
|
||||
"用户学习数据2下载完毕，共" + str(data_student_works.shape[0]) + "条数据，总耗时" + str((datetime.datetime.now() - start).seconds) + "秒")
|
||||
|
||||
sql_text3 = f"""
|
||||
SELECT
|
||||
t1.user_id,
|
||||
t1.score,
|
||||
t1.created_at
|
||||
FROM
|
||||
exercise_users t1
|
||||
LEFT JOIN users t2 ON t1.user_id = t2.id
|
||||
where
|
||||
DATE_FORMAT(t2.last_login_on, '%Y-%m-%d') >= '{latest_data_date}'
|
||||
and DATE_FORMAT(t1.created_at, '%Y-%m-%d') >= '{latest_data_date}'
|
||||
"""
|
||||
|
||||
data_exercise = pd.read_sql(sql_text3, con=sql_conn)
|
||||
data_exercise.to_csv(learning_ability_analysis_path + 'data/exercise_users.csv', index=False, header=True, sep='\t')
|
||||
|
||||
logger.info(
|
||||
"用户学习数据3下载完毕，共" + str(data_exercise.shape[0]) + "条数据，总耗时" + str((datetime.datetime.now() - start).seconds) + "秒")
|
||||
|
||||
def read_action_data():
|
||||
"""
|
||||
处理爱好数据
|
||||
"""
|
||||
data_watch_course_videos = pd.read_csv(learning_ability_analysis_path+ 'data/watch_course_videos.csv', sep='\t')
|
||||
data_watch_course_videos = data_watch_course_videos[['user_id','is_finished','watch_duration']]
|
||||
|
||||
data_student_works = pd.read_csv(learning_ability_analysis_path + 'data/student_works.csv', sep='\t')
|
||||
data_student_works = data_student_works[['user_id', 'final_score', 'compelete_status']]
|
||||
data_student_works = data_student_works.fillna(0)
|
||||
|
||||
data_exercise = pd.read_csv(learning_ability_analysis_path + 'data/exercise_users.csv', sep='\t')
|
||||
data_exercise = data_exercise[['user_id','score']]
|
||||
# 将所有没成绩的 NaN 值填充为 0
|
||||
data_exercise= data_exercise.fillna(0)
|
||||
data_exercise['user_id'] = data_exercise['user_id'].astype("float").astype("int")
|
||||
|
||||
return data_watch_course_videos,data_student_works,data_exercise
|
||||
|
||||
def get_learn_data():
|
||||
"""
|
||||
生成学习数据
|
||||
"""
|
||||
df_watch, df_work, df_exercise = read_action_data()
|
||||
|
||||
# 计算用户观看视频个数，完成观看视频任务个数，观看时常
|
||||
df_watch_cal = df_watch.groupby('user_id').agg({'is_finished': 'sum', 'user_id': 'count', 'watch_duration': 'sum'})
|
||||
df_watch_cal = df_watch_cal.rename(columns={'user_id': 'total'})
|
||||
|
||||
# 计算用户完成率，并将上述数据重建一个表保存
|
||||
df_watch_cal['is_finished'] = df_watch_cal['is_finished'] / df_watch_cal['total']
|
||||
df_watch_result = pd.DataFrame(list(df_watch_cal.index), index=range(len(list(df_watch_cal.index))),
|
||||
columns=['user_id'])
|
||||
df_watch_result['is_finished'] = list(df_watch_cal.is_finished.values.reshape(1, len(list(df_watch_cal.index)))[0])
|
||||
df_watch_result['total'] = list(df_watch_cal.total.values.reshape(1, len(list(df_watch_cal.index)))[0])
|
||||
df_watch_result['watch_duration'] = list(df_watch_cal.watch_duration.values.reshape(1, len(list(df_watch_cal.index)))[0])
|
||||
|
||||
# 计算用户作业个数，作业分数和实际完成作业个数
|
||||
df_work_cal = df_work.groupby('user_id').agg({'user_id': 'count', 'final_score': 'sum', 'compelete_status': 'sum'})
|
||||
df_work_cal = df_work_cal.rename(columns={'user_id': 'total_work'})
|
||||
|
||||
# 计算用户作业完成率，并将上述数据重建一个表进行保存
|
||||
df_work_cal['compelete_status'] = df_work_cal['compelete_status'] / df_work_cal['total_work']
|
||||
df_work_result = pd.DataFrame(list(df_work_cal.index), index=range(len(list(df_work_cal.index))),
|
||||
columns=['user_id'])
|
||||
df_work_result['compelete_status'] = list(
|
||||
df_work_cal.compelete_status.values.reshape(1, len(list(df_work_cal.index)))[0])
|
||||
df_work_result['total_work'] = list(df_work_cal.total_work.values.reshape(1, len(list(df_work_cal.index)))[0])
|
||||
df_work_result['final_score'] = list(df_work_cal.final_score.values.reshape(1, len(list(df_work_cal.index)))[0])
|
||||
|
||||
# 计算用户考试次数和得分，并将数据重建表保存
|
||||
df_exercise_cal = df_exercise.groupby('user_id').agg({'user_id': 'count', 'score': 'sum'})
|
||||
df_exercise_result = pd.DataFrame(list(df_exercise_cal.index), index=range(len(list(df_exercise_cal.index))),
|
||||
columns=['user_id'])
|
||||
df_exercise_result['score'] = list(df_exercise_cal.score.values.reshape(1, len(list(df_exercise_cal.index)))[0])
|
||||
df_exercise_result['user_id'] = df_exercise_result['user_id'].astype("float").astype("int")
|
||||
|
||||
# 将上述数据合并，计算实际完成的视频和作业数
|
||||
df_merge_1 = pd.merge(df_watch_result, df_work_result)
|
||||
df_merge_2 = pd.merge(df_exercise_result, df_merge_1)
|
||||
df_merge_2['watch'] = df_merge_2['is_finished'] * df_merge_2['total']
|
||||
df_merge_2['work'] = df_merge_2['compelete_status'] * df_merge_2['total_work']
|
||||
|
||||
data = df_merge_2[['user_id', 'watch', 'work', 'final_score', 'score']]
|
||||
return data
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
# 取数据
|
||||
latest_data_date = get_before_date(data_before_days)
|
||||
get_data_from_mysql(latest_data_date)
|
||||
Binary file not shown.
Binary file not shown.
|
|
@ -1,22 +0,0 @@
|
|||
import pickle
|
||||
from config import logger
|
||||
from config import test_user_id
|
||||
from config import learning_ability_analysis_path
|
||||
|
||||
logger.info('加载用户学习能力字典')
|
||||
user_learning_ability_dict = pickle.load(open(learning_ability_analysis_path + 'results/user_learning_ability_dict.pkl', 'rb'))
|
||||
|
||||
def user_learning_ability(user_id):
|
||||
"""
|
||||
用户学习能力预测
|
||||
"""
|
||||
if user_id not in user_learning_ability_dict:
|
||||
result = -1
|
||||
else:
|
||||
result = user_learning_ability_dict[user_id]
|
||||
|
||||
return result
|
||||
|
||||
if __name__ == '__main__':
|
||||
result = user_learning_ability(user_id=test_user_id)
|
||||
print('用户ID：', test_user_id, '学习能力：', result)
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
## 这里面存放模型训练结果
|
||||
|
||||
|
|
@ -1,72 +0,0 @@
|
|||
import os
|
||||
import pandas as pd
|
||||
import pickle
|
||||
from sklearn.cluster import KMeans
|
||||
from config import logger
|
||||
from config import RANDOM_SEED
|
||||
from data_process import get_learn_data
|
||||
from config import learning_ability_analysis_path
|
||||
|
||||
def iflabel(x):
|
||||
if x == 4:
|
||||
return 5
|
||||
elif x == 3:
|
||||
return 4
|
||||
elif x == 2:
|
||||
return 3
|
||||
elif x == 1:
|
||||
return 2
|
||||
elif x == 0:
|
||||
return 1
|
||||
def train():
|
||||
"""
|
||||
用户学习能力模型训练
|
||||
"""
|
||||
logger.info
|
||||
logger.info('开始训练用户学习能力分析模型...')
|
||||
|
||||
data = get_learn_data()
|
||||
# 根据观看视频个数，作业，作业分数，考试分数进行聚类
|
||||
df_cluster = data[['user_id', 'watch', 'work', 'final_score', 'score']]
|
||||
kmeans_model = KMeans(n_clusters=8, random_state=RANDOM_SEED).fit(df_cluster)
|
||||
df_cluster["label"] = kmeans_model.labels_
|
||||
df_mean = df_cluster[['watch', 'work', 'final_score', 'score', 'label']].groupby("label").agg("mean")
|
||||
|
||||
# 这里使用先计算均值再计算分数
|
||||
# 将每一个指标计算均值，超多均值得1分，未超过得0分
|
||||
df_mean["IFWA"] = df_mean["watch"].apply(lambda x: 1 if x > df_mean["watch"].mean() else 0)
|
||||
df_mean["IFWO"] = df_mean["work"].apply(lambda x: 1 if x > df_mean["work"].mean() else 0)
|
||||
df_mean["IFF"] = df_mean["final_score"].apply(lambda x: 1 if x > df_mean["final_score"].mean() else 0)
|
||||
df_mean["IFS"] = df_mean["score"].apply(lambda x: 1 if x > df_mean["score"].mean() else 0)
|
||||
|
||||
# 将各项指标分数求和汇总然后根据打分函数进行打分
|
||||
df_mean["temp"] = df_mean["IFWA"] + df_mean["IFWO"] + df_mean["IFF"] + df_mean["IFS"]
|
||||
df_mean["label"] = df_mean["temp"].apply(lambda x: iflabel(x))
|
||||
|
||||
# 这里使用聚类在进行打分
|
||||
# 和上面进行相同的操作，最后返回每个用户的学习能力分数
|
||||
df_cluster["IFWA"] = df_cluster["watch"].apply(lambda x: 1 if x > df_mean["watch"].mean() else 0)
|
||||
df_cluster["IFWO"] = df_cluster["work"].apply(lambda x: 1 if x > df_mean["work"].mean() else 0)
|
||||
df_cluster["IFF"] = df_cluster["final_score"].apply(lambda x: 1 if x > df_mean["final_score"].mean() else 0)
|
||||
df_cluster["IFS"] = df_cluster["score"].apply(lambda x: 1 if x > df_mean["score"].mean() else 0)
|
||||
df_cluster["temp"] = df_cluster["IFWA"] + df_cluster["IFWO"] + df_cluster["IFF"] + df_cluster["IFS"]
|
||||
df_cluster["label"] = df_cluster["temp"].apply(lambda x: iflabel(x))
|
||||
|
||||
# 将用户 id 和学习能力提取出来保存
|
||||
result = df_cluster[['label']]
|
||||
result_csv = pd.DataFrame(list(result.index), index=range(len(list(result.index))), columns=['user_id'])
|
||||
result_csv['learning_ability'] = list(result.values.reshape(1, len(list(result.index)))[0])
|
||||
result_csv['user_id'] = result_csv['user_id'].astype(int)
|
||||
result_csv['learning_ability'] = result_csv['learning_ability'].astype(int)
|
||||
results_dict = dict(zip(result_csv['user_id'], result_csv['learning_ability']))
|
||||
|
||||
# 保存模型和结果
|
||||
pickle.dump(kmeans_model, open(learning_ability_analysis_path + 'results/user_learning_ability_model.pkl', 'wb'))
|
||||
pickle.dump(results_dict, open(learning_ability_analysis_path + 'results/user_learning_ability_dict.pkl', 'wb'))
|
||||
|
||||
logger.info('用户学习能力分析模型训练完成')
|
||||
|
||||
return results_dict
|
||||
|
||||
if __name__ == '__main__':
|
||||
train()
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
## 这里面存放的是数据
|
||||
|
||||
|
|
@ -1,146 +0,0 @@
|
|||
import os
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from config import logger
|
||||
import datetime
|
||||
import pymysql
|
||||
from utils import get_before_date
|
||||
from config import mysql_database, mysql_passwd
|
||||
from config import mysql_host, mysql_port, mysql_user
|
||||
from config import professional_ability_analysis_path
|
||||
from config import data_before_days
|
||||
|
||||
|
||||
def get_data_from_mysql(latest_data_date):
|
||||
"""
|
||||
从mysql数据库获取原始数据
|
||||
"""
|
||||
start = datetime.datetime.now()
|
||||
logger.info("开始获取用户专业知识好数据...")
|
||||
|
||||
sql_conn = pymysql.connect(host=mysql_host,
|
||||
user=mysql_user,
|
||||
passwd=mysql_passwd,
|
||||
port=mysql_port,
|
||||
db=mysql_database)
|
||||
|
||||
sql_text1 = f"""
|
||||
SELECT id,name
|
||||
FROM repertoires
|
||||
"""
|
||||
|
||||
data_repertoires = pd.read_sql(sql_text1, con=sql_conn)
|
||||
data_repertoires.to_csv(professional_ability_analysis_path + 'data/repertoires.csv', index=False, header=True, sep='\t')
|
||||
|
||||
logger.info("用户专业知识能力数据1数据下载完毕，共" + str(data_repertoires.shape[0]) + "条数据，总耗时" + str((datetime.datetime.now() - start).seconds) + "秒")
|
||||
|
||||
sql_text2 = f"""
|
||||
SELECT id,shixun_id,user_id,score
|
||||
FROM challenges
|
||||
"""
|
||||
|
||||
data_challenges = pd.read_sql(sql_text2, con=sql_conn)
|
||||
data_challenges.to_csv(professional_ability_analysis_path + 'data/challenges.csv', index=False, header=True, sep='\t')
|
||||
|
||||
logger.info(
|
||||
"用户专业知识能力数据2下载完毕，共" + str(data_challenges.shape[0]) + "条数据，总耗时" + str((datetime.datetime.now() - start).seconds) + "秒")
|
||||
|
||||
sql_text3 = f"""
|
||||
SELECT shixun_id,tag_repertoire_id
|
||||
FROM shixun_tag_repertoires
|
||||
"""
|
||||
|
||||
data_shixun_tag_repertoires = pd.read_sql(sql_text3, con=sql_conn)
|
||||
data_shixun_tag_repertoires.to_csv(professional_ability_analysis_path + 'data/shixun_tag_repertoires.csv', index=False, header=True, sep='\t')
|
||||
|
||||
logger.info(
|
||||
"用户专业知识能力数据3下载完毕，共" + str(data_shixun_tag_repertoires.shape[0]) + "条数据，总耗时" + str((datetime.datetime.now() - start).seconds) + "秒")
|
||||
|
||||
sql_text4 = f"""
|
||||
SELECT id,sub_repertoire_id
|
||||
FROM tag_repertoires
|
||||
"""
|
||||
|
||||
data_tag_repertoires = pd.read_sql(sql_text4, con=sql_conn)
|
||||
data_tag_repertoires.to_csv(professional_ability_analysis_path + 'data/tag_repertoires.csv', index=False, header=True, sep='\t')
|
||||
|
||||
logger.info(
|
||||
"用户专业知识能力数据4下载完毕，共" + str(data_tag_repertoires.shape[0]) + "条数据，总耗时" + str((datetime.datetime.now() - start).seconds) + "秒")
|
||||
|
||||
sql_text5 = f"""
|
||||
SELECT id,repertoire_id
|
||||
FROM sub_repertoires
|
||||
"""
|
||||
|
||||
data_sub_repertoires = pd.read_sql(sql_text5, con=sql_conn)
|
||||
data_sub_repertoires.to_csv(professional_ability_analysis_path + 'data/sub_repertoires.csv', index=False, header=True, sep='\t')
|
||||
|
||||
logger.info(
|
||||
"用户专业知识能力数据5下载完毕，共" + str(data_sub_repertoires.shape[0]) + "条数据，总耗时" + str((datetime.datetime.now() - start).seconds) + "秒")
|
||||
|
||||
|
||||
def read_action_data():
|
||||
"""
|
||||
处理用户专业知识能力数据
|
||||
"""
|
||||
data_repertoires= pd.read_csv(professional_ability_analysis_path + 'data/repertoires.csv', sep='\t')
|
||||
data_repertoires = data_repertoires[['id', 'name']]
|
||||
data_repertoires = data_repertoires.rename(columns={'id': 'repertoire_id'})
|
||||
|
||||
data_challenges = pd.read_csv(professional_ability_analysis_path + 'data/challenges.csv', sep='\t')
|
||||
data_challenges = data_challenges[['user_id', 'id', 'shixun_id', 'score']]
|
||||
data_challenges = data_challenges.rename(columns={'id': 'challenge_id'})
|
||||
data_challenges = data_challenges[['user_id', 'shixun_id', 'score']]
|
||||
|
||||
data_shixun_tag_repertoires = pd.read_csv(professional_ability_analysis_path + 'data/shixun_tag_repertoires.csv', sep='\t')
|
||||
data_shixun_tag_repertoires = data_shixun_tag_repertoires[['shixun_id', 'tag_repertoire_id']]
|
||||
data_shixun_tag_repertoires = data_shixun_tag_repertoires.dropna()
|
||||
|
||||
data_tag_repertoires = pd.read_csv(professional_ability_analysis_path + 'data/tag_repertoires.csv', sep='\t')
|
||||
data_tag_repertoires = data_tag_repertoires[['id', 'sub_repertoire_id']]
|
||||
data_tag_repertoires = data_tag_repertoires.rename(columns={'id': 'tag_repertoire_id'})
|
||||
|
||||
data_sub_repertoires = pd.read_csv(professional_ability_analysis_path + 'data/sub_repertoires.csv', sep='\t')
|
||||
data_sub_repertoires = data_sub_repertoires[['id', 'repertoire_id']]
|
||||
data_sub_repertoires = data_sub_repertoires.rename(columns={'id': 'sub_repertoire_id'})
|
||||
|
||||
return data_repertoires,data_challenges,data_shixun_tag_repertoires,data_tag_repertoires,data_sub_repertoires
|
||||
|
||||
|
||||
def get_profession_data():
|
||||
"""
|
||||
生成用户专业知识能力数据
|
||||
"""
|
||||
# 提取用户实训进行搜索，将用户所做实训的大类挑选出来
|
||||
data_repertoires, data_challenges, data_shixun_tag_repertoires, data_tag_repertoires, data_sub_repertoires = read_action_data()
|
||||
data_merge_1 = pd.merge(data_shixun_tag_repertoires, data_tag_repertoires, on='tag_repertoire_id')
|
||||
data_merge_1 = data_merge_1[['shixun_id', 'sub_repertoire_id']]
|
||||
data_merge_2 = pd.merge(data_merge_1, data_sub_repertoires, on='sub_repertoire_id')
|
||||
data_merge_2 = data_merge_2[['shixun_id', 'repertoire_id']]
|
||||
data_merge_3 = pd.merge(data_challenges, data_merge_2, on='shixun_id')
|
||||
data_merge_3 = data_merge_3[['user_id', 'score', 'repertoire_id']]
|
||||
cols1 = list(set(list(data_merge_3.repertoire_id.values)))
|
||||
cols2 = list(data_repertoires.repertoire_id.values)
|
||||
cols3 = list(set(cols1).union(set(cols2)))
|
||||
data = data_merge_3.pivot_table(index='user_id', columns='repertoire_id', values='score', aggfunc='sum').fillna(0)
|
||||
|
||||
# 根据用户在每个专业类别的得分给用户在每个专业的能力进行打分
|
||||
for col in list(data.columns):
|
||||
data.sort_values(by=col, ascending=False)
|
||||
data[col] = data[col].rank(ascending=1, method='first')
|
||||
data[col] = data[col] / data[col].max()
|
||||
q = pd.cut(data[col], 10, labels=[1, 2, 3, 4, 5, 6, 7, 8, 9, 10])
|
||||
data[col] = q
|
||||
|
||||
# 数据库更新时，专业表的名称会增删，此处将删掉的专业列删掉，新增的专业加上去
|
||||
for i in range(len(cols1)):
|
||||
cols3.remove(cols1[i])
|
||||
for i in cols3:
|
||||
data[i] = 0
|
||||
return data, data_repertoires
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
# 取数据
|
||||
latest_data_date = get_before_date(data_before_days)
|
||||
get_data_from_mysql(latest_data_date)
|
||||
Binary file not shown.
Binary file not shown.
|
|
@ -1,23 +0,0 @@
|
|||
import pickle
|
||||
from config import logger
|
||||
from config import test_user_id
|
||||
from config import activity_analysis_path
|
||||
from config import professional_ability_analysis_path
|
||||
|
||||
logger.info('加载用户专业能力字典')
|
||||
user_professional_ability_dict = pickle.load(open(professional_ability_analysis_path + 'results/user_professional_ability_dict.pkl', 'rb'))
|
||||
|
||||
def user_professional_ability_predict(user_id):
|
||||
"""
|
||||
用户专业能力预测
|
||||
"""
|
||||
if user_id not in user_professional_ability_dict:
|
||||
result = -1
|
||||
else:
|
||||
result = user_professional_ability_dict[user_id]
|
||||
|
||||
return result
|
||||
|
||||
if __name__ == '__main__':
|
||||
result = user_professional_ability_predict(user_id=test_user_id)
|
||||
print('用户ID：', test_user_id, '专业编程能力：', result)
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
## 这里面存放模型训练结果
|
||||
|
||||
|
|
@ -1,88 +0,0 @@
|
|||
import os
|
||||
import pandas as pd
|
||||
import pickle
|
||||
from sklearn.cluster import KMeans
|
||||
from config import logger
|
||||
from config import RANDOM_SEED
|
||||
from data_process import get_profession_data
|
||||
from config import professional_ability_analysis_path
|
||||
def iflabel(x):
|
||||
if x == 8 :
|
||||
return 5
|
||||
elif x == 7 :
|
||||
return 4
|
||||
elif x == 6:
|
||||
return 4
|
||||
elif x == 5:
|
||||
return 3
|
||||
elif x == 4:
|
||||
return 3
|
||||
elif x == 3:
|
||||
return 2
|
||||
elif x == 2:
|
||||
return 2
|
||||
elif x == 1:
|
||||
return 1
|
||||
elif x == 0:
|
||||
return 1
|
||||
def train():
|
||||
"""
|
||||
用户专业知识能力分析模型训练
|
||||
"""
|
||||
logger.info('开始训练用户专业知识能力分析模型...')
|
||||
|
||||
data, data_repertoires = get_profession_data()
|
||||
cols1 = list(data_repertoires.repertoire_id.values)
|
||||
|
||||
# 根据用户在每个专业的得分作为特征进行聚类
|
||||
df_cluster = data[cols1]
|
||||
kmeans_model = KMeans(n_clusters=8, random_state=RANDOM_SEED).fit(df_cluster)
|
||||
df_cluster["label"] = kmeans_model.labels_
|
||||
|
||||
# 重建一个表
|
||||
user_index = list(df_cluster.index)
|
||||
cols2 = [str(x) for x in cols1]
|
||||
cols3 = cols2.copy()
|
||||
cols3.append("label")
|
||||
df_cluster = pd.DataFrame(df_cluster.values, index=range(len(list(df_cluster.index))), columns=cols3)
|
||||
df_cluster['user_id'] = user_index
|
||||
|
||||
# 计算每类均值，并将用户每类的值和均值进行比较，大于均值得1分，小于均值得0分，然后根据打分函数进行打分
|
||||
df_mean = df_cluster[cols3].groupby("label").agg("mean")
|
||||
for num in cols2:
|
||||
df_mean["IF" + num] = df_mean[num].apply(lambda x: 1 if x > df_mean[num].mean() else 0)
|
||||
|
||||
df_mean["temp"] = 0
|
||||
for num in cols2:
|
||||
df_mean["temp"] += df_mean["IF" + num]
|
||||
df_mean["label"] = df_mean["temp"].apply(lambda x: iflabel(x))
|
||||
|
||||
for num in cols2:
|
||||
df_cluster["IF" + num] = df_cluster[num].apply(lambda x: 1 if x > df_mean[num].mean() else 0)
|
||||
|
||||
df_cluster["temp"] = 0
|
||||
for num in cols2:
|
||||
df_cluster["temp"] += df_cluster["IF" + num]
|
||||
|
||||
# 将用户 id 和专业能力得分提取出来保存
|
||||
df_cluster["label"] = df_cluster["temp"].apply(lambda x: iflabel(x))
|
||||
df_cluster = df_cluster.rename(columns={'label': 'professional_ability'})
|
||||
result_csv = df_cluster[['user_id', 'professional_ability']]
|
||||
|
||||
result_csv['user_id'] = result_csv['user_id'].astype(int)
|
||||
result_csv['professional_ability'] = result_csv['professional_ability'].astype(int)
|
||||
results_dict = dict(zip(result_csv['user_id'], result_csv['professional_ability']))
|
||||
|
||||
# 保存模型和结果
|
||||
pickle.dump(kmeans_model,
|
||||
open(professional_ability_analysis_path + 'results/user_professional_ability_model.pkl', 'wb'))
|
||||
pickle.dump(results_dict,
|
||||
open(professional_ability_analysis_path + 'results/user_professional_ability_dict.pkl', 'wb'))
|
||||
|
||||
logger.info('用户专业知识能力分析模型训练完成')
|
||||
|
||||
return results_dict
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
train()
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
## 这里面存放的是数据
|
||||
|
||||
|
|
@ -1,95 +0,0 @@
|
|||
import os
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from config import logger
|
||||
import datetime
|
||||
import pymysql
|
||||
from config import mysql_database, mysql_passwd
|
||||
from config import mysql_host, mysql_port, mysql_user
|
||||
from utils import get_before_date
|
||||
from config import programming_ability_analysis_path
|
||||
from config import data_before_days
|
||||
|
||||
def get_data_from_mysql(latest_data_date):
|
||||
"""
|
||||
从mysql数据库获取原始数据
|
||||
"""
|
||||
start = datetime.datetime.now()
|
||||
logger.info("开始获取用户标签数据...")
|
||||
|
||||
sql_conn = pymysql.connect(host=mysql_host,
|
||||
user=mysql_user,
|
||||
passwd=mysql_passwd,
|
||||
port=mysql_port,
|
||||
db=mysql_database)
|
||||
|
||||
sql_text1 = f"""
|
||||
SELECT id, user_id, language
|
||||
FROM shixuns
|
||||
"""
|
||||
|
||||
data_shixuns = pd.read_sql(sql_text1, con=sql_conn)
|
||||
data_shixuns.to_csv(programming_ability_analysis_path + 'data/shixuns.csv', index=False, header=True, sep='\t')
|
||||
logger.info("用户编程数据下载完毕，共" + str(data_shixuns.shape[0]) + "条数据，总耗时" + str((datetime.datetime.now() - start).seconds) + "秒")
|
||||
|
||||
def read_action_data():
|
||||
"""
|
||||
处理用户编程数据
|
||||
"""
|
||||
data_shixuns = pd.read_csv(programming_ability_analysis_path + 'data/shixuns.csv', sep='\t')
|
||||
data_shixuns = data_shixuns[['id', 'user_id', 'language']]
|
||||
data_shixuns = data_shixuns.fillna({'language': 'Other'})
|
||||
data_shixuns = data_shixuns.dropna()
|
||||
data_shixuns["user_id"] = data_shixuns["user_id"].astype("float").astype("int")
|
||||
map_list = {
|
||||
'Java': 'Java',
|
||||
'MySQL/Java': 'MySQL',
|
||||
'Python': 'Python',
|
||||
'C++': 'C/C++',
|
||||
'Python3.6': 'Python',
|
||||
'JFinal': 'JFinal',
|
||||
'Python2.7': 'Python',
|
||||
'Dynamips': 'Dynamips',
|
||||
'Ethereum': 'Ethereum',
|
||||
'Html': 'Html',
|
||||
'MachineLearning': 'MachineLearning',
|
||||
'Docker': 'Docker',
|
||||
'C': 'C/C++',
|
||||
'MySQL/Python3.6': 'MySQL',
|
||||
'Verilog': 'Verilog',
|
||||
'PHP/Web': 'PHP/Web',
|
||||
'Android': 'Android',
|
||||
'Golang': 'Golang',
|
||||
'Hadoop': 'Hadoop',
|
||||
'Matlab': 'Matlab',
|
||||
'Shell': 'Shell',
|
||||
'Git': 'Git',
|
||||
'Ruby': 'Ruby',
|
||||
'Perl6': 'Perl6',
|
||||
'Kotlin': 'Kotlin',
|
||||
'JavaScript': 'JavaScript',
|
||||
'Other': 'Other'
|
||||
}
|
||||
data_shixuns['language'] = data_shixuns['language'].map(map_list)
|
||||
return data_shixuns
|
||||
|
||||
|
||||
def get_program_data():
|
||||
"""
|
||||
生成用户编程训练数据
|
||||
"""
|
||||
data = read_action_data()
|
||||
data = data[['user_id', 'language']]
|
||||
# 计算用户在每一类语言实训的总数
|
||||
data['value'] = 1
|
||||
data = data.pivot_table(index='user_id', columns='language', values='value', aggfunc='count').fillna(0)
|
||||
data = data.astype('int')
|
||||
# 计算用户每类语言的使用占比
|
||||
data = data.div(data.sum(axis=0), axis=1)
|
||||
return data
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
# 取数据
|
||||
latest_data_date = get_before_date(data_before_days)
|
||||
get_data_from_mysql(latest_data_date)
|
||||
|
|
@ -1,104 +0,0 @@
|
|||
import os
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from config import logger
|
||||
import datetime
|
||||
import pymysql
|
||||
from config import mysql_database, mysql_passwd
|
||||
from config import mysql_host, mysql_port, mysql_user
|
||||
from utils import get_before_date
|
||||
from config import programming_ability_analysis_path
|
||||
from config import data_before_days
|
||||
|
||||
def get_data_from_mysql(latest_data_date):
|
||||
"""
|
||||
从mysql数据库获取原始数据
|
||||
"""
|
||||
start = datetime.datetime.now()
|
||||
logger.info("开始获取用户标签数据...")
|
||||
|
||||
sql_conn = pymysql.connect(host=mysql_host,
|
||||
user=mysql_user,
|
||||
passwd=mysql_passwd,
|
||||
port=mysql_port,
|
||||
db=mysql_database)
|
||||
|
||||
sql_text1 = f"""
|
||||
SELECT
|
||||
t1.id,
|
||||
t1.user_id,
|
||||
t2.language
|
||||
FROM
|
||||
myshixuns t1
|
||||
LEFT JOIN shixuns t2 ON t1.shixun_id = t2.id
|
||||
LEFT JOIN users t3 ON t1.user_id = t3.id
|
||||
WHERE
|
||||
DATE_FORMAT(t3.last_login_on, '%Y-%m-%d') >= '{latest_data_date}'
|
||||
AND DATE_FORMAT(t1.created_at, '%Y-%m-%d') >= '{latest_data_date}'
|
||||
"""
|
||||
|
||||
data_shixuns = pd.read_sql(sql_text1, con=sql_conn)
|
||||
data_shixuns.to_csv(programming_ability_analysis_path + 'data/shixuns.csv', index=False, header=True, sep='\t')
|
||||
logger.info("用户编程数据下载完毕，共" + str(data_shixuns.shape[0]) + "条数据，总耗时" + str((datetime.datetime.now() - start).seconds) + "秒")
|
||||
|
||||
def read_action_data():
|
||||
"""
|
||||
处理用户编程数据
|
||||
"""
|
||||
data_shixuns = pd.read_csv(programming_ability_analysis_path + 'data/shixuns.csv', sep='\t')
|
||||
data_shixuns = data_shixuns[['id', 'user_id', 'language']]
|
||||
data_shixuns = data_shixuns.fillna({'language': 'Other'})
|
||||
data_shixuns = data_shixuns.dropna()
|
||||
data_shixuns["user_id"] = data_shixuns["user_id"].astype("float").astype("int")
|
||||
map_list = {
|
||||
'Java': 'Java',
|
||||
'MySQL/Java': 'MySQL',
|
||||
'Python': 'Python',
|
||||
'C++': 'C/C++',
|
||||
'Python3.6': 'Python',
|
||||
'JFinal': 'JFinal',
|
||||
'Python2.7': 'Python',
|
||||
'Dynamips': 'Dynamips',
|
||||
'Ethereum': 'Ethereum',
|
||||
'Html': 'Html',
|
||||
'MachineLearning': 'MachineLearning',
|
||||
'Docker': 'Docker',
|
||||
'C': 'C/C++',
|
||||
'MySQL/Python3.6': 'MySQL',
|
||||
'Verilog': 'Verilog',
|
||||
'PHP/Web': 'PHP/Web',
|
||||
'Android': 'Android',
|
||||
'Golang': 'Golang',
|
||||
'Hadoop': 'Hadoop',
|
||||
'Matlab': 'Matlab',
|
||||
'Shell': 'Shell',
|
||||
'Git': 'Git',
|
||||
'Ruby': 'Ruby',
|
||||
'Perl6': 'Perl6',
|
||||
'Kotlin': 'Kotlin',
|
||||
'JavaScript': 'JavaScript',
|
||||
'Other': 'Other'
|
||||
}
|
||||
data_shixuns['language'] = data_shixuns['language'].map(map_list)
|
||||
return data_shixuns
|
||||
|
||||
|
||||
def get_program_data():
|
||||
"""
|
||||
生成用户编程训练数据
|
||||
"""
|
||||
data = read_action_data()
|
||||
data = data[['user_id', 'language']]
|
||||
# 计算用户在每一类语言实训的总数
|
||||
data['value'] = 1
|
||||
data = data.pivot_table(index='user_id', columns='language', values='value', aggfunc='count').fillna(0)
|
||||
data = data.astype('int')
|
||||
# 计算用户每类语言的使用占比
|
||||
data = data.div(data.sum(axis=0), axis=1)
|
||||
return data
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
# 取数据
|
||||
latest_data_date = get_before_date(data_before_days)
|
||||
get_data_from_mysql(latest_data_date)
|
||||
Binary file not shown.
Binary file not shown.
|
|
@ -1,22 +0,0 @@
|
|||
import pickle
|
||||
from config import logger
|
||||
from config import test_user_id
|
||||
from config import programming_ability_analysis_path
|
||||
|
||||
logger.info('加载用户用户编程能力字典')
|
||||
user_programming_ability_dict = pickle.load(open(programming_ability_analysis_path + 'results/user_programming_ability_dict.pkl', 'rb'))
|
||||
|
||||
def user_programming_ability_predict(user_id):
|
||||
"""
|
||||
用户编程能力预测
|
||||
"""
|
||||
if user_id not in user_programming_ability_dict:
|
||||
result = -1
|
||||
else:
|
||||
result = user_programming_ability_dict[user_id]
|
||||
|
||||
return result
|
||||
|
||||
if __name__ == '__main__':
|
||||
result = user_programming_ability_predict(user_id=test_user_id)
|
||||
print('用户ID：', test_user_id, '用户编程能力：', result)
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
## 这里面存放模型训练结果
|
||||
|
||||
|
|
@ -1,82 +0,0 @@
|
|||
import os
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import pickle
|
||||
from sklearn.decomposition import PCA
|
||||
from sklearn.cluster import KMeans
|
||||
from config import logger
|
||||
from config import RANDOM_SEED
|
||||
from data_process import get_program_data
|
||||
from config import programming_ability_analysis_path
|
||||
def iflabel(x):
|
||||
if x == "高高高":
|
||||
return 5
|
||||
elif x == "高低高":
|
||||
return 4
|
||||
elif x == "高高低":
|
||||
return 4
|
||||
elif x == "高低低":
|
||||
return 3
|
||||
elif x == "低高高":
|
||||
return 4
|
||||
elif x == "低低高":
|
||||
return 2
|
||||
elif x == "低高低":
|
||||
return 2
|
||||
elif x == "低低低":
|
||||
return 1
|
||||
def train():
|
||||
"""
|
||||
用户编程数据分析模型训练
|
||||
"""
|
||||
logger.info('开始训练用编程能力分析模型...')
|
||||
|
||||
data = get_program_data()
|
||||
|
||||
# 跟据PCA算法计算出影响力最大的三种语言
|
||||
pca_model = PCA(n_components=3)
|
||||
pca_data = pca_model.fit_transform(data)
|
||||
new_data = pd.DataFrame(np.array(pca_data), index=range(len(list(pca_data))), columns=['1', '2', '3'])
|
||||
new_data['user_id'] = list(new_data.index)
|
||||
|
||||
# 根据这三种语言给用户进行聚类
|
||||
df_cluster = new_data[['user_id', '1', '2', '3']]
|
||||
kmeans_model = KMeans(n_clusters=8, random_state=RANDOM_SEED).fit(df_cluster[['1', '2', '3']])
|
||||
df_cluster["label"] = kmeans_model.labels_
|
||||
|
||||
# 计算每一类的均值，超过均值设置为'高'，否则设置为'低'
|
||||
df_mean = df_cluster[['1', '2', '3', 'label']].groupby("label").agg("mean")
|
||||
df_mean["IF1"] = df_mean["1"].apply(lambda x: "高" if x > df_mean["1"].mean() else "低")
|
||||
df_mean["IF2"] = df_mean["2"].apply(lambda x: "高" if x > df_mean["2"].mean() else "低")
|
||||
df_mean["IF3"] = df_mean["3"].apply(lambda x: "高" if x > df_mean["3"].mean() else "低")
|
||||
|
||||
# 根据打分函数给用户进行打分
|
||||
df_mean["temp"] = df_mean["IF1"] + df_mean["IF2"] + df_mean["IF3"]
|
||||
df_mean["label"] = df_mean["temp"].apply(lambda x: iflabel(x))
|
||||
|
||||
df_cluster["IF1"] = df_cluster["1"].apply(lambda x: "高" if x > df_mean["1"].mean() else "低")
|
||||
df_cluster["IF2"] = df_cluster["2"].apply(lambda x: "高" if x > df_mean["1"].mean() else "低")
|
||||
df_cluster["IF3"] = df_cluster["3"].apply(lambda x: "高" if x > df_mean["1"].mean() else "低")
|
||||
df_cluster["temp"] = df_cluster["IF1"] + df_cluster["IF2"] + df_cluster["IF3"]
|
||||
df_cluster["label"] = df_cluster["temp"].apply(lambda x: iflabel(x))
|
||||
|
||||
# 将用户 id 和编程能力得分提取存储起来
|
||||
df_cluster = df_cluster.rename(columns={'label': 'programming_ability'})
|
||||
result_csv = df_cluster[['user_id','programming_ability']]
|
||||
|
||||
result_csv['user_id'] = result_csv['user_id'].astype(int)
|
||||
result_csv['programming_ability'] = result_csv['programming_ability'].astype(int)
|
||||
results_dict = dict(zip(result_csv['user_id'], result_csv['programming_ability']))
|
||||
|
||||
# 保存模型和结果
|
||||
pickle.dump(pca_model, open(programming_ability_analysis_path + 'results/user_programming_ability_pca_model.pkl', 'wb'))
|
||||
pickle.dump(kmeans_model, open(programming_ability_analysis_path + 'results/user_programming_ability_cluster_model.pkl', 'wb'))
|
||||
pickle.dump(results_dict, open(programming_ability_analysis_path + 'results/user_programming_ability_dict.pkl', 'wb'))
|
||||
|
||||
logger.info('用户编程能力分析模型训练完成')
|
||||
|
||||
return results_dict
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
train()
|
||||
|
|
@ -1,103 +0,0 @@
|
|||
import argparse
|
||||
import os
|
||||
from config import logger
|
||||
|
||||
|
||||
def del_output_file(parent_path):
|
||||
"""
|
||||
删除之前的数据和输出
|
||||
"""
|
||||
files_list = os.listdir(parent_path)
|
||||
for file_name in files_list:
|
||||
if not os.path.isdir(file_name) and ('.py' not in file_name):
|
||||
file_name = parent_path + file_name
|
||||
if os.path.exists(file_name):
|
||||
logger.info('删除文件: ' + file_name)
|
||||
os.system('rm ' + file_name)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--silent', action='store_true', help='是否以silent模式运行')
|
||||
args = parser.parse_args()
|
||||
|
||||
silent_mode = args.silent
|
||||
|
||||
if silent_mode:
|
||||
confirm = 'y'
|
||||
else:
|
||||
confirm = input('确认要重新开始训练所有模型吗？y/n' + '\n').strip()
|
||||
|
||||
if confirm == 'y' or confirm == 'Y':
|
||||
|
||||
if silent_mode:
|
||||
confirm = 'y'
|
||||
else:
|
||||
confirm = input('是否删除之前的数据和所有输出？y/n' + '\n').strip()
|
||||
|
||||
if confirm == 'y' or confirm == 'Y':
|
||||
parent_path_list = []
|
||||
parent_path_list.clear()
|
||||
parent_path_list.append('./activity_analysis/data/')
|
||||
parent_path_list.append('./activity_analysis/results/')
|
||||
|
||||
parent_path_list.append('./contribution_analysis/data/')
|
||||
parent_path_list.append('./contribution_analysis/results/')
|
||||
|
||||
parent_path_list.append('./interests_analysis/data/')
|
||||
parent_path_list.append('./interests_analysis/results/')
|
||||
|
||||
parent_path_list.append('./learning_ability_analysis/data/')
|
||||
parent_path_list.append('./learning_ability_analysis/results/')
|
||||
|
||||
parent_path_list.append('./professional_ability_analysis/data/')
|
||||
parent_path_list.append('./professional_ability_analysis/results/')
|
||||
|
||||
parent_path_list.append('./programming_ability_analysis/data/')
|
||||
parent_path_list.append('./programming_ability_analysis/results/')
|
||||
|
||||
parent_path_list.append('./user_label_analysis/data/')
|
||||
parent_path_list.append('./user_label_analysis/results/')
|
||||
|
||||
for parent_path in parent_path_list:
|
||||
if not os.path.exists(parent_path):
|
||||
os.mkdir(parent_path)
|
||||
del_output_file(parent_path)
|
||||
|
||||
# 用户活跃度分析数据处理
|
||||
os.system('python ./activity_analysis/data_process.py')
|
||||
# 用户活跃度分析模型训练
|
||||
os.system('python ./activity_analysis/train.py')
|
||||
|
||||
# 用户贡献度分析数据处理
|
||||
os.system('python ./contribution_analysis/data_process.py')
|
||||
# 用户贡献度分析模型训练
|
||||
os.system('python ./contribution_analysis/train.py')
|
||||
|
||||
# 用户爱好分析数据处理
|
||||
os.system('python ./interests_analysis/data_process.py')
|
||||
# 用户爱好分析模型训练
|
||||
os.system('python ./interests_analysis/train.py')
|
||||
|
||||
# 用户学习能力分析数据处理
|
||||
os.system('python ./learning_ability_analysis/data_process.py')
|
||||
# 用户学习能力分析模型训练
|
||||
os.system('python ./learning_ability_analysis/train.py')
|
||||
|
||||
# 用户专业能力分析数据处理
|
||||
os.system('python ./professional_ability_analysis/data_process.py')
|
||||
# 用户专业能力分析模型训练
|
||||
os.system('python ./professional_ability_analysis/train.py')
|
||||
|
||||
# 用户编程能力分析数据处理
|
||||
os.system('python ./programming_ability_analysis/data_process.py')
|
||||
# 用户编程能力分析模型训练
|
||||
os.system('python ./programming_ability_analysis/train.py')
|
||||
|
||||
# 用户标签分析数据处理
|
||||
os.system('python ./user_label_analysis/data_process.py')
|
||||
# 用户标签分析模型训练
|
||||
os.system('python ./user_label_analysis/train.py')
|
||||
|
||||
# 启动模型预测接口
|
||||
os.system('python ./flask_app.py')
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
## 这里面存放的是数据
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
|
|
@ -1,153 +0,0 @@
|
|||
import os
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from config import logger
|
||||
import datetime
|
||||
import pymysql
|
||||
import pickle
|
||||
from config import mysql_database, mysql_passwd
|
||||
from config import mysql_host, mysql_port, mysql_user
|
||||
from utils import get_before_date
|
||||
from config import data_before_days
|
||||
from config import activity_analysis_path
|
||||
from config import contribution_analysis_path
|
||||
from config import user_label_analysis_path
|
||||
from config import learning_ability_analysis_path
|
||||
from config import professional_ability_analysis_path
|
||||
from config import programming_ability_analysis_path
|
||||
|
||||
maplist = {
|
||||
"上海":"华东",
|
||||
"江苏":"华东",
|
||||
"浙江":"华东",
|
||||
"安徽":"华东",
|
||||
"福建":"华东",
|
||||
"江西":"华东",
|
||||
"山东":"华东",
|
||||
"台湾":"华东",
|
||||
"北京":"华北",
|
||||
"天津":"华北",
|
||||
"河北":"华北",
|
||||
"山西":"华北",
|
||||
"内蒙古":"华北",
|
||||
"广东":"华南",
|
||||
"广西":"华南",
|
||||
"海南":"华南",
|
||||
"香港":"华南",
|
||||
"澳门":"华南",
|
||||
"河南":"华中",
|
||||
"湖北":"华中",
|
||||
"湖南":"华中",
|
||||
"陕西":"西北",
|
||||
"甘肃":"西北",
|
||||
"青海":"西北",
|
||||
"宁夏":"西北",
|
||||
"新疆":"西北"
|
||||
}
|
||||
def get_data_from_mysql(latest_data_date):
|
||||
"""
|
||||
从mysql数据库获取原始数据
|
||||
"""
|
||||
start = datetime.datetime.now()
|
||||
logger.info("开始获取用户标签数据...")
|
||||
|
||||
sql_conn = pymysql.connect(host=mysql_host,
|
||||
user=mysql_user,
|
||||
passwd=mysql_passwd,
|
||||
port=mysql_port,
|
||||
db=mysql_database)
|
||||
|
||||
sql_text1 = f"""
|
||||
SELECT id, name, province
|
||||
FROM schools
|
||||
"""
|
||||
|
||||
data_schools = pd.read_sql(sql_text1, con=sql_conn)
|
||||
data_schools.to_csv( user_label_analysis_path + 'data/schools.csv', index=False, header=True, sep='\t')
|
||||
logger.info("用户标签数据1下载完毕，共" + str(data_schools.shape[0]) + "条数据，总耗时" + str((datetime.datetime.now() - start).seconds) + "秒")
|
||||
|
||||
sql_text2 = f"""
|
||||
SELECT user_id,username,school_name
|
||||
FROM subject_user_infos
|
||||
"""
|
||||
|
||||
data_user_infos = pd.read_sql(sql_text2, con=sql_conn)
|
||||
data_user_infos.to_csv(user_label_analysis_path + 'data/subject_user_infos.csv', index=False, header=True, sep='\t')
|
||||
logger.info(
|
||||
"用户标签数据2下载完毕，共" + str(data_user_infos.shape[0]) + "条数据，总耗时" + str((datetime.datetime.now() - start).seconds) + "秒")
|
||||
|
||||
def read_action_data():
|
||||
"""
|
||||
处理用户标签数据
|
||||
"""
|
||||
# 提取学校数据并将学校所在地区的分区标上
|
||||
data_school = pd.read_csv(user_label_analysis_path + 'data/schools.csv', sep='\t')
|
||||
data_school = data_school[['id', 'name', 'province']]
|
||||
data_school = data_school.dropna()
|
||||
data_school['area'] = data_school['province'].map(maplist)
|
||||
|
||||
data_user_infos = pd.read_csv(user_label_analysis_path + 'data/subject_user_infos.csv', sep='\t')
|
||||
data_user_infos = data_user_infos[['user_id', 'username', 'school_name']]
|
||||
data_user_infos = data_user_infos.drop_duplicates(subset='user_id')
|
||||
|
||||
return data_school, data_user_infos
|
||||
|
||||
|
||||
def get_label_data():
|
||||
"""
|
||||
生成用户排序数据
|
||||
"""
|
||||
data_school, data_user_infos = read_action_data()
|
||||
# 提取之前所有用户各项指标的得分
|
||||
user_activity_dict = pickle.load(open(activity_analysis_path + 'results/user_activity_dict.pkl', 'rb'))
|
||||
user_contribution_dict = pickle.load(open(contribution_analysis_path + 'results/user_contribution_dict.pkl', 'rb'))
|
||||
user_learning_ability_dict = pickle.load(
|
||||
open(learning_ability_analysis_path + 'results/user_learning_ability_dict.pkl', 'rb'))
|
||||
user_professional_ability_dict = pickle.load(
|
||||
open(professional_ability_analysis_path + 'results/user_professional_ability_dict.pkl', 'rb'))
|
||||
user_programming_ability_dict = pickle.load(
|
||||
open(programming_ability_analysis_path + 'results/user_programming_ability_dict.pkl', 'rb'))
|
||||
|
||||
df_user_activity = pd.DataFrame.from_dict(user_activity_dict, orient='index', columns=['activity'])
|
||||
df_user_activity = df_user_activity.reset_index()
|
||||
df_user_activity = df_user_activity.rename(columns={'index': 'user_id'})
|
||||
|
||||
df_user_contribution = pd.DataFrame.from_dict(user_contribution_dict, orient='index', columns=['contribution'])
|
||||
df_user_contribution = df_user_contribution.reset_index()
|
||||
df_user_contribution = df_user_contribution.rename(columns={'index': 'user_id'})
|
||||
|
||||
df_learning_ability = pd.DataFrame.from_dict(user_learning_ability_dict, orient='index',
|
||||
columns=['learning_ability'])
|
||||
df_learning_ability = df_learning_ability.reset_index()
|
||||
df_learning_ability = df_learning_ability.rename(columns={'index': 'user_id'})
|
||||
|
||||
df_professional_ability = pd.DataFrame.from_dict(user_professional_ability_dict, orient='index',
|
||||
columns=['professional_ability'])
|
||||
df_professional_ability = df_professional_ability.reset_index()
|
||||
df_professional_ability = df_professional_ability.rename(columns={'index': 'user_id'})
|
||||
|
||||
df_programming_ability = pd.DataFrame.from_dict(user_programming_ability_dict, orient='index',
|
||||
columns=['programming_ability'])
|
||||
df_programming_ability = df_programming_ability.reset_index()
|
||||
df_programming_ability = df_programming_ability.rename(columns={'index': 'user_id'})
|
||||
|
||||
# 将未统计到用户得分的项目都给他打1分
|
||||
df_merge = pd.merge(df_user_activity, df_user_contribution, on='user_id', how='outer')
|
||||
df_merge = df_merge.fillna(1)
|
||||
|
||||
df_merge = pd.merge(df_merge, df_learning_ability, on='user_id', how='outer')
|
||||
df_merge = df_merge.fillna(1)
|
||||
|
||||
df_merge = pd.merge(df_merge, df_professional_ability, on='user_id', how='outer')
|
||||
df_merge = df_merge.fillna(1)
|
||||
|
||||
df_merge = pd.merge(df_merge, df_programming_ability, on='user_id', how='outer')
|
||||
df_merge = df_merge.fillna(1)
|
||||
|
||||
return df_merge, data_school, data_user_infos
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
# 取数据
|
||||
latest_data_date = get_before_date(data_before_days)
|
||||
get_data_from_mysql(latest_data_date)
|
||||
|
|
@ -1,170 +0,0 @@
|
|||
import os
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from config import logger
|
||||
import datetime
|
||||
import pymysql
|
||||
import pickle
|
||||
from config import mysql_database, mysql_passwd
|
||||
from config import mysql_host, mysql_port, mysql_user
|
||||
from utils import get_before_date
|
||||
from config import data_before_days
|
||||
from config import activity_analysis_path
|
||||
from config import contribution_analysis_path
|
||||
from config import user_label_analysis_path
|
||||
from config import learning_ability_analysis_path
|
||||
from config import professional_ability_analysis_path
|
||||
from config import programming_ability_analysis_path
|
||||
|
||||
maplist = {
|
||||
"上海":"华东",
|
||||
"江苏":"华东",
|
||||
"浙江":"华东",
|
||||
"安徽":"华东",
|
||||
"福建":"华东",
|
||||
"江西":"华东",
|
||||
"山东":"华东",
|
||||
"台湾":"华东",
|
||||
"北京":"华北",
|
||||
"天津":"华北",
|
||||
"河北":"华北",
|
||||
"山西":"华北",
|
||||
"内蒙古":"华北",
|
||||
"广东":"华南",
|
||||
"广西":"华南",
|
||||
"海南":"华南",
|
||||
"香港":"华南",
|
||||
"澳门":"华南",
|
||||
"河南":"华中",
|
||||
"湖北":"华中",
|
||||
"湖南":"华中",
|
||||
"陕西":"西北",
|
||||
"甘肃":"西北",
|
||||
"青海":"西北",
|
||||
"宁夏":"西北",
|
||||
"新疆":"西北"
|
||||
}
|
||||
def get_data_from_mysql(latest_data_date):
|
||||
"""
|
||||
从mysql数据库获取原始数据
|
||||
"""
|
||||
start = datetime.datetime.now()
|
||||
logger.info("开始获取用户标签数据...")
|
||||
|
||||
sql_conn = pymysql.connect(host=mysql_host,
|
||||
user=mysql_user,
|
||||
passwd=mysql_passwd,
|
||||
port=mysql_port,
|
||||
db=mysql_database)
|
||||
|
||||
sql_text1 = f"""
|
||||
select
|
||||
t1.id,
|
||||
t1.name,
|
||||
t1.province
|
||||
from
|
||||
schools t1
|
||||
left join user_extensions t2 on t1.id = t2.school_id
|
||||
left join users t3 on t2.user_id = t3.id
|
||||
where
|
||||
DATE_FORMAT(t3.last_login_on, '%Y-%m-%d') >= '{latest_data_date}'
|
||||
and t1.province is not null
|
||||
and t1.province != ''
|
||||
"""
|
||||
|
||||
data_schools = pd.read_sql(sql_text1, con=sql_conn)
|
||||
data_schools.to_csv( user_label_analysis_path + 'data/schools.csv', index=False, header=True, sep='\t')
|
||||
logger.info("用户标签数据1下载完毕，共" + str(data_schools.shape[0]) + "条数据，总耗时" + str((datetime.datetime.now() - start).seconds) + "秒")
|
||||
|
||||
sql_text2 = f"""
|
||||
SELECT
|
||||
user_id,
|
||||
username,
|
||||
school_name
|
||||
FROM
|
||||
subject_user_infos t1
|
||||
LEFT JOIN users t2 ON t1.user_id = t2.id
|
||||
WHERE
|
||||
DATE_FORMAT(t2.last_login_on, '%Y-%m-%d') >= '{latest_data_date}'
|
||||
"""
|
||||
|
||||
data_user_infos = pd.read_sql(sql_text2, con=sql_conn)
|
||||
data_user_infos.to_csv(user_label_analysis_path + 'data/subject_user_infos.csv', index=False, header=True, sep='\t')
|
||||
logger.info(
|
||||
"用户标签数据2下载完毕，共" + str(data_user_infos.shape[0]) + "条数据，总耗时" + str((datetime.datetime.now() - start).seconds) + "秒")
|
||||
|
||||
def read_action_data():
|
||||
"""
|
||||
处理用户标签数据
|
||||
"""
|
||||
# 提取学校数据并将学校所在地区的分区标上
|
||||
data_school = pd.read_csv(user_label_analysis_path + 'data/schools.csv', sep='\t')
|
||||
data_school = data_school[['id', 'name', 'province']]
|
||||
data_school = data_school.dropna()
|
||||
data_school['area'] = data_school['province'].map(maplist)
|
||||
|
||||
data_user_infos = pd.read_csv(user_label_analysis_path + 'data/subject_user_infos.csv', sep='\t')
|
||||
data_user_infos = data_user_infos[['user_id', 'username', 'school_name']]
|
||||
data_user_infos = data_user_infos.drop_duplicates(subset='user_id')
|
||||
|
||||
return data_school, data_user_infos
|
||||
|
||||
|
||||
def get_label_data():
|
||||
"""
|
||||
生成用户排序数据
|
||||
"""
|
||||
data_school, data_user_infos = read_action_data()
|
||||
# 提取之前所有用户各项指标的得分
|
||||
user_activity_dict = pickle.load(open(activity_analysis_path + 'results/user_activity_dict.pkl', 'rb'))
|
||||
user_contribution_dict = pickle.load(open(contribution_analysis_path + 'results/user_contribution_dict.pkl', 'rb'))
|
||||
user_learning_ability_dict = pickle.load(
|
||||
open(learning_ability_analysis_path + 'results/user_learning_ability_dict.pkl', 'rb'))
|
||||
user_professional_ability_dict = pickle.load(
|
||||
open(professional_ability_analysis_path + 'results/user_professional_ability_dict.pkl', 'rb'))
|
||||
user_programming_ability_dict = pickle.load(
|
||||
open(programming_ability_analysis_path + 'results/user_programming_ability_dict.pkl', 'rb'))
|
||||
|
||||
df_user_activity = pd.DataFrame.from_dict(user_activity_dict, orient='index', columns=['activity'])
|
||||
df_user_activity = df_user_activity.reset_index()
|
||||
df_user_activity = df_user_activity.rename(columns={'index': 'user_id'})
|
||||
|
||||
df_user_contribution = pd.DataFrame.from_dict(user_contribution_dict, orient='index', columns=['contribution'])
|
||||
df_user_contribution = df_user_contribution.reset_index()
|
||||
df_user_contribution = df_user_contribution.rename(columns={'index': 'user_id'})
|
||||
|
||||
df_learning_ability = pd.DataFrame.from_dict(user_learning_ability_dict, orient='index',
|
||||
columns=['learning_ability'])
|
||||
df_learning_ability = df_learning_ability.reset_index()
|
||||
df_learning_ability = df_learning_ability.rename(columns={'index': 'user_id'})
|
||||
|
||||
df_professional_ability = pd.DataFrame.from_dict(user_professional_ability_dict, orient='index',
|
||||
columns=['professional_ability'])
|
||||
df_professional_ability = df_professional_ability.reset_index()
|
||||
df_professional_ability = df_professional_ability.rename(columns={'index': 'user_id'})
|
||||
|
||||
df_programming_ability = pd.DataFrame.from_dict(user_programming_ability_dict, orient='index',
|
||||
columns=['programming_ability'])
|
||||
df_programming_ability = df_programming_ability.reset_index()
|
||||
df_programming_ability = df_programming_ability.rename(columns={'index': 'user_id'})
|
||||
|
||||
# 将未统计到用户得分的项目都给他打1分
|
||||
df_merge = pd.merge(df_user_activity, df_user_contribution, on='user_id', how='outer')
|
||||
df_merge = df_merge.fillna(1)
|
||||
|
||||
df_merge = pd.merge(df_merge, df_learning_ability, on='user_id', how='outer')
|
||||
df_merge = df_merge.fillna(1)
|
||||
|
||||
df_merge = pd.merge(df_merge, df_professional_ability, on='user_id', how='outer')
|
||||
df_merge = df_merge.fillna(1)
|
||||
|
||||
df_merge = pd.merge(df_merge, df_programming_ability, on='user_id', how='outer')
|
||||
df_merge = df_merge.fillna(1)
|
||||
|
||||
return df_merge, data_school, data_user_infos
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
# 取数据
|
||||
latest_data_date = get_before_date(data_before_days)
|
||||
get_data_from_mysql(latest_data_date)
|
||||
Binary file not shown.
Binary file not shown.
|
|
@ -1 +0,0 @@
|
|||
## 这里是模型的代码说明
|
||||
|
|
@ -1,36 +0,0 @@
|
|||
import pickle
|
||||
from config import logger
|
||||
from config import test_user_id
|
||||
from config import user_label_analysis_path
|
||||
|
||||
logger.info('加载用户排序字典')
|
||||
results_dict_rank = pickle.load(open(user_label_analysis_path + 'results/user_rank_dict.pkl', 'rb'))
|
||||
results_dict_area_rank = pickle.load(open(user_label_analysis_path + 'results/user_area_rank_dict.pkl', 'rb'))
|
||||
results_dict_school_rank = pickle.load(open(user_label_analysis_path+ 'results/user_school_rank_dict.pkl', 'rb'))
|
||||
results_dict_label = pickle.load(open(user_label_analysis_path + 'results/user_label_dict.pkl', 'rb'))
|
||||
|
||||
def user_label_predict(user_id):
|
||||
"""
|
||||
用户贡献度预测
|
||||
"""
|
||||
if user_id not in results_dict_rank:
|
||||
results_rank = -1
|
||||
results_area_rank = -1
|
||||
results_school_rank = -1
|
||||
results_label = -1
|
||||
else:
|
||||
results_rank = results_dict_rank[user_id]
|
||||
results_area_rank = results_dict_area_rank[user_id]
|
||||
results_school_rank = results_dict_school_rank[user_id]
|
||||
results_label = results_dict_label[user_id]
|
||||
|
||||
return results_rank, results_area_rank, results_school_rank, results_label
|
||||
|
||||
if __name__ == '__main__':
|
||||
results_rank, results_area_rank, results_school_rank, results_label = user_label_predict(user_id=test_user_id)
|
||||
print('用户ID：', test_user_id,
|
||||
'全国排名：', results_rank,
|
||||
'地区排名：',results_area_rank,
|
||||
'学校排名：',results_school_rank,
|
||||
'用户标签：',results_label)
|
||||
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
## 这里面存放模型训练结果
|
||||
|
||||
|
|
@ -1,128 +0,0 @@
|
|||
import os
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import pickle
|
||||
from sklearn.cluster import KMeans
|
||||
from config import logger
|
||||
from config import RANDOM_SEED
|
||||
from data_process import get_label_data
|
||||
from config import user_label_analysis_path
|
||||
from learning2rank.rank import RankNet
|
||||
import chainer
|
||||
# 根据比例打标签
|
||||
def iflabel(x,data):
|
||||
class_names = ['学霸型', '学习优秀型','学习进步型' ,'学习懒散型']
|
||||
num = len(data)
|
||||
if x >= 0 and x < num*0.15:
|
||||
return class_names[0]
|
||||
elif x >= num*0.15 and x < num*0.5:
|
||||
return class_names[1]
|
||||
elif x >= num*0.5 and x < num*0.85:
|
||||
return class_names[2]
|
||||
else:
|
||||
return class_names[3]
|
||||
def train():
|
||||
"""
|
||||
用户排序模型训练
|
||||
"""
|
||||
# 根据 5 想指标计算总和并根据总和将所有用户切分为 4 类作为用户标签
|
||||
logger.info('开始训练用户排序分析模型...')
|
||||
data, data_shool, data_user = get_label_data()
|
||||
data['label'] = data[
|
||||
['activity', 'contribution', 'learning_ability', 'professional_ability', 'programming_ability']].sum(1)
|
||||
q = pd.cut(data.label, 4, labels=[1, 2, 3, 4])
|
||||
data['label'] = q
|
||||
|
||||
'''
|
||||
此处为另一种思路的代码，采用k_means聚类
|
||||
'''
|
||||
# cluster = data[['activity','contribution','learning_ability','professional_ability','programming_ability']]
|
||||
# kmeans = KMeans(n_clusters=5, random_state=0).fit(cluster)
|
||||
# data["label"] = kmeans.labels_ + 1
|
||||
|
||||
# 将用户 5 个维度的数据作为特征，上面切分的类别作为标签送入排序模型进行训练然后到处预测结果
|
||||
rank_model = RankNet.RankNet()
|
||||
X = np.array(data.iloc[:, 1:6].values)
|
||||
y = np.array(data.iloc[:, -1].values)
|
||||
rank_model.fit(X, y)
|
||||
user_rank = rank_model.predict(X)
|
||||
|
||||
# 根据预测结果将分值转化为排名
|
||||
ranks_ = []
|
||||
for i in range(len(user_rank)):
|
||||
ranks_.append(user_rank[i][0])
|
||||
ranks1 = pd.Series(ranks_, data.index)
|
||||
data['RankNet_predict_value'] = ranks1
|
||||
|
||||
# 根据排名将用户设置为4种类型：'学习懒散型', '学习进步型', '学习优秀型', '学霸型'
|
||||
data = data.sort_values(by='RankNet_predict_value', ascending=False)
|
||||
data['rank'] = data['RankNet_predict_value'].rank(ascending=False, method='min')
|
||||
data['rank'] = data['rank'].astype('float').astype('int')
|
||||
data = data.reset_index(drop=True)
|
||||
data['index'] = data.index
|
||||
data['final_label'] = data['index'].apply(lambda x: iflabel(x, data))
|
||||
data.drop('index', axis=1, inplace=True)
|
||||
|
||||
# 提取用户 id ,姓名，学校，地区
|
||||
data_user_info_1 = pd.merge(data, data_user, on='user_id')
|
||||
data_user_info_2 = data_shool[['name', 'area']]
|
||||
data_user_info_2 = data_user_info_2.rename(columns={'name': 'school_name'})
|
||||
data_user_infos = pd.merge(data_user_info_1, data_user_info_2, on='school_name')
|
||||
|
||||
# 根据所属地区，将每个地区的用户的排名
|
||||
areas = list(data_user_infos.area.unique())
|
||||
df_area_rank = data_user_infos.copy()
|
||||
df_area_rank['area_rank'] = 0
|
||||
df_area_rank = df_area_rank.drop(df_area_rank.index[0:len(df_area_rank)], 0)
|
||||
for area in areas:
|
||||
df_user_copy = data_user_infos[data_user_infos.area == area].copy()
|
||||
df_user_copy['area_rank'] = df_user_copy['RankNet_predict_value'].rank(ascending=False, method='min')
|
||||
df_user_copy['area_rank'] = df_user_copy['area_rank'].astype('int')
|
||||
df_area_rank = pd.concat([df_user_copy, df_area_rank])
|
||||
|
||||
# 根据学校名称，将每个学校的用户进行内部排名
|
||||
schools = list(data_user_infos.school_name.unique())
|
||||
df_school_rank = df_area_rank.copy()
|
||||
df_school_rank['school_rank'] = 0
|
||||
df_school_rank = df_school_rank.drop(df_school_rank.index[0:len(df_school_rank)], 0)
|
||||
for school in schools:
|
||||
df_school_copy = df_area_rank[df_area_rank.school_name == school].copy()
|
||||
df_school_copy['school_rank'] = df_school_copy['RankNet_predict_value'].rank(ascending=False, method='min')
|
||||
df_school_copy['school_rank'] = df_school_copy['school_rank'].astype('int')
|
||||
df_school_rank = pd.concat([df_school_copy, df_school_rank])
|
||||
|
||||
# 将用户 id ，全国排名，标签，地区排名和学校排名提取并且保存
|
||||
result_csv = df_school_rank[['user_id',
|
||||
'rank',
|
||||
'final_label',
|
||||
'area_rank',
|
||||
'school_rank']]
|
||||
|
||||
df_school_rank.to_csv(user_label_analysis_path + 'results/user_label_analysis.csv',columns=["user_id","activity"
|
||||
,"contribution","learning_ability","professional_ability","programming_ability","label","RankNet_predict_value","rank",
|
||||
"final_label","username","school_name"])
|
||||
|
||||
|
||||
result_csv['user_id'] = result_csv['user_id'].astype(int)
|
||||
result_csv['rank'] = result_csv['rank'].astype(int)
|
||||
result_csv['area_rank'] = result_csv['area_rank'].astype(int)
|
||||
result_csv['school_rank'] = result_csv['school_rank'].astype(int)
|
||||
|
||||
results_dict_rank = dict(zip(result_csv['user_id'], result_csv['rank']))
|
||||
results_dict_area_rank = dict(zip(result_csv['user_id'], result_csv['area_rank']))
|
||||
results_dict_school_rank = dict(zip(result_csv['user_id'], result_csv['school_rank']))
|
||||
results_dict_label = dict(zip(result_csv['user_id'], result_csv['final_label']))
|
||||
|
||||
# # 保存模型和结果
|
||||
pickle.dump(rank_model, open(user_label_analysis_path + 'results/user_label_rankmodel_dict.pkl', 'wb'))
|
||||
pickle.dump(results_dict_rank, open(user_label_analysis_path + 'results/user_rank_dict.pkl', 'wb'))
|
||||
pickle.dump(results_dict_area_rank, open(user_label_analysis_path + 'results/user_area_rank_dict.pkl', 'wb'))
|
||||
pickle.dump(results_dict_school_rank, open(user_label_analysis_path + 'results/user_school_rank_dict.pkl', 'wb'))
|
||||
pickle.dump(results_dict_label, open(user_label_analysis_path + 'results/user_label_dict.pkl', 'wb'))
|
||||
logger.info('用户排序模型训练完成')
|
||||
|
||||
return results_dict_rank, results_dict_area_rank, results_dict_school_rank, results_dict_label
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
train()
|
||||
|
|
@ -1,55 +0,0 @@
|
|||
import os
|
||||
import logging
|
||||
import random
|
||||
import numpy as np
|
||||
import datetime
|
||||
|
||||
|
||||
def create_logger(log_path):
|
||||
"""
|
||||
将日志输出到日志文件和控制台
|
||||
"""
|
||||
logger = logging.getLogger(__name__)
|
||||
logger.setLevel(logging.INFO)
|
||||
|
||||
formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
|
||||
|
||||
# 创建一个handler，用于写入日志文件
|
||||
file_handler = logging.FileHandler(filename=log_path)
|
||||
file_handler.setFormatter(formatter)
|
||||
file_handler.setLevel(logging.INFO)
|
||||
if file_handler not in logger.handlers:
|
||||
logger.addHandler(file_handler)
|
||||
|
||||
# 创建一个handler，用于将日志输出到控制台
|
||||
console = logging.StreamHandler()
|
||||
console.setLevel(logging.DEBUG)
|
||||
console.setFormatter(formatter)
|
||||
if console not in logger.handlers:
|
||||
logger.addHandler(console)
|
||||
|
||||
return logger
|
||||
|
||||
def get_file_name(fname):
|
||||
"""
|
||||
获取文件名
|
||||
"""
|
||||
return os.path.split(fname)[-1].split(".")[0]
|
||||
|
||||
def get_file_size(fname):
|
||||
"""
|
||||
获取文件大小MB
|
||||
"""
|
||||
fsize = os.path.getsize(fname)
|
||||
fsize = fsize/float(1024 * 1024)
|
||||
return round(fsize, 2)
|
||||
|
||||
def get_before_date(n):
|
||||
"""
|
||||
获取前N天的日期
|
||||
"""
|
||||
today = datetime.datetime.now()
|
||||
# 计算偏移量
|
||||
offset = datetime.timedelta(days=-n)
|
||||
re_date = (today + offset).strftime('%Y-%m-%d')
|
||||
return re_date
|
||||
4359856
xml2json/db.json
4359856
xml2json/db.json
File diff suppressed because one or more lines are too long
|
|
@ -1,114 +0,0 @@
|
|||
import re
|
||||
import json
|
||||
|
||||
|
||||
def load_json(xml_paths):
|
||||
result_list = []
|
||||
|
||||
for xml_path in xml_paths:
|
||||
xml_file = open(xml_path, 'r', encoding='utf-8')
|
||||
xml_str = xml_file.read()
|
||||
|
||||
name_pattern = r'<name>(.*?)</name>'
|
||||
name_matches = re.findall(name_pattern, xml_str)
|
||||
|
||||
vuln_id_pattern = r'<vuln-id>(.*?)</vuln-id>'
|
||||
vuln_id_matches = re.findall(vuln_id_pattern, xml_str)
|
||||
|
||||
published_pattern = r'<published>(.*?)</published>'
|
||||
published_matches = re.findall(published_pattern, xml_str)
|
||||
|
||||
modified_pattern = r'<modified>(.*?)</modified>'
|
||||
modified_matches = re.findall(modified_pattern, xml_str)
|
||||
|
||||
severity_pattern = r'<severity>(.*?)</severity>'
|
||||
severity_matches = re.findall(severity_pattern, xml_str)
|
||||
|
||||
vuln_type_pattern = r'<vuln-type>(.*?)</vuln-type>'
|
||||
vuln_type_matches = re.findall(vuln_type_pattern, xml_str)
|
||||
|
||||
descript_pattern = r'<vuln-descript>(.*?)</vuln-descript>'
|
||||
descript_matches = re.findall(descript_pattern, xml_str)
|
||||
|
||||
cve_id_pattern = r'<cve-id>(.*?)</cve-id>'
|
||||
cve_id_matches = re.findall(cve_id_pattern, xml_str)
|
||||
|
||||
for i in range(len(name_matches)):
|
||||
name = name_matches[i]
|
||||
vuln_id = vuln_id_matches[i]
|
||||
published = published_matches[i]
|
||||
modified = modified_matches[i]
|
||||
severity = severity_matches[i]
|
||||
if severity == '高危':
|
||||
severity = 1
|
||||
elif severity == '中危':
|
||||
severity = 2
|
||||
elif severity == '低危':
|
||||
severity = 3
|
||||
else:
|
||||
severity = 4
|
||||
vuln_type = vuln_type_matches[i]
|
||||
descript = descript_matches[i]
|
||||
cve_id = cve_id_matches[i]
|
||||
|
||||
_dict = {
|
||||
"vendor": "",
|
||||
"product": "",
|
||||
"version": "",
|
||||
"language": "",
|
||||
"name": name,
|
||||
"id": "",
|
||||
"cve_id": cve_id,
|
||||
"cnnvd_id": vuln_id,
|
||||
"cnvd_id": "",
|
||||
"cwe_id": "",
|
||||
"description": descript,
|
||||
"description_en": "",
|
||||
"suggestion": "",
|
||||
"attack_type": vuln_type,
|
||||
"release_date": modified,
|
||||
"security_level_id": severity,
|
||||
"exploit_level_id": 1
|
||||
}
|
||||
result_list.append(_dict)
|
||||
json.dump(result_list, open('db.json', 'w', encoding='utf-8'), indent=4, ensure_ascii=False)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
xml_paths = [
|
||||
'xml/1999年之前_e3d7b377-f26f-402c-af13-e402a764773a.xml',
|
||||
'xml/1月_2537f56e-2f4e-45fa-b8b5-b4298e6467f2.xml',
|
||||
'xml/2000年_f435df04-83a0-458c-9030-fbd278131072.xml',
|
||||
'xml/2001年_1029f67d-bed6-41f5-8594-f7f8be046cbf.xml',
|
||||
'xml/2002年_4da5f5f5-1ab3-4e63-8ea9-5d04c91d0860.xml',
|
||||
'xml/2003年_1742c3f6-3c07-4ee0-a32b-21c69fa8d701.xml',
|
||||
'xml/2004年_1c2188fe-bb78-4591-85a7-5b614a017d9b.xml',
|
||||
'xml/2006年_a15d20d7-f984-4dbe-8b7e-a76e5ab1bd95.xml',
|
||||
'xml/2007年_2c740fa7-af4f-4737-8673-41886b0d292f.xml',
|
||||
'xml/2008年_ef9b84b5-8222-405a-8623-0a3ad70bd610.xml',
|
||||
'xml/2009年_f5a59523-2e45-4c39-8562-0749180edb26.xml',
|
||||
'xml/2010年_cd8211e1-c358-4d8c-84a0-eb7d2e436c11.xml',
|
||||
'xml/2011年_0cf822e5-d663-4656-8dee-c3a515780e58.xml',
|
||||
'xml/2012年_ba5eb19e-ff15-4303-92a0-c128f19fe0d2.xml',
|
||||
'xml/2013年_47167073-c83f-49d2-811a-f46244ace505.xml',
|
||||
'xml/2013年_b472b9e1-c92e-4922-b568-441c671b66f7.xml',
|
||||
'xml/2014年_82c115d6-bfbb-41fb-bde5-62bd637f60f1.xml',
|
||||
'xml/2015年_ccf6f9ce-59ed-40af-8f69-0b368d02437e.xml',
|
||||
'xml/2016年_bf9b9ed1-903f-4c3b-8e1d-103dec96c546.xml',
|
||||
'xml/2017年_bd9b5558-4a27-47d6-b0d9-6c896f8997c5.xml',
|
||||
'xml/2018年_ac33df3d-5595-4731-89e4-85c52fe0e19a.xml',
|
||||
'xml/2019年_76549afa-69b3-4acf-8822-45d6003aebef.xml',
|
||||
'xml/2020年_fac21a28-9ea0-4917-a856-71f25acefcde.xml',
|
||||
'xml/2021年_8f26e26c-3569-46cc-9cfe-3854680c823e.xml',
|
||||
'xml/2022年_e95d5f08-83c1-41a4-8d8e-729b57c6c940.xml',
|
||||
'xml/2月_3dca592a-f114-4212-886f-e873bdf24631.xml',
|
||||
'xml/3月_289684c2-a147-4654-9f6a-acb7d6e4b731.xml',
|
||||
'xml/4月_25cdbed9-5465-4b54-aa9b-5d6eb8ef9d54.xml',
|
||||
'xml/5月_38a9a0a4-39bd-4a56-a5f1-0f5c7409a3ec.xml',
|
||||
'xml/6月_dbcae8bf-082d-4c7d-8049-6b83acef8e8b.xml',
|
||||
'xml/7月_b299cdc0-0e89-4e8f-b350-4bb62c5582fb.xml',
|
||||
'xml/8月_576e923b-79c1-4acf-a73a-d5df0a469540.xml',
|
||||
'xml/当日_e050b4c8-6e70-4a25-94f3-f5fbcf0babe6.xml',
|
||||
'xml/当月_aba548a6-08bf-4137-8875-fa82b717d88d.xml',
|
||||
]
|
||||
load_json(xml_paths)
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because one or more lines are too long
File diff suppressed because it is too large
Load Diff
File diff suppressed because one or more lines are too long
File diff suppressed because it is too large
Load Diff
File diff suppressed because one or more lines are too long
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue