用户活跃度

config.py

@@ -5,14 +5,31 @@ from utils import create_logger
 # 项目根目录
 root_path = os.path.abspath(os.path.dirname(__file__))
 
-
 # 创建日志对象
 logger = create_logger(root_path + '/logs/' + str(datetime.date(datetime.now())) + '.log')
 
-
 # PostgreSQL连接配置
 host = '118.31.13.117'
 user = 'postgres'
 passwd = 'Edu_123123pos'
 port = 55432
-database = 'opendata'
+database = 'opendata'
+
+# 各子模型目录
+activity_analysis_path = root_path + '/user_portrait_analysis/activity_analysis/'
+contribution_analysis_path = root_path + '/contribution_analysis/'
+interests_analysis_path = root_path + '/interests_analysis/'
+learning_ability_analysis_path = root_path + '/learning_ability_analysis/'
+programming_ability_analysis_path = root_path + '/programming_ability_analysis/'
+professional_ability_analysis_path = root_path + '/professional_ability_analysis/'
+user_label_analysis_path = root_path + '/user_label_analysis/'
+
+
+# 之前登录的天数
+before_login_days = 30
+data_before_days = 100000
+
+test_user_id = 10
+
+# 随机数种子
+RANDOM_SEED = 42

user_portrait_analysis/activity_analysis/data_process.py

@@ -3,55 +3,65 @@ import numpy as np
 import pandas as pd
 from config import logger
 import datetime
-import pymysql
-from config import mysql_database, mysql_passwd
-from config import mysql_host, mysql_port, mysql_user
+
+from sqlalchemy import create_engine
+from config import host, port, user, passwd, database
+
 from utils import get_before_date
 from config import activity_analysis_path
 from config import data_before_days
 
 
-def get_data_from_mysql(latest_data_date):
+def get_data_from_pgsql(latest_data_date):
     """
     从mysql数据库获取原始数据
     """
     start = datetime.datetime.now()
     logger.info("开始获取用户活跃度数据...")
 
-    sql_conn = pymysql.connect(host=mysql_host, 
-                                user=mysql_user, 
-                                passwd=mysql_passwd, 
-                                port=mysql_port, 
-                                db=mysql_database)    
+    engine = create_engine(f'postgresql+psycopg2://{user}:{passwd}@{host}:{port}/{database}', pool_recycle=3600)
+    conn = engine.connect()
 
     sql_text = f"""
-        SELECT id, user_id, action_type, action_id, created_at, updated_at, ip
-        FROM user_actions
-        WHERE created_at >= '{latest_data_date}'
+        SELECT
+            u_id user_id,
+            'login' action_type,
+            to_char(to_timestamp(created_unix), 'YYYY-MM-DD') created_at 
+        FROM
+            user_login_log 
+        WHERE to_char(to_timestamp(created_unix), 'YYYY-MM-DD') >= '{latest_data_date}'
+        UNION
+        SELECT
+            t1.user_id,
+            CASE WHEN t1.status = 2 THEN 'clickNotice' ELSE' closeNotice' END action_type,
+            to_char(to_timestamp(created_unix), 'YYYY-MM-DD') created_at 
+        FROM
+            notification t1 
+        WHERE
+            t1.status IN ( 2, 4 )
+            AND to_char(to_timestamp(created_unix), 'YYYY-MM-DD') >= '{latest_data_date}'
         """
-
-    data_df = pd.read_sql(sql_text, con=sql_conn)
+    data_df = pd.read_sql(sql_text, conn)
     data_df.to_csv(activity_analysis_path + 'data/user_actions.csv', index=False, header=True, sep='\t')
-
     logger.info("用户活跃度数据下载完毕，共" + str(data_df.shape[0]) + "条数据，总耗时" + str((datetime.datetime.now() - start).seconds) + "秒")
 
 
-def read_action_data():  
+def read_action_data():
     """
     处理用户活跃度数据
     """
     data_user_action = pd.read_csv(activity_analysis_path + 'data/user_actions.csv', sep='\t')
 
-    data_user_action = data_user_action[['user_id','action_type','created_at']]
+    data_user_action = data_user_action[['user_id', 'action_type', 'created_at']]
     # 提取出三种最常用的操作
-    some_action = ['Attachment','Login','closeNotice', 'clickNotice']
+    some_action = ['login', 'closeNotice', 'clickNotice']
     data_user_action = data_user_action.loc[data_user_action['action_type'].isin(some_action)]
     # 将日期格式进行转换
     data_user_action['created_at'] = pd.to_datetime(data_user_action['created_at'], dayfirst=True)
 
     data_user_action['created_at'] = data_user_action['created_at'].dt.date
     return data_user_action
-    
+
 
 def get_rfm_data():
     """
@@ -59,29 +69,28 @@ def get_rfm_data():
     """
     data = read_action_data()
 
-    df_rfm = data[['user_id','action_type','created_at']]
-    # 给不同操作赋值，来区分不同操作的重要度
+    df_rfm = data[['user_id', 'action_type', 'created_at']]
     class_dict = {
-        'Attachment': 2,
-        'Login': 1,
-        'closeNotice': 4,
-        'closeNotice': 4
+        'login': 1,
+        'clickNotice': 2,
+        'closeNotice': 3,
     }
-    df_rfm['action_values'] = df_rfm['action_type'].map(lambda x:x)
+    # 给不同操作赋值，来区分不同操作的重要度
+    df_rfm['action_values'] = df_rfm['action_type'].map(lambda x: x)
     df_rfm['action_values'] = df_rfm['action_values'].map(class_dict)
     # 计算 R,F,M 值
-    df_rfm = df_rfm.groupby("user_id").agg({'created_at':'max','user_id':'count','action_values':'sum'})
-    df_rfm = df_rfm.rename(columns ={'created_at':'Recentdate','user_id':'F','action_values':'M'})
+    df_rfm = df_rfm.groupby("user_id").agg({'created_at': 'max', 'user_id': 'count', 'action_values': 'sum'})
+    df_rfm = df_rfm.rename(columns={'created_at': 'Recentdate', 'user_id': 'F', 'action_values': 'M'})
 
-    df_rfm["R"] =(df_rfm['Recentdate'].max() - df_rfm['Recentdate'])/np.timedelta64(1,'D')
-    df_rfm["R"] = df_rfm["R"].astype('str').str.split(" ",expand=True)
+    df_rfm["R"] = (df_rfm['Recentdate'].max() - df_rfm['Recentdate']) / np.timedelta64(1, 'D')
+    df_rfm["R"] = df_rfm["R"].astype('str').str.split(" ", expand=True)
     df_rfm["R"] = df_rfm["R"].astype("float").astype("int")
-    df_rfm.drop(columns='Recentdate',inplace=True)
+    df_rfm.drop(columns='Recentdate', inplace=True)
 
     return df_rfm
 
-if __name__ == '__main__':
 
+if __name__ == '__main__':
     # 取之前多少天登录的数据
     latest_data_date = get_before_date(data_before_days)
-    get_data_from_mysql(latest_data_date)
+    get_data_from_pgsql(latest_data_date)

user_portrait_analysis/activity_analysis/data_process_jw.py

@@ -1,98 +0,0 @@
-import os
-import numpy as np
-import pandas as pd
-from config import logger
-import datetime
-import pymysql
-from config import mysql_database, mysql_passwd
-from config import mysql_host, mysql_port, mysql_user
-from utils import get_before_date
-from config import activity_analysis_path
-from config import data_before_days
-
-
-def get_data_from_mysql(latest_data_date):
-    """
-    从mysql数据库获取原始数据
-    """
-    start = datetime.datetime.now()
-    logger.info("开始获取用户活跃度数据...")
-
-    sql_conn = pymysql.connect(host=mysql_host, 
-                                user=mysql_user, 
-                                passwd=mysql_passwd, 
-                                port=mysql_port, 
-                                db=mysql_database)    
-
-    sql_text = f"""
-        SELECT 
-            t1.id,
-            t1.user_id,
-            t1.action_type,
-            t1.action_id,
-            t1.created_at,
-            t1.updated_at,
-            t1.ip 
-        FROM
-            user_actions t1 
-            LEFT JOIN users t2 ON t1.user_id = t2.id 
-        WHERE t1.action_type IN ('Attachment', 'Login', 'closeNotice', 'clickNotice') 
-            AND DATE_FORMAT(t1.created_at, '%Y-%m-%d') >= '{latest_data_date}'
-            AND DATE_FORMAT(t2.last_login_on, '%Y-%m-%d') >= '{latest_data_date}'
-        """
-
-    data_df = pd.read_sql(sql_text, con=sql_conn)
-    data_df.to_csv(activity_analysis_path + 'data/user_actions.csv', index=False, header=True, sep='\t')
-
-    logger.info("用户活跃度数据下载完毕，共" + str(data_df.shape[0]) + "条数据，总耗时" + str((datetime.datetime.now() - start).seconds) + "秒")
-
-
-def read_action_data():  
-    """
-    处理用户活跃度数据
-    """
-    data_user_action = pd.read_csv(activity_analysis_path + 'data/user_actions.csv', sep='\t')
-
-    data_user_action = data_user_action[['user_id','action_type','created_at']]
-    # 提取出三种最常用的操作
-    some_action = ['Attachment','Login','closeNotice', 'clickNotice']
-    data_user_action = data_user_action.loc[data_user_action['action_type'].isin(some_action)]
-    # 将日期格式进行转换
-    data_user_action['created_at'] = pd.to_datetime(data_user_action['created_at'], dayfirst=True)
-
-    data_user_action['created_at'] = data_user_action['created_at'].dt.date
-    return data_user_action
-    
-
-def get_rfm_data():
-    """
-    生成RFM聚类模型训练数据
-    """
-    data = read_action_data()
-
-    df_rfm = data[['user_id','action_type','created_at']]
-    # 给不同操作赋值，来区分不同操作的重要度
-    class_dict = {
-        'Attachment': 2,
-        'Login': 1,
-        'closeNotice': 4,
-        'closeNotice': 4
-    }
-    df_rfm['action_values'] = df_rfm['action_type'].map(lambda x:x)
-    df_rfm['action_values'] = df_rfm['action_values'].map(class_dict)
-    # 计算 R,F,M 值
-    df_rfm = df_rfm.groupby("user_id").agg({'created_at':'max','user_id':'count','action_values':'sum'})
-    df_rfm = df_rfm.rename(columns ={'created_at':'Recentdate','user_id':'F','action_values':'M'})
-
-    df_rfm["R"] =(df_rfm['Recentdate'].max() - df_rfm['Recentdate'])/np.timedelta64(1,'D')
-    df_rfm["R"] = df_rfm["R"].astype('str').str.split(" ",expand=True)
-    df_rfm["R"] = df_rfm["R"].astype("float").astype("int")
-    df_rfm.drop(columns='Recentdate',inplace=True)
-
-    return df_rfm
-
-if __name__ == '__main__':
-
-    # 取之前多少天登录的数据
-    latest_data_date = get_before_date(data_before_days)
-    get_data_from_mysql(latest_data_date)

user_portrait_analysis/activity_analysis/predict.py

@@ -6,6 +6,7 @@ from config import activity_analysis_path
 logger.info('加载用户活跃度字典')
 user_activity_dict = pickle.load(open(activity_analysis_path + 'results/user_activity_dict.pkl', 'rb'))
 
+
 def user_activity_predict(user_id):
     """
     用户活跃度预测
@@ -17,6 +18,7 @@ def user_activity_predict(user_id):
 
     return result
 
+
 if __name__ == '__main__':
     result = user_activity_predict(user_id=test_user_id)
     print('用户ID：', test_user_id, '贡献度：', result)

user_portrait_analysis/activity_analysis/train.py

@@ -7,6 +7,7 @@ from config import RANDOM_SEED
 from data_process import get_rfm_data
 from config import activity_analysis_path
 
+
 def iflabel(x):
     if x == "高高高":
         return 5
@@ -25,6 +26,7 @@ def iflabel(x):
     elif x == "低低低":
         return 1
 
+
 def train():
     """
     用户活跃度分析模型训练
@@ -34,21 +36,21 @@ def train():
     df_rfm = get_rfm_data()
     rfm_cluster = df_rfm.copy()
     # 根据 R,F,M 值将所有数据分为 8 类
-    Cluster = df_rfm[["R","F","M"]].values
-    kmeans_model = KMeans(n_clusters=8, random_state=RANDOM_SEED).fit(Cluster)   
+    Cluster = df_rfm[["R", "F", "M"]].values
+    kmeans_model = KMeans(n_clusters=8, random_state=RANDOM_SEED).fit(Cluster)
 
     rfm_cluster["label"] = kmeans_model.labels_
 
     # 此处计算均值 RFM 模型，每个用户的R、F、M值同总体均值进行比较，比均值高的就为“高”，低的就标明“低”，然后通过三个指标的高低来打标签。
-    df_rfm_mean = rfm_cluster[["label","R","F","M"]].groupby("label").agg("mean")
+    df_rfm_mean = rfm_cluster[["label", "R", "F", "M"]].groupby("label").agg("mean")
     df_rfm_mean["IFR"] = df_rfm_mean["R"].apply(lambda x: "高" if x > df_rfm_mean["R"].mean() else "低")
     df_rfm_mean["IFF"] = df_rfm_mean["F"].apply(lambda x: "高" if x > df_rfm_mean["F"].mean() else "低")
-    df_rfm_mean["IFM"] = df_rfm_mean["M"].apply(lambda x: "高" if x > df_rfm_mean["M"].mean()  else "低")
-    
-    #方法和上面一样，#将三个指标合并起来，生成临时列temp，表示RFM综合指标
-    df_rfm_mean["temp"] = df_rfm_mean["IFR"] +df_rfm_mean["IFF"]+df_rfm_mean["IFM"]
+    df_rfm_mean["IFM"] = df_rfm_mean["M"].apply(lambda x: "高" if x > df_rfm_mean["M"].mean() else "低")
+
+    # 方法和上面一样，#将三个指标合并起来，生成临时列temp，表示RFM综合指标
+    df_rfm_mean["temp"] = df_rfm_mean["IFR"] + df_rfm_mean["IFF"] + df_rfm_mean["IFM"]
     # 对指标进行综合判断
-    df_rfm_mean["label"] = df_rfm_mean["temp"].apply(lambda x:  iflabel(x))
+    df_rfm_mean["label"] = df_rfm_mean["temp"].apply(lambda x: iflabel(x))
     # 此处在使用聚类RFM模型，比较的时候使用总体均值进行比较
     rfm_cluster["IFR"] = rfm_cluster["R"].apply(lambda x: "高" if x > df_rfm_mean["R"].mean() else "低")
     rfm_cluster["IFF"] = rfm_cluster["F"].apply(lambda x: "高" if x > df_rfm_mean["F"].mean() else "低")
@@ -60,21 +62,20 @@ def train():
 
     result = rfm_cluster[['label']]
 
-    result_csv = pd.DataFrame(list(result.index),index=range(len(list(result.index))),columns = ['user_id'])
-    result_csv['activity'] = list(result.values.reshape(1,len(list(result.index)))[0])
+    result_csv = pd.DataFrame(list(result.index), index=range(len(list(result.index))), columns=['user_id'])
+    result_csv['activity'] = list(result.values.reshape(1, len(list(result.index)))[0])
     # 将用户 id 和活跃度分值提取出来并创新建立一个表     
     result_csv['user_id'] = result_csv['user_id'].astype(int)
     result_csv['activity'] = result_csv['activity'].astype(int)
-    resuts_dict  = dict(zip(result_csv['user_id'], result_csv['activity']))
+    resuts_dict = dict(zip(result_csv['user_id'], result_csv['activity']))
 
     # 保存模型和结果
     pickle.dump(kmeans_model, open(activity_analysis_path + 'results/user_activity_model.pkl', 'wb'))
     pickle.dump(resuts_dict, open(activity_analysis_path + 'results/user_activity_dict.pkl', 'wb'))
 
     logger.info('用户活跃度分析模型训练完成')
-
     return resuts_dict
 
 
 if __name__ == '__main__':
-    train()
+    train()

user_portrait_analysis/utils.py

@@ -1,7 +1,6 @@
 import os
 import logging
 import random
-import torch
 import numpy as np
 import datetime
 
@@ -45,16 +44,6 @@ def get_file_size(fname):
     fsize = fsize/float(1024 * 1024)
     return round(fsize, 2)
 
-def set_seed(seed):
-    """
-    设置随机数种子
-    """
-    random.seed(seed)
-    np.random.seed(seed)
-    torch.manual_seed(seed)
-    if torch.cuda.is_available():
-        torch.cuda.manual_seed_all(seed)
-
 def get_before_date(n):
     """
     获取前N天的日期