89 lines
3.4 KiB
Python
89 lines
3.4 KiB
Python
import sys
|
|
import os
|
|
|
|
import config
|
|
|
|
curPath = os.path.abspath(os.path.dirname(__file__))
|
|
rootPath = os.path.split(curPath)[0]
|
|
sys.path.append(rootPath)
|
|
|
|
import warnings
|
|
import pandas as pd
|
|
from datetime import datetime
|
|
from sqlalchemy import create_engine
|
|
from utils import get_before_date, compare_date, get_between_dates
|
|
from config import host, port, user, passwd, database
|
|
warnings.filterwarnings("ignore")
|
|
|
|
|
|
def generate_statistics_data():
|
|
"""
|
|
生成数据
|
|
:return:
|
|
"""
|
|
engine = create_engine(f'postgresql+psycopg2://{user}:{passwd}@{host}:{port}/{database}', pool_recycle=3600)
|
|
conn = engine.connect()
|
|
|
|
query_last_date_sql = f"""
|
|
select date_str from ai_repo_trend_statistics t group by date_str order by date_str desc limit 1
|
|
"""
|
|
query_last_date_df = pd.read_sql(query_last_date_sql, conn)
|
|
|
|
# 生成距今最多60天以前的数据
|
|
last_date = ''
|
|
before_60_day_date = get_before_date(60)
|
|
if not query_last_date_df.empty:
|
|
last_date = query_last_date_df.iloc[0, 0]
|
|
if compare_date(last_date, before_60_day_date):
|
|
last_date = before_60_day_date
|
|
else:
|
|
last_date = before_60_day_date
|
|
|
|
current_date = datetime.now().strftime('%Y-%m-%d')
|
|
date_list = get_between_dates(last_date, current_date)
|
|
|
|
if len(date_list) > 0:
|
|
for date in date_list:
|
|
if config.generate_fake_data_enable == False:
|
|
if date == current_date:
|
|
generate_statistics_data_single(conn, date)
|
|
else:
|
|
config.logger.info(f"因generate_fake_data_enable=False,故取消插入日期{date}的statistics数据")
|
|
continue
|
|
generate_statistics_data_single(conn, date)
|
|
|
|
config.logger.info(f"统计个人画像数据完成")
|
|
|
|
def generate_statistics_data_single(conn, date):
|
|
config.logger.info(f"统计个人画像数据：{date}")
|
|
sql = f"""select * from ai_repository_history where date_str = '{date}'"""
|
|
all_df = pd.read_sql(sql, conn)
|
|
|
|
# 发展趋势
|
|
tmp_df = all_df[['num_watches', 'num_stars', 'num_forks', 'clone_cnt', 'num_issues', 'num_pulls', 'num_commit', 'is_test']]
|
|
all_df['sum'] = tmp_df.sum(axis=1)
|
|
all_df['all_avg'] = all_df['sum'].mean()
|
|
all_df = all_df.round(2)
|
|
db_df = all_df[['repo_id', 'date_str', 'num_watches', 'num_stars', 'num_forks', 'clone_cnt', 'num_issues', 'num_pulls', 'num_commit', 'is_test', 'sum', 'all_avg']]
|
|
db_df.insert(2, 'type', 'repo_trend')
|
|
db_df.insert(3, 'add_time', datetime.now())
|
|
db_df = db_df.rename(columns={'sum': 'sum_value', 'all_avg': 'all_repo_avg_value'})
|
|
db_df.to_sql('ai_repo_trend_statistics', conn, index=False, if_exists='append')
|
|
|
|
# 开源潜力
|
|
tmp_df = all_df[['num_watches', 'num_stars', 'num_forks', 'clone_cnt', 'is_test']]
|
|
all_df['sum'] = tmp_df.sum(axis=1)
|
|
all_df['all_avg'] = all_df['sum'].mean()
|
|
all_df = all_df.round(2)
|
|
db_df = all_df[['repo_id', 'date_str', 'num_watches', 'num_stars', 'num_forks', 'clone_cnt', 'is_test', 'sum', 'all_avg']]
|
|
db_df.insert(2, 'type', 'repo_potential')
|
|
db_df.insert(3, 'add_time', datetime.now())
|
|
db_df.insert(4, 'num_issues', 0)
|
|
db_df.insert(5, 'num_pulls', 0)
|
|
db_df.insert(6, 'num_commit', 0)
|
|
db_df = db_df.rename(columns={'sum': 'sum_value', 'all_avg': 'all_repo_avg_value'})
|
|
db_df.to_sql('ai_repo_trend_statistics', conn, index=False, if_exists='append')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
generate_statistics_data() |