67 lines
2.3 KiB
Python
67 lines
2.3 KiB
Python
import warnings
|
|
import pandas as pd
|
|
from datetime import datetime
|
|
from sqlalchemy import create_engine
|
|
from utils import get_month_date
|
|
from config import host, port, user, passwd, database
|
|
warnings.filterwarnings("ignore")
|
|
|
|
|
|
def generate_statistics_data_by_test():
|
|
"""
|
|
按月生成历史数据
|
|
:return:
|
|
"""
|
|
engine = create_engine(f'postgresql+psycopg2://{user}:{passwd}@{host}:{port}/{database}', pool_recycle=3600)
|
|
conn = engine.connect()
|
|
|
|
months = [7, 8]
|
|
for month in months:
|
|
for date in get_month_date(2022, month):
|
|
generate_statistics_data_single(conn, date)
|
|
|
|
|
|
def generate_statistics_data():
|
|
"""
|
|
生成当天数据
|
|
:return:
|
|
"""
|
|
engine = create_engine(f'postgresql+psycopg2://{user}:{passwd}@{host}:{port}/{database}', pool_recycle=3600)
|
|
conn = engine.connect()
|
|
|
|
generate_statistics_data_single(conn, datetime.now().strftime('%Y-%m-%d'))
|
|
|
|
|
|
def generate_statistics_data_single(conn, date):
|
|
sql = f"""select * from ai_repository_history where date_str = '{date}'"""
|
|
all_df = pd.read_sql(sql, conn)
|
|
|
|
# 发展趋势
|
|
tmp_df = all_df[['num_watches', 'num_stars', 'num_forks', 'clone_cnt', 'num_issues', 'num_pulls', 'num_commit']]
|
|
all_df['sum'] = tmp_df.sum(axis=1)
|
|
all_df['all_avg'] = all_df['sum'].mean()
|
|
all_df = all_df.round(2)
|
|
db_df = all_df[['repo_id', 'date_str', 'sum', 'all_avg']]
|
|
db_df.insert(2, 'type', 'repo_trend')
|
|
db_df.insert(3, 'add_time', datetime.now())
|
|
db_df = db_df.rename(columns={'sum': 'value', 'all_avg': 'all_value'})
|
|
db_df.to_sql('ai_repo_trend_statistics', conn, index=False, if_exists='append')
|
|
|
|
# 开源潜力
|
|
tmp_df = all_df[['num_watches', 'num_stars', 'num_forks', 'clone_cnt']]
|
|
all_df['sum'] = tmp_df.sum(axis=1)
|
|
all_df['all_avg'] = all_df['sum'].mean()
|
|
all_df = all_df.round(2)
|
|
db_df = all_df[['repo_id', 'date_str', 'sum', 'all_avg']]
|
|
db_df.insert(2, 'type', 'repo_potential')
|
|
db_df.insert(3, 'add_time', datetime.now())
|
|
db_df = db_df.rename(columns={'sum': 'value', 'all_avg': 'all_value'})
|
|
db_df.to_sql('ai_repo_trend_statistics', conn, index=False, if_exists='append')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
# 生成测试数据
|
|
generate_statistics_data_by_test()
|
|
|
|
# 生成正式数据
|
|
# generate_statistics_data() |