wangwei10061
/
aiforge_python
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

修改一天重复插入数据bug

This commit is contained in:
crystal20277 2023-02-22 17:30:51 +08:00
parent dadf850385
commit 64830b0dbf
3 changed files with 114 additions and 91 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

129
project_evaluation_analysis/generate_history_data.py
View File

@ -6,82 +6,83 @@ sys.path.append(rootPath)
import warnings
from datetime import datetime
from utils import get_conn, get_month_date
from utils import get_conn, get_before_date, compare_date, get_between_dates
warnings.filterwarnings("ignore")
def generate_history_data_by_test():
"""
按月生成历史数据
:return:
"""
conn = get_conn()
curs = conn.cursor()
months = [1]
for month in months:
date_list = get_month_date(2023, month)
for date in date_list:
sql = f"""
INSERT INTO
ai_repository_history
SELECT
nextval('ai_repository_history_id_seq') as id, owner_id, owner_name, lower_name, name, description,
website, original_service_type, original_url, default_branch, creator_id,
(ceil(random() * 10) + num_watches) as num_watches,
(ceil(random() * 10) + num_stars) as num_stars,
(ceil(random() * 10) + num_forks) as num_forks,
(ceil(random() * 10) + num_issues) as num_issues,
(ceil(random() * 10) + num_closed_issues) as num_closed_issues,
(ceil(random() * 10) + num_pulls) as num_pulls,
(ceil(random() * 10) + num_closed_pulls) as num_closed_pulls,
num_milestones, num_closed_milestones,
(ceil(random() * 10) + num_commit) as num_commit, repo_type, is_private, is_empty,
is_archived, is_mirror, status, is_fork, fork_id, is_template, template_id, size,
is_fsck_enabled, close_issues_via_commit_in_any_branch, topics, avatar, contract_address,
balance, block_chain_status,
(ceil(random() * 10) + clone_cnt) as clone_cnt, git_clone_cnt, created_unix, updated_unix,
alias, lower_alias, id as repo_id, '{date}' as date_str
FROM
repository
"""
curs.execute(sql)
conn.commit()
conn.close()
def generate_history_data():
"""
生成当天数据
生成数据
:return:
"""
conn = get_conn()
curs = conn.cursor()
date = datetime.now().strftime('%Y-%m-%d')
sql = f"""
INSERT INTO
ai_repository_history
SELECT
nextval('ai_repository_history_id_seq') as id, owner_id, owner_name, lower_name, name, description,
website, original_service_type, original_url, default_branch, creator_id,
num_watches, num_stars, num_forks, num_issues, num_closed_issues, num_pulls, num_closed_pulls,
num_milestones, num_closed_milestones, num_commit, repo_type, is_private, is_empty,
is_archived, is_mirror, status, is_fork, fork_id, is_template, template_id, size,
is_fsck_enabled, close_issues_via_commit_in_any_branch, topics, avatar, contract_address,
balance, block_chain_status, clone_cnt, git_clone_cnt, created_unix, updated_unix,
alias, lower_alias, id as repo_id, '{date}' as date_str
FROM
repository
query_last_date_sql = f"""
select date_str from ai_repository_history t group by date_str order by date_str desc limit 1
"""
curs.execute(sql)
curs.execute(query_last_date_sql)
rows = curs.fetchall()
# 生成距今最多60天以前的数据
last_date = ''
before_60_day_date = get_before_date(60)
if len(rows) > 0:
last_date = rows[0][0]
if compare_date(last_date, before_60_day_date):
last_date = before_60_day_date
else:
last_date = before_60_day_date
current_date = datetime.now().strftime('%Y-%m-%d')
date_list = get_between_dates(last_date, current_date)
if len(date_list) > 0:
for date in date_list:
if current_date == date:
sql = f"""
INSERT INTO
ai_repository_history
SELECT
nextval('ai_repository_history_id_seq') as id, owner_id, owner_name, lower_name, name, description,
website, original_service_type, original_url, default_branch, creator_id,
num_watches, num_stars, num_forks, num_issues, num_closed_issues, num_pulls, num_closed_pulls,
num_milestones, num_closed_milestones, num_commit, repo_type, is_private, is_empty,
is_archived, is_mirror, status, is_fork, fork_id, is_template, template_id, size,
is_fsck_enabled, close_issues_via_commit_in_any_branch, topics, avatar, contract_address,
balance, block_chain_status, clone_cnt, git_clone_cnt, created_unix, updated_unix,
alias, lower_alias, id as repo_id, '{date}' as date_str, {False}
FROM
repository
"""
else:
sql = f"""
INSERT INTO
ai_repository_history
SELECT
nextval('ai_repository_history_id_seq') as id, owner_id, owner_name, lower_name, name, description,
website, original_service_type, original_url, default_branch, creator_id,
(ceil(random() * 10) + num_watches) as num_watches,
(ceil(random() * 10) + num_stars) as num_stars,
(ceil(random() * 10) + num_forks) as num_forks,
(ceil(random() * 10) + num_issues) as num_issues,
(ceil(random() * 10) + num_closed_issues) as num_closed_issues,
(ceil(random() * 10) + num_pulls) as num_pulls,
(ceil(random() * 10) + num_closed_pulls) as num_closed_pulls,
num_milestones, num_closed_milestones,
(ceil(random() * 10) + num_commit) as num_commit, repo_type, is_private, is_empty,
is_archived, is_mirror, status, is_fork, fork_id, is_template, template_id, size,
is_fsck_enabled, close_issues_via_commit_in_any_branch, topics, avatar, contract_address,
balance, block_chain_status,
(ceil(random() * 10) + clone_cnt) as clone_cnt, git_clone_cnt, created_unix, updated_unix,
alias, lower_alias, id as repo_id, '{date}' as date_str, {True}
FROM
repository
"""
curs.execute(sql)
conn.commit()
conn.close()
if __name__ == '__main__':
# 生成测试数据
generate_history_data_by_test()
# 生成正式数据
# generate_history_data()
generate_history_data()

54
project_evaluation_analysis/generate_statistics_data.py
View File

@ -8,34 +8,40 @@ import warnings
import pandas as pd
from datetime import datetime
from sqlalchemy import create_engine
from utils import get_month_date
from utils import get_before_date, compare_date, get_between_dates
from config import host, port, user, passwd, database
warnings.filterwarnings("ignore")
def generate_statistics_data_by_test():
"""
按月生成历史数据
:return:
"""
engine = create_engine(f'postgresql+psycopg2://{user}:{passwd}@{host}:{port}/{database}', pool_recycle=3600)
conn = engine.connect()
months = [1]
for month in months:
for date in get_month_date(2023, month):
generate_statistics_data_single(conn, date)
def generate_statistics_data():
"""
生成当天数据
生成数据
:return:
"""
engine = create_engine(f'postgresql+psycopg2://{user}:{passwd}@{host}:{port}/{database}', pool_recycle=3600)
conn = engine.connect()
generate_statistics_data_single(conn, datetime.now().strftime('%Y-%m-%d'))
query_last_date_sql = f"""
select date_str from ai_repo_trend_statistics t group by date_str order by date_str desc limit 1
"""
query_last_date_df = pd.read_sql(query_last_date_sql, conn)
# 生成距今最多60天以前的数据
last_date = ''
before_60_day_date = get_before_date(60)
if not query_last_date_df.empty:
last_date = query_last_date_df.iloc[0, 0]
if compare_date(last_date, before_60_day_date):
last_date = before_60_day_date
else:
last_date = before_60_day_date
current_date = datetime.now().strftime('%Y-%m-%d')
date_list = get_between_dates(last_date, current_date)
if len(date_list) > 0:
for date in date_list:
generate_statistics_data_single(conn, date)
def generate_statistics_data_single(conn, date):
@ -43,22 +49,22 @@ def generate_statistics_data_single(conn, date):
all_df = pd.read_sql(sql, conn)
# 发展趋势
tmp_df = all_df[['num_watches', 'num_stars', 'num_forks', 'clone_cnt', 'num_issues', 'num_pulls', 'num_commit']]
tmp_df = all_df[['num_watches', 'num_stars', 'num_forks', 'clone_cnt', 'num_issues', 'num_pulls', 'num_commit', 'is_test']]
all_df['sum'] = tmp_df.sum(axis=1)
all_df['all_avg'] = all_df['sum'].mean()
all_df = all_df.round(2)
db_df = all_df[['repo_id', 'date_str', 'num_watches', 'num_stars', 'num_forks', 'clone_cnt', 'num_issues', 'num_pulls', 'num_commit', 'sum', 'all_avg']]
db_df = all_df[['repo_id', 'date_str', 'num_watches', 'num_stars', 'num_forks', 'clone_cnt', 'num_issues', 'num_pulls', 'num_commit', 'is_test', 'sum', 'all_avg']]
db_df.insert(2, 'type', 'repo_trend')
db_df.insert(3, 'add_time', datetime.now())
db_df = db_df.rename(columns={'sum': 'sum_value', 'all_avg': 'all_repo_avg_value'})
db_df.to_sql('ai_repo_trend_statistics', conn, index=False, if_exists='append')
# 开源潜力
tmp_df = all_df[['num_watches', 'num_stars', 'num_forks', 'clone_cnt']]
tmp_df = all_df[['num_watches', 'num_stars', 'num_forks', 'clone_cnt', 'is_test']]
all_df['sum'] = tmp_df.sum(axis=1)
all_df['all_avg'] = all_df['sum'].mean()
all_df = all_df.round(2)
db_df = all_df[['repo_id', 'date_str', 'num_watches', 'num_stars', 'num_forks', 'clone_cnt', 'sum', 'all_avg']]
db_df = all_df[['repo_id', 'date_str', 'num_watches', 'num_stars', 'num_forks', 'clone_cnt', 'is_test', 'sum', 'all_avg']]
db_df.insert(2, 'type', 'repo_potential')
db_df.insert(3, 'add_time', datetime.now())
db_df.insert(4, 'num_issues', 0)
@ -69,8 +75,4 @@ def generate_statistics_data_single(conn, date):
if __name__ == '__main__':
# 生成测试数据
generate_statistics_data_by_test()
# 生成正式数据
# generate_statistics_data()
generate_statistics_data()

22
utils.py
View File

@ -71,4 +71,24 @@ def get_before_date(n):
# 计算偏移量
offset = datetime.timedelta(days=-n)
re_date = (today + offset).strftime('%Y-%m-%d')
return re_date
return re_date
def compare_date(date1, date2):
return date1 < date2
def get_between_dates(begin_date_str, end_date_str):
date_list = []
begin_date = datetime.datetime.strptime(begin_date_str, "%Y-%m-%d")
end_date = datetime.datetime.strptime(end_date_str, "%Y-%m-%d")
while begin_date < end_date:
begin_date += datetime.timedelta(days=1)
date_list.append(begin_date.strftime('%Y-%m-%d'))
return date_list
if __name__ == '__main__':
# print(get_before_date(60))
# print(compare_date('2023-01-05', '2023-01-03'))
print(get_between_dates('2023-02-09', '2023-03-04'))