From 64830b0dbfdf803ba552260fc48fb0338932615b Mon Sep 17 00:00:00 2001 From: crystal20277 Date: Wed, 22 Feb 2023 17:30:51 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E4=B8=80=E5=A4=A9=E9=87=8D?= =?UTF-8?q?=E5=A4=8D=E6=8F=92=E5=85=A5=E6=95=B0=E6=8D=AEbug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../generate_history_data.py | 129 +++++++++--------- .../generate_statistics_data.py | 54 ++++---- utils.py | 22 ++- 3 files changed, 114 insertions(+), 91 deletions(-) diff --git a/project_evaluation_analysis/generate_history_data.py b/project_evaluation_analysis/generate_history_data.py index 1bf26be..a1076fe 100644 --- a/project_evaluation_analysis/generate_history_data.py +++ b/project_evaluation_analysis/generate_history_data.py @@ -6,82 +6,83 @@ sys.path.append(rootPath) import warnings from datetime import datetime -from utils import get_conn, get_month_date +from utils import get_conn, get_before_date, compare_date, get_between_dates warnings.filterwarnings("ignore") -def generate_history_data_by_test(): - """ - 按月生成历史数据 - :return: - """ - conn = get_conn() - curs = conn.cursor() - - months = [1] - for month in months: - date_list = get_month_date(2023, month) - for date in date_list: - sql = f""" - INSERT INTO - ai_repository_history - SELECT - nextval('ai_repository_history_id_seq') as id, owner_id, owner_name, lower_name, name, description, - website, original_service_type, original_url, default_branch, creator_id, - (ceil(random() * 10) + num_watches) as num_watches, - (ceil(random() * 10) + num_stars) as num_stars, - (ceil(random() * 10) + num_forks) as num_forks, - (ceil(random() * 10) + num_issues) as num_issues, - (ceil(random() * 10) + num_closed_issues) as num_closed_issues, - (ceil(random() * 10) + num_pulls) as num_pulls, - (ceil(random() * 10) + num_closed_pulls) as num_closed_pulls, - num_milestones, num_closed_milestones, - (ceil(random() * 10) + num_commit) as num_commit, repo_type, is_private, is_empty, - is_archived, is_mirror, status, is_fork, fork_id, is_template, template_id, size, - is_fsck_enabled, close_issues_via_commit_in_any_branch, topics, avatar, contract_address, - balance, block_chain_status, - (ceil(random() * 10) + clone_cnt) as clone_cnt, git_clone_cnt, created_unix, updated_unix, - alias, lower_alias, id as repo_id, '{date}' as date_str - FROM - repository - """ - curs.execute(sql) - - conn.commit() - conn.close() - - def generate_history_data(): """ - 生成当天数据 + 生成数据 :return: """ conn = get_conn() curs = conn.cursor() - date = datetime.now().strftime('%Y-%m-%d') - sql = f""" - INSERT INTO - ai_repository_history - SELECT - nextval('ai_repository_history_id_seq') as id, owner_id, owner_name, lower_name, name, description, - website, original_service_type, original_url, default_branch, creator_id, - num_watches, num_stars, num_forks, num_issues, num_closed_issues, num_pulls, num_closed_pulls, - num_milestones, num_closed_milestones, num_commit, repo_type, is_private, is_empty, - is_archived, is_mirror, status, is_fork, fork_id, is_template, template_id, size, - is_fsck_enabled, close_issues_via_commit_in_any_branch, topics, avatar, contract_address, - balance, block_chain_status, clone_cnt, git_clone_cnt, created_unix, updated_unix, - alias, lower_alias, id as repo_id, '{date}' as date_str - FROM - repository + + query_last_date_sql = f""" + select date_str from ai_repository_history t group by date_str order by date_str desc limit 1 """ - curs.execute(sql) + curs.execute(query_last_date_sql) + rows = curs.fetchall() + + # 生成距今最多60天以前的数据 + last_date = '' + before_60_day_date = get_before_date(60) + if len(rows) > 0: + last_date = rows[0][0] + if compare_date(last_date, before_60_day_date): + last_date = before_60_day_date + else: + last_date = before_60_day_date + + current_date = datetime.now().strftime('%Y-%m-%d') + date_list = get_between_dates(last_date, current_date) + + if len(date_list) > 0: + for date in date_list: + if current_date == date: + sql = f""" + INSERT INTO + ai_repository_history + SELECT + nextval('ai_repository_history_id_seq') as id, owner_id, owner_name, lower_name, name, description, + website, original_service_type, original_url, default_branch, creator_id, + num_watches, num_stars, num_forks, num_issues, num_closed_issues, num_pulls, num_closed_pulls, + num_milestones, num_closed_milestones, num_commit, repo_type, is_private, is_empty, + is_archived, is_mirror, status, is_fork, fork_id, is_template, template_id, size, + is_fsck_enabled, close_issues_via_commit_in_any_branch, topics, avatar, contract_address, + balance, block_chain_status, clone_cnt, git_clone_cnt, created_unix, updated_unix, + alias, lower_alias, id as repo_id, '{date}' as date_str, {False} + FROM + repository + """ + else: + sql = f""" + INSERT INTO + ai_repository_history + SELECT + nextval('ai_repository_history_id_seq') as id, owner_id, owner_name, lower_name, name, description, + website, original_service_type, original_url, default_branch, creator_id, + (ceil(random() * 10) + num_watches) as num_watches, + (ceil(random() * 10) + num_stars) as num_stars, + (ceil(random() * 10) + num_forks) as num_forks, + (ceil(random() * 10) + num_issues) as num_issues, + (ceil(random() * 10) + num_closed_issues) as num_closed_issues, + (ceil(random() * 10) + num_pulls) as num_pulls, + (ceil(random() * 10) + num_closed_pulls) as num_closed_pulls, + num_milestones, num_closed_milestones, + (ceil(random() * 10) + num_commit) as num_commit, repo_type, is_private, is_empty, + is_archived, is_mirror, status, is_fork, fork_id, is_template, template_id, size, + is_fsck_enabled, close_issues_via_commit_in_any_branch, topics, avatar, contract_address, + balance, block_chain_status, + (ceil(random() * 10) + clone_cnt) as clone_cnt, git_clone_cnt, created_unix, updated_unix, + alias, lower_alias, id as repo_id, '{date}' as date_str, {True} + FROM + repository + """ + curs.execute(sql) conn.commit() conn.close() if __name__ == '__main__': - # 生成测试数据 - generate_history_data_by_test() - - # 生成正式数据 - # generate_history_data() \ No newline at end of file + generate_history_data() \ No newline at end of file diff --git a/project_evaluation_analysis/generate_statistics_data.py b/project_evaluation_analysis/generate_statistics_data.py index c085bcf..e8c8f74 100644 --- a/project_evaluation_analysis/generate_statistics_data.py +++ b/project_evaluation_analysis/generate_statistics_data.py @@ -8,34 +8,40 @@ import warnings import pandas as pd from datetime import datetime from sqlalchemy import create_engine -from utils import get_month_date +from utils import get_before_date, compare_date, get_between_dates from config import host, port, user, passwd, database warnings.filterwarnings("ignore") -def generate_statistics_data_by_test(): - """ - 按月生成历史数据 - :return: - """ - engine = create_engine(f'postgresql+psycopg2://{user}:{passwd}@{host}:{port}/{database}', pool_recycle=3600) - conn = engine.connect() - - months = [1] - for month in months: - for date in get_month_date(2023, month): - generate_statistics_data_single(conn, date) - - def generate_statistics_data(): """ - 生成当天数据 + 生成数据 :return: """ engine = create_engine(f'postgresql+psycopg2://{user}:{passwd}@{host}:{port}/{database}', pool_recycle=3600) conn = engine.connect() - generate_statistics_data_single(conn, datetime.now().strftime('%Y-%m-%d')) + query_last_date_sql = f""" + select date_str from ai_repo_trend_statistics t group by date_str order by date_str desc limit 1 + """ + query_last_date_df = pd.read_sql(query_last_date_sql, conn) + + # 生成距今最多60天以前的数据 + last_date = '' + before_60_day_date = get_before_date(60) + if not query_last_date_df.empty: + last_date = query_last_date_df.iloc[0, 0] + if compare_date(last_date, before_60_day_date): + last_date = before_60_day_date + else: + last_date = before_60_day_date + + current_date = datetime.now().strftime('%Y-%m-%d') + date_list = get_between_dates(last_date, current_date) + + if len(date_list) > 0: + for date in date_list: + generate_statistics_data_single(conn, date) def generate_statistics_data_single(conn, date): @@ -43,22 +49,22 @@ def generate_statistics_data_single(conn, date): all_df = pd.read_sql(sql, conn) # 发展趋势 - tmp_df = all_df[['num_watches', 'num_stars', 'num_forks', 'clone_cnt', 'num_issues', 'num_pulls', 'num_commit']] + tmp_df = all_df[['num_watches', 'num_stars', 'num_forks', 'clone_cnt', 'num_issues', 'num_pulls', 'num_commit', 'is_test']] all_df['sum'] = tmp_df.sum(axis=1) all_df['all_avg'] = all_df['sum'].mean() all_df = all_df.round(2) - db_df = all_df[['repo_id', 'date_str', 'num_watches', 'num_stars', 'num_forks', 'clone_cnt', 'num_issues', 'num_pulls', 'num_commit', 'sum', 'all_avg']] + db_df = all_df[['repo_id', 'date_str', 'num_watches', 'num_stars', 'num_forks', 'clone_cnt', 'num_issues', 'num_pulls', 'num_commit', 'is_test', 'sum', 'all_avg']] db_df.insert(2, 'type', 'repo_trend') db_df.insert(3, 'add_time', datetime.now()) db_df = db_df.rename(columns={'sum': 'sum_value', 'all_avg': 'all_repo_avg_value'}) db_df.to_sql('ai_repo_trend_statistics', conn, index=False, if_exists='append') # 开源潜力 - tmp_df = all_df[['num_watches', 'num_stars', 'num_forks', 'clone_cnt']] + tmp_df = all_df[['num_watches', 'num_stars', 'num_forks', 'clone_cnt', 'is_test']] all_df['sum'] = tmp_df.sum(axis=1) all_df['all_avg'] = all_df['sum'].mean() all_df = all_df.round(2) - db_df = all_df[['repo_id', 'date_str', 'num_watches', 'num_stars', 'num_forks', 'clone_cnt', 'sum', 'all_avg']] + db_df = all_df[['repo_id', 'date_str', 'num_watches', 'num_stars', 'num_forks', 'clone_cnt', 'is_test', 'sum', 'all_avg']] db_df.insert(2, 'type', 'repo_potential') db_df.insert(3, 'add_time', datetime.now()) db_df.insert(4, 'num_issues', 0) @@ -69,8 +75,4 @@ def generate_statistics_data_single(conn, date): if __name__ == '__main__': - # 生成测试数据 - generate_statistics_data_by_test() - - # 生成正式数据 - # generate_statistics_data() \ No newline at end of file + generate_statistics_data() \ No newline at end of file diff --git a/utils.py b/utils.py index 7584d7b..ebc5feb 100644 --- a/utils.py +++ b/utils.py @@ -71,4 +71,24 @@ def get_before_date(n): # 计算偏移量 offset = datetime.timedelta(days=-n) re_date = (today + offset).strftime('%Y-%m-%d') - return re_date \ No newline at end of file + return re_date + + +def compare_date(date1, date2): + return date1 < date2 + + +def get_between_dates(begin_date_str, end_date_str): + date_list = [] + begin_date = datetime.datetime.strptime(begin_date_str, "%Y-%m-%d") + end_date = datetime.datetime.strptime(end_date_str, "%Y-%m-%d") + while begin_date < end_date: + begin_date += datetime.timedelta(days=1) + date_list.append(begin_date.strftime('%Y-%m-%d')) + return date_list + + +if __name__ == '__main__': + # print(get_before_date(60)) + # print(compare_date('2023-01-05', '2023-01-03')) + print(get_between_dates('2023-02-09', '2023-03-04')) \ No newline at end of file