207 lines
7.0 KiB
Python
207 lines
7.0 KiB
Python
#!/usr/bin/env python3
|
|
|
|
from github import Github
|
|
from psycopg2 import sql
|
|
import os
|
|
import logging
|
|
import psycopg2
|
|
import dateutil
|
|
import argparse
|
|
|
|
def init_logger():
|
|
LOGLEVEL = os.environ.get('LOGLEVEL', 'INFO').upper()
|
|
logging.basicConfig(level=LOGLEVEL,
|
|
format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
|
|
datefmt='%m-%d-%Y %H:%M:%S')
|
|
|
|
def make_parser():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument('-r', '--repository-name', type=str, required=True,
|
|
help='Repository name in OWNER/REPOSITORY format')
|
|
parser.add_argument('--run-id', type=str, required=True,
|
|
help='Workflow Run ID')
|
|
|
|
return parser
|
|
|
|
def create_db_tables(conn, cur):
|
|
cur.execute('''CREATE TABLE IF NOT EXISTS workflow_runs(
|
|
id SERIAL PRIMARY KEY,
|
|
run_id BIGINT,
|
|
html_url TEXT,
|
|
name VARCHAR(255),
|
|
run_started_at TIMESTAMP,
|
|
created_at TIMESTAMP,
|
|
updated_at TIMESTAMP,
|
|
triggering_actor_login VARCHAR(255),
|
|
conclusion VARCHAR(25),
|
|
event VARCHAR(50),
|
|
run_attempt INT,
|
|
repository_full_name VARCHAR(255),
|
|
head_repository_full_name VARCHAR(255),
|
|
head_branch VARCHAR(255),
|
|
status VARCHAR(25),
|
|
display_title TEXT,
|
|
path TEXT,
|
|
total_duration_seconds INT
|
|
);
|
|
''')
|
|
cur.execute('''CREATE TABLE IF NOT EXISTS workflow_jobs(
|
|
id SERIAL PRIMARY KEY,
|
|
job_id BIGINT,
|
|
parent_run_id BIGINT,
|
|
html_url TEXT,
|
|
name VARCHAR(255),
|
|
created_at TIMESTAMP,
|
|
started_at TIMESTAMP,
|
|
completed_at TIMESTAMP,
|
|
queued_duration_seconds INT,
|
|
duration_seconds INT,
|
|
runner_name VARCHAR(255),
|
|
status VARCHAR(25),
|
|
conclusion VARCHAR(25),
|
|
head_branch VARCHAR(255),
|
|
run_attempt INT,
|
|
workflow_name TEXT
|
|
);
|
|
''')
|
|
cur.execute('''CREATE TABLE IF NOT EXISTS workflow_steps(
|
|
id SERIAL PRIMARY KEY,
|
|
parent_job_id BIGINT,
|
|
name VARCHAR(255),
|
|
conclusion VARCHAR(25),
|
|
number INT,
|
|
started_at TIMESTAMP,
|
|
completed_at TIMESTAMP,
|
|
duration_seconds INT
|
|
);
|
|
''')
|
|
conn.commit()
|
|
|
|
def main():
|
|
init_logger()
|
|
parser = make_parser()
|
|
args = parser.parse_args()
|
|
logger = logging.getLogger(__name__)
|
|
|
|
github_token = os.environ.get('GITHUB_TOKEN')
|
|
if not github_token:
|
|
raise ValueError('GITHUB_TOKEN environment variable is not set!')
|
|
|
|
run_id = args.run_id
|
|
repo_name = args.repository_name
|
|
|
|
|
|
# this should be specified in runner's env
|
|
db_username = os.environ.get('PGUSER')
|
|
db_password = os.environ.get('PGPASSWORD')
|
|
db_host = os.environ.get('PGHOST')
|
|
db_database = os.environ.get('PGDATABASE')
|
|
db_port = os.environ.get('PGPORT')
|
|
conn = psycopg2.connect(host=db_host,
|
|
port=db_port,
|
|
user=db_username,
|
|
password=db_password,
|
|
database=db_database)
|
|
|
|
# Create tables
|
|
cur = conn.cursor()
|
|
create_db_tables(conn, cur)
|
|
|
|
# Get the data
|
|
g = Github(github_token)
|
|
repo = g.get_repo(repo_name)
|
|
|
|
run = repo.get_workflow_run(int(run_id))
|
|
logger.debug('Processing run ID %s - %s', run_id, run.name)
|
|
if run.status != 'completed':
|
|
logger.error('Run %s is not completed! Only completed runs should be in the database', run_id)
|
|
raise SystemExit(1)
|
|
|
|
# We rely on the following assumptions:
|
|
# - The workflow run is completed. When run.status != 'completed' we should not add it to the database
|
|
# theoretically the second attempt can be triggerred right after the completion of the first one
|
|
# or while the runner which executes this script is deploying
|
|
#
|
|
# - Job's queued duration equals "job.started_at - job.created_at" if started_at > created_at.
|
|
# Otherwise the job should not be added to the database
|
|
total_duration_seconds = round(run.timing().run_duration_ms / 1000)
|
|
workflow_data_query = f'''INSERT INTO workflow_runs(
|
|
run_id, html_url, name,
|
|
run_started_at, created_at, updated_at, triggering_actor_login, conclusion,
|
|
event, run_attempt, repository_full_name,
|
|
head_branch, display_title, path, total_duration_seconds)
|
|
VALUES(
|
|
'{run_id}', '{run.html_url}', '{run.name}', '{run.run_started_at}',
|
|
'{run.created_at}', '{run.updated_at}',
|
|
'{run.raw_data['triggering_actor']['login']}',
|
|
'{run.conclusion}', '{run.event}',
|
|
'{run.run_attempt}', '{run.raw_data['repository']['full_name']}',
|
|
'{run.head_branch}', '{run.display_title}', '{run.path}', '{total_duration_seconds}'
|
|
);
|
|
'''
|
|
|
|
logger.debug('Workflow run query: %s', workflow_data_query)
|
|
cur.execute(workflow_data_query)
|
|
|
|
for job in run.jobs():
|
|
job_id = job.id
|
|
logger.debug('Processing job %s', job.name)
|
|
queued_duration_seconds = 0
|
|
duration_seconds = 0
|
|
|
|
job_created_at_date = dateutil.parser.parse(job.raw_data['created_at'])
|
|
if job_created_at_date > job.started_at:
|
|
logger.warning('Skipping job %s of run %s - most likely a stub \
|
|
job created after workflow restart', job.name, run_id)
|
|
continue
|
|
|
|
queued_duration_timedelta = job.started_at - job_created_at_date
|
|
queued_duration_seconds = round(queued_duration_timedelta.total_seconds())
|
|
|
|
duration_timedelta = job.completed_at - job.started_at
|
|
duration_seconds = round(duration_timedelta.total_seconds())
|
|
|
|
job_data_query = f'''
|
|
INSERT INTO workflow_jobs(
|
|
job_id, parent_run_id, html_url, name,
|
|
created_at, started_at, completed_at,
|
|
queued_duration_seconds, duration_seconds,
|
|
runner_name, status, conclusion, head_branch,
|
|
run_attempt, workflow_name
|
|
)
|
|
VALUES(
|
|
'{job_id}', '{run_id}', '{job.html_url}', '{job.name}',
|
|
'{job.raw_data['created_at']}', '{job.started_at}', '{job.completed_at}',
|
|
'{queued_duration_seconds}', '{duration_seconds}',
|
|
'{job.raw_data['runner_name']}', '{job.status}', '{job.conclusion}',
|
|
'{job.raw_data['head_branch']}', '{job.raw_data['run_attempt']}', '{job.raw_data['workflow_name']}'
|
|
);
|
|
'''
|
|
logger.debug('Job query: %s', job_data_query)
|
|
cur.execute(job_data_query)
|
|
for step in job.steps:
|
|
logger.debug('Processing step %s', step.name)
|
|
duration_seconds_timedelta = step.completed_at - step.started_at
|
|
duration_seconds = round(duration_seconds_timedelta.total_seconds())
|
|
|
|
step_data_query = f'''
|
|
INSERT INTO workflow_steps(
|
|
parent_job_id, name, conclusion,
|
|
number, started_at, completed_at,
|
|
duration_seconds)
|
|
VALUES(
|
|
'{job_id}', '{step.name}','{step.conclusion}',
|
|
'{step.number}', '{step.started_at}', '{step.completed_at}',
|
|
'{duration_seconds}'
|
|
);
|
|
'''
|
|
logger.debug('Step query: %s', step_data_query)
|
|
cur.execute(step_data_query)
|
|
|
|
conn.commit()
|
|
cur.close()
|
|
conn.close()
|
|
g.close()
|
|
if __name__ == "__main__":
|
|
main()
|