[CI] [GHA] Collect workflow rerunner statistics (#24054)
Based on #23865. ### Tickets: - *138639*
This commit is contained in:
Andrei Kashchikhin
GitHub
parent
fbac30c538
commit
05f06192d6
|
|
@ -13,8 +13,13 @@ def get_arguments() -> argparse.Namespace:
|
|||
required=True,
|
||||
help='Workflow Run ID')
|
||||
parser.add_argument('--errors-to-look-for-file',
|
||||
type=str,
|
||||
type=Path,
|
||||
required=False,
|
||||
help='.json file with the errors to look for in logs',
|
||||
default=Path(__file__).resolve().parent.joinpath('errors_to_look_for.json'))
|
||||
parser.add_argument('--dry-run',
|
||||
required=False,
|
||||
action='store_true',
|
||||
help='Whether to run in dry mode and not actually retrigger the pipeline'
|
||||
' and only collect and analyze logs')
|
||||
return parser.parse_args()
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ class ErrorData(TypedDict):
|
|||
|
||||
|
||||
class LogAnalyzer:
|
||||
def __init__(self,
|
||||
def __init__(self,
|
||||
path_to_log_archive: Path,
|
||||
path_to_errors_file: Path) -> None:
|
||||
self._path_to_log_archive = path_to_log_archive
|
||||
|
|
@ -29,18 +29,18 @@ class LogAnalyzer:
|
|||
self._collect_errors_to_look_for()
|
||||
|
||||
self._log_dir = tempfile.TemporaryDirectory().name
|
||||
|
||||
|
||||
self._log_files: list[LogFile] = []
|
||||
self._collect_log_files()
|
||||
|
||||
all_txt_log_files_pretty = '\n'.join(map(lambda item: str(item['path']), self._log_files))
|
||||
LOGGER.info(f'ALL .txt LOG FILES: \n{all_txt_log_files_pretty}')
|
||||
|
||||
self.found_matching_error = False
|
||||
|
||||
self.found_error_ticket = None
|
||||
|
||||
def _collect_errors_to_look_for(self) -> None:
|
||||
with open(file=self._path_to_errors_file,
|
||||
mode='r',
|
||||
with open(file=self._path_to_errors_file,
|
||||
mode='r',
|
||||
encoding='utf-8') as errors_file:
|
||||
errors_data = json.load(errors_file)
|
||||
for error_data in errors_data:
|
||||
|
|
@ -68,34 +68,34 @@ class LogAnalyzer:
|
|||
|
||||
We need to only analyze the `*.txt` files
|
||||
"""
|
||||
|
||||
with ZipFile(file=self._path_to_log_archive,
|
||||
|
||||
with ZipFile(file=self._path_to_log_archive,
|
||||
mode='r') as zip_file:
|
||||
zip_file.extractall(self._log_dir)
|
||||
|
||||
|
||||
for _file in Path(self._log_dir).iterdir():
|
||||
if _file.is_dir():
|
||||
for log_file in _file.iterdir():
|
||||
self._log_files.append(LogFile(file_name=log_file.name,
|
||||
self._log_files.append(LogFile(file_name=log_file.name,
|
||||
path=log_file.resolve()))
|
||||
|
||||
def _is_error_in_log(self,
|
||||
error_to_look_for: str,
|
||||
def _is_error_in_log(self,
|
||||
error_to_look_for: str,
|
||||
log_file_path: Path) -> bool:
|
||||
"""
|
||||
Searches for the error in the provided log
|
||||
"""
|
||||
|
||||
error_to_look_for = self._clean_up_string(error_to_look_for)
|
||||
error_to_look_for = self._clean_up_string(error_to_look_for)
|
||||
|
||||
with open(file=log_file_path,
|
||||
mode='r',
|
||||
with open(file=log_file_path,
|
||||
mode='r',
|
||||
encoding='utf-8') as log_file:
|
||||
for line in log_file:
|
||||
if error_to_look_for in self._clean_up_string(line):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
@staticmethod
|
||||
def _clean_up_string(string: str) -> str:
|
||||
"""
|
||||
|
|
@ -113,19 +113,21 @@ class LogAnalyzer:
|
|||
Iterates over the known errors and tries to find them in the collected log files
|
||||
"""
|
||||
for error in self._errors_to_look_for:
|
||||
|
||||
|
||||
LOGGER.info(f'LOOKING FOR "{error["error_text"]}" ERROR...')
|
||||
|
||||
|
||||
for log_file in self._log_files:
|
||||
if self._is_error_in_log(error_to_look_for=error['error_text'],
|
||||
if self._is_error_in_log(error_to_look_for=error['error_text'],
|
||||
log_file_path=log_file['path']):
|
||||
LOGGER.info(f'FOUND "{error["error_text"]}" ERROR IN {log_file["path"]}. TICKET: {error["ticket"]}')
|
||||
self.found_matching_error = True
|
||||
self.found_error_ticket = error['ticket']
|
||||
return
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
# Usage example
|
||||
log_analyzer = LogAnalyzer(path_to_log_archive=Path('/tmp/logs/log.zip'),
|
||||
log_analyzer = LogAnalyzer(path_to_log_archive=Path('/tmp/logs/log.zip'),
|
||||
path_to_errors_file=Path('/tmp/errors_to_look_for.json'))
|
||||
log_analyzer.analyze()
|
||||
if log_analyzer.found_matching_error:
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
|
@ -13,19 +14,23 @@ if __name__ == '__main__':
|
|||
args = get_arguments()
|
||||
run_id = args.run_id
|
||||
repository_name = args.repository_name
|
||||
errors_file = args.errors_to_look_for_file
|
||||
is_dry_run = args.dry_run
|
||||
if is_dry_run:
|
||||
LOGGER.info('RUNNING IN DRY RUN MODE. IF ERROR WILL BE FOUND, WILL NOT RETRIGGER')
|
||||
|
||||
github = Github(auth=Auth.Token(token=GITHUB_TOKEN))
|
||||
gh_repo = github.get_repo(full_name_or_id=repository_name)
|
||||
run = gh_repo.get_workflow_run(id_=run_id)
|
||||
|
||||
|
||||
LOGGER.info(f'CHECKING IF RERUN IS NEEDED FOR {run.html_url} RUN IN {repository_name}.')
|
||||
|
||||
|
||||
# Check if the run has already been retriggered
|
||||
# we do not want to fall into a loop with retriggers
|
||||
if run.run_attempt > 1:
|
||||
LOGGER.info(f'THERE ARE {run.run_attempt} ATTEMPTS ALREADY. NOT CHECKING LOGS AND NOT RETRIGGERING. EXITING')
|
||||
sys.exit(0)
|
||||
|
||||
|
||||
log_archive_path = Path(tempfile.NamedTemporaryFile(suffix='.zip').name)
|
||||
|
||||
collect_logs_for_run(
|
||||
|
|
@ -35,18 +40,28 @@ if __name__ == '__main__':
|
|||
|
||||
log_analyzer = LogAnalyzer(
|
||||
path_to_log_archive=log_archive_path,
|
||||
path_to_errors_file=args.error_to_look_for_file,
|
||||
path_to_errors_file=errors_file
|
||||
)
|
||||
log_analyzer.analyze()
|
||||
|
||||
|
||||
if log_analyzer.found_matching_error:
|
||||
LOGGER.info(f'FOUND MATCHING ERROR, RETRIGGERING {run.html_url}')
|
||||
if is_dry_run:
|
||||
LOGGER.info(f'RUNNING IN DRY RUN MODE, NOT RETRIGGERING, EXITING')
|
||||
sys.exit(0)
|
||||
|
||||
status = run.rerun()
|
||||
if status:
|
||||
LOGGER.info(f'RUN RETRIGGERED SUCCESSFULLY: {run.html_url}')
|
||||
else:
|
||||
LOGGER.info(f'RUN WAS NOT RETRIGGERED, SEE ABOVE')
|
||||
|
||||
|
||||
# Needed to run a step after for statistics
|
||||
with open(file=os.environ['GITHUB_ENV'],
|
||||
mode='a') as fh:
|
||||
fh.write('PIPELINE_RETRIGGERED=true')
|
||||
fh.write(f'FOUND_ERROR_TICKET={log_analyzer.found_error_ticket}')
|
||||
|
||||
# "status" is True (which is 1) if everything is ok, False (which is 0) otherwise
|
||||
sys.exit(not status)
|
||||
else:
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ on:
|
|||
- Webassembly
|
||||
- Windows (VS 2019, Python 3.11)
|
||||
- Windows Conditional Compilation (VS 2022, Python 3.11)
|
||||
- Rerun Workflow with Known Errors
|
||||
types:
|
||||
- completed
|
||||
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ on:
|
|||
jobs:
|
||||
rerun:
|
||||
name: Rerun Workflow
|
||||
if: ${{ github.event.workflow_run.conclusion == 'failure' }} # Run only for the completed workflows
|
||||
if: ${{ github.event.workflow_run.conclusion == 'failure' }} # Run only for the failed workflows
|
||||
runs-on: aks-linux-2-cores-8gb
|
||||
permissions:
|
||||
actions: write
|
||||
|
|
@ -49,6 +49,10 @@ jobs:
|
|||
--run-id ${{ github.event.workflow_run.id }} \
|
||||
--repository-name ${GITHUB_REPOSITORY}
|
||||
|
||||
- name: Rerun Retriggered (Ticket ${{ env.FOUND_ERROR_TICKET }})
|
||||
if: ${{ env.PIPELINE_RETRIGGERED == 'true' }}
|
||||
run: echo "Rerun retriggered for ${{ github.event.workflow_run.html_url }} with ticket ${{ env.FOUND_ERROR_TICKET }}"
|
||||
|
||||
rerunner_tests:
|
||||
name: Rerunner Tests
|
||||
if: ${{ github.event_name == 'pull_request' }}
|
||||
|
|
@ -63,10 +67,23 @@ jobs:
|
|||
- name: Install deps
|
||||
run: pip3 install PyGithub==2.2.0 requests==2.31.0
|
||||
|
||||
- name: Test Rerunner
|
||||
- name: Test Rerunner (Tests)
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
working-directory: ${{ github.workspace }}/.github/scripts/workflow_rerun
|
||||
run: |
|
||||
export PYTHONPATH=${{ github.workspace }}/.github/scripts/workflow_rerun:${{ github.workspace }}/.github/scripts:$PYTHONPATH
|
||||
python3 -m unittest tests/*_test.py
|
||||
|
||||
- name: Test Rerunner (CLI)
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
working-directory: ${{ github.workspace }}/.github/scripts/workflow_rerun
|
||||
run: |
|
||||
export PYTHONPATH=${{ github.workspace }}/.github/scripts/workflow_rerun:${{ github.workspace }}/.github/scripts:$PYTHONPATH
|
||||
|
||||
# Need to get a run id with successful status for log analyzing
|
||||
# cannot lock a run id as logs get deleted after some time
|
||||
run_id=$(python3 -c "from github import Github, Auth; import os; github=Github(auth=Auth.Token(token=os.environ.get('GITHUB_TOKEN'))); repo = github.get_repo('${GITHUB_REPOSITORY}'); run_id = repo.get_workflow_runs(status='success')[0].id; print(run_id)")
|
||||
|
||||
python3 rerunner.py --repository-name ${GITHUB_REPOSITORY} --run-id $run_id --dry-run
|
||||
|
|
|
|||
Loading…
Reference in New Issue