From 9ea33715a082ae23f085898f7bf08ea973b57f00 Mon Sep 17 00:00:00 2001 From: Smirnova Maria Date: Mon, 24 Oct 2022 14:57:37 +0300 Subject: [PATCH] Modify stress tests so that data can be pushed to OVVP (#13452) * Change stress tests db fields * Add option for upload data to database by API handler * Push modified data only to new database * Fix adding log info to new db --- tests/stress_tests/scripts/memcheck_upload.py | 54 +++++++++++++++++++ tests/stress_tests/scripts/run_memcheck.py | 9 +++- 2 files changed, 62 insertions(+), 1 deletion(-) diff --git a/tests/stress_tests/scripts/memcheck_upload.py b/tests/stress_tests/scripts/memcheck_upload.py index b0524a07f04..0f50ead0881 100644 --- a/tests/stress_tests/scripts/memcheck_upload.py +++ b/tests/stress_tests/scripts/memcheck_upload.py @@ -21,6 +21,8 @@ import xml.etree.ElementTree as ET from glob import glob from inspect import getsourcefile from types import SimpleNamespace +import requests +from copy import deepcopy import yaml from pymongo import MongoClient @@ -164,6 +166,51 @@ def upload_memcheck_records(records, db_url, db_collection): collection.replace_one({'_id': record['_id']}, record, upsert=True) +def modify_data_for_push_to_new_db(records): + new_records = deepcopy(records) + + for record in new_records: + if 'os_name' in record and 'os_version' in record: + record['os'] = '{} {}.{}'.format(record['os_name'], record['os_version'][0], record['os_version'][1]) + del record['os_name'] + del record['os_version'] + if 'repo_url' in record: + del record['repo_url'] + if 'commit_sha' in record: + del record['commit_sha'] + if 'event_type' in record: + del record['event_type'] + + try: + with open(record['log_path'], 'r') as log_file: + log = log_file.read() + except FileNotFoundError: + log = '' + record['log'] = log + + return new_records + + +def push_to_db_facade(records, db_api_handler): + headers = {"Content-Type": "application/json", "accept": "application/json"} + uploaded = False + errors = [] + for record in records: + try: + response = requests.post(db_api_handler, json=record, headers=headers) + if response.ok: + uploaded = True + except Exception as e: + errors.append(e) + + if uploaded and not errors: + logging.info("Uploaded records by API url {}".format(db_api_handler)) + elif errors: + logging.info("Failed to upload records by API url {} due to errors {}".format(db_api_handler, errors)) + else: + logging.info("Failed to upload records by API url {}".format(db_api_handler)) + + def _transpose_dicts(items, template=None): """ Build dictionary of arrays from array of dictionaries Example: @@ -274,6 +321,10 @@ def main(): parser.add_argument('--db_collection', required=not is_dryrun, help=f'Collection name in {DATABASE} database to upload.', choices=DB_COLLECTIONS) + parser.add_argument('--db_api_handler', + help='API handler url for push data to database', + default='', + ) parser.add_argument('--artifact_root', required=True, help=f'A root directory to strip from log path before upload.') parser.add_argument('--append', help='JSON to append to each item.') @@ -298,6 +349,9 @@ def main(): if not args.dryrun: upload_memcheck_records(records, args.db_url, args.db_collection) logging.info('Uploaded to %s', args.db_url) + if args.db_api_handler: + new_format_records = modify_data_for_push_to_new_db(records) + push_to_db_facade(new_format_records, args.db_api_handler) else: print(json.dumps(records, sort_keys=True, indent=4)) diff --git a/tests/stress_tests/scripts/run_memcheck.py b/tests/stress_tests/scripts/run_memcheck.py index aa74a177045..54fef7a7112 100755 --- a/tests/stress_tests/scripts/run_memcheck.py +++ b/tests/stress_tests/scripts/run_memcheck.py @@ -20,7 +20,7 @@ from compare_memcheck_2_runs import compare_memcheck_2_runs, get_memcheck_record # Database arguments from memcheck_upload import DATABASE, DB_COLLECTIONS from memcheck_upload import create_memcheck_records, upload_memcheck_records, create_memcheck_report, \ - metadata_from_manifest, info_from_test_config + metadata_from_manifest, info_from_test_config, push_to_db_facade, modify_data_for_push_to_new_db def run(args, log=None, verbose=True): @@ -113,6 +113,10 @@ def main(): required=args.timeline_report or args.upload, help=f'use collection name in {DATABASE} database', choices=DB_COLLECTIONS) + parser.add_argument('--db_api_handler', + help='API handler url for push data to database', + default='', + ) parser.add_argument('--manifest', help=f'extract commit information from build manifest') parser.add_argument('--metadata', @@ -186,6 +190,9 @@ def main(): if records: upload_memcheck_records(records, args.db_url, args.db_collection) logging.info('Uploaded to %s/%s.%s', args.db_url, DATABASE, args.db_collection) + if args.db_api_handler: + new_format_records = modify_data_for_push_to_new_db(records) + push_to_db_facade(new_format_records, args.db_api_handler) else: logging.warning('No records to upload')