Modify stress tests so that data can be pushed to OVVP (#13452)

* Change stress tests db fields

* Add option for upload data to database by API handler

* Push modified data only to new database

* Fix adding log info to new db

tests/stress_tests/scripts/memcheck_upload.py

@@ -21,6 +21,8 @@ import xml.etree.ElementTree as ET
 from glob import glob
 from inspect import getsourcefile
 from types import SimpleNamespace
+import requests
+from copy import deepcopy
 
 import yaml
 from pymongo import MongoClient
@@ -164,6 +166,51 @@ def upload_memcheck_records(records, db_url, db_collection):
         collection.replace_one({'_id': record['_id']}, record, upsert=True)
 
 
+def modify_data_for_push_to_new_db(records):
+    new_records = deepcopy(records)
+
+    for record in new_records:
+        if 'os_name' in record and 'os_version' in record:
+            record['os'] = '{} {}.{}'.format(record['os_name'], record['os_version'][0], record['os_version'][1])
+            del record['os_name']
+            del record['os_version']
+        if 'repo_url' in record:
+            del record['repo_url']
+        if 'commit_sha' in record:
+            del record['commit_sha']
+        if 'event_type' in record:
+            del record['event_type']
+
+        try:
+            with open(record['log_path'], 'r') as log_file:
+                log = log_file.read()
+        except FileNotFoundError:
+            log = ''
+        record['log'] = log
+
+    return new_records
+
+
+def push_to_db_facade(records, db_api_handler):
+    headers = {"Content-Type": "application/json", "accept": "application/json"}
+    uploaded = False
+    errors = []
+    for record in records:
+        try:
+            response = requests.post(db_api_handler, json=record, headers=headers)
+            if response.ok:
+                uploaded = True
+        except Exception as e:
+            errors.append(e)
+
+    if uploaded and not errors:
+        logging.info("Uploaded records by API url {}".format(db_api_handler))
+    elif errors:
+        logging.info("Failed to upload records by API url {} due to errors {}".format(db_api_handler, errors))
+    else:
+        logging.info("Failed to upload records by API url {}".format(db_api_handler))
+
+
 def _transpose_dicts(items, template=None):
     """ Build dictionary of arrays from array of dictionaries
     Example:
@@ -274,6 +321,10 @@ def main():
     parser.add_argument('--db_collection', required=not is_dryrun,
                         help=f'Collection name in {DATABASE} database to upload.',
                         choices=DB_COLLECTIONS)
+    parser.add_argument('--db_api_handler',
+                        help='API handler url for push data to database',
+                        default='',
+                        )
     parser.add_argument('--artifact_root', required=True,
                         help=f'A root directory to strip from log path before upload.')
     parser.add_argument('--append', help='JSON to append to each item.')
@@ -298,6 +349,9 @@ def main():
     if not args.dryrun:
         upload_memcheck_records(records, args.db_url, args.db_collection)
         logging.info('Uploaded to %s', args.db_url)
+        if args.db_api_handler:
+            new_format_records = modify_data_for_push_to_new_db(records)
+            push_to_db_facade(new_format_records, args.db_api_handler)
     else:
         print(json.dumps(records, sort_keys=True, indent=4))
 

tests/stress_tests/scripts/run_memcheck.py

@@ -20,7 +20,7 @@ from compare_memcheck_2_runs import compare_memcheck_2_runs, get_memcheck_record
 # Database arguments
 from memcheck_upload import DATABASE, DB_COLLECTIONS
 from memcheck_upload import create_memcheck_records, upload_memcheck_records, create_memcheck_report, \
-    metadata_from_manifest, info_from_test_config
+    metadata_from_manifest, info_from_test_config, push_to_db_facade, modify_data_for_push_to_new_db
 
 
 def run(args, log=None, verbose=True):
@@ -113,6 +113,10 @@ def main():
                         required=args.timeline_report or args.upload,
                         help=f'use collection name in {DATABASE} database',
                         choices=DB_COLLECTIONS)
+    parser.add_argument('--db_api_handler',
+                        help='API handler url for push data to database',
+                        default='',
+                        )
     parser.add_argument('--manifest',
                         help=f'extract commit information from build manifest')
     parser.add_argument('--metadata',
@@ -186,6 +190,9 @@ def main():
             if records:
                 upload_memcheck_records(records, args.db_url, args.db_collection)
                 logging.info('Uploaded to %s/%s.%s', args.db_url, DATABASE, args.db_collection)
+                if args.db_api_handler:
+                    new_format_records = modify_data_for_push_to_new_db(records)
+                    push_to_db_facade(new_format_records, args.db_api_handler)
             else:
                 logging.warning('No records to upload')