From b90f908c589c20520a39c72e0233cfc7a988ee04 Mon Sep 17 00:00:00 2001 From: Mateusz Bencer Date: Wed, 27 Jan 2021 10:22:16 +0100 Subject: [PATCH] Enable Bersquad-10 in ONNX CI (#3889) --- ngraph/python/tests/__init__.py | 2 ++ ngraph/python/tests/runtime.py | 18 ++++++++++++++++-- .../tests/test_onnx/model_zoo_preprocess.sh | 2 +- .../python/tests/test_onnx/test_zoo_models.py | 9 ++++++--- 4 files changed, 25 insertions(+), 6 deletions(-) diff --git a/ngraph/python/tests/__init__.py b/ngraph/python/tests/__init__.py index 126465d6ea5..053760b209b 100644 --- a/ngraph/python/tests/__init__.py +++ b/ngraph/python/tests/__init__.py @@ -211,3 +211,5 @@ xfail_issue_43208 = xfail_test(reason="GPT-2 - AssertionError: zoo models result xfail_issue_43209 = xfail_test(reason="GPT-2-LM-HEAD - AssertionError: zoo models results mismatch") xfail_issue_43213 = xfail_test(reason="RetinaNet Resnet101 - AssertionError: zoo models results mismatch") xfail_issue_37973 = xfail_test(reason="TF Inception V2 - AssertionError: zoo models results mismatch") +xfail_issue_47430 = xfail_test(reason="FCN ResNet models - AssertionError: zoo models results mismatch") +xfail_issue_47495 = xfail_test(reason="BertSquad-10 from MSFT - AssertionError: zoo models results mismatch") diff --git a/ngraph/python/tests/runtime.py b/ngraph/python/tests/runtime.py index 831c84a71e4..5397d7874b3 100644 --- a/ngraph/python/tests/runtime.py +++ b/ngraph/python/tests/runtime.py @@ -18,10 +18,11 @@ import logging from typing import Dict, List, Union import numpy as np -from openvino.inference_engine import IECore, IENetwork +from openvino.inference_engine import IECore, IENetwork, Blob from ngraph.exceptions import UserInputError from ngraph.impl import Function, Node, PartialShape +from ngraph.opset1.ops import result from ngraph.utils.types import NumericData, get_shape, get_dtype import tests @@ -102,6 +103,16 @@ class Computation(object): params_string = ", ".join([param.name for param in self.parameters]) return "".format(self.function.get_name(), params_string) + def __get_ie_output_blob_buffer(self, output_blobs: Dict[str, Blob], ng_result: result) -> np.ndarray: + if len(self.results) == 1: + return next(iter(output_blobs.values())).buffer + else: + prev_layer = ng_result.input(0).get_source_output() + out_name = prev_layer.get_node().get_friendly_name() + if prev_layer.get_node().get_output_size() != 1: + out_name += "." + str(prev_layer.get_index()) + return output_blobs[out_name].buffer + def __call__(self, *input_values: NumericData) -> List[NumericData]: """Run computation on input values and return result.""" input_values = [np.array(input_value) for input_value in input_values] @@ -140,9 +151,12 @@ class Computation(object): request = executable_network.requests[0] request.infer(dict(zip(param_names, input_values))) + # Set order of output blobs compatible with nG Function + result_buffers = [self.__get_ie_output_blob_buffer(request.output_blobs, result) + for result in self.results] + # Since OV overwrite result data type we have to convert results to the original one. original_dtypes = [get_dtype(result.get_output_element_type(0)) for result in self.results] - result_buffers = [blob.buffer for blob in request.output_blobs.values()] converted_buffers = [buffer.astype(original_dtype) for buffer, original_dtype in zip(result_buffers, original_dtypes)] return converted_buffers diff --git a/ngraph/python/tests/test_onnx/model_zoo_preprocess.sh b/ngraph/python/tests/test_onnx/model_zoo_preprocess.sh index b50f8e2a5fa..2da2c91da52 100755 --- a/ngraph/python/tests/test_onnx/model_zoo_preprocess.sh +++ b/ngraph/python/tests/test_onnx/model_zoo_preprocess.sh @@ -2,7 +2,7 @@ set -e # provide ONNX Model Zoo commit hash ID to update: -ONNX_SHA=7d9ae32726d872fe9143b7aede149508bdf0b30f +ONNX_SHA=d58213534f2a4d1c4b19ba62b3bb5f544353256e MODELS_DIR="$HOME/.onnx/model_zoo" ENABLE_MSFT=false diff --git a/ngraph/python/tests/test_onnx/test_zoo_models.py b/ngraph/python/tests/test_onnx/test_zoo_models.py index 7ed8c2e32c7..9b1bf9550c8 100644 --- a/ngraph/python/tests/test_onnx/test_zoo_models.py +++ b/ngraph/python/tests/test_onnx/test_zoo_models.py @@ -39,7 +39,9 @@ from tests import ( xfail_issue_43208, xfail_issue_43209, xfail_issue_43213, - xfail_issue_37973) + xfail_issue_37973, + xfail_issue_47430, + xfail_issue_47495) MODELS_ROOT_DIR = tests.MODEL_ZOO_DIR @@ -178,12 +180,13 @@ if len(zoo_models) > 0: (xfail_issue_43213, "test_onnx_model_zoo_vision_object_detection_segmentation_retinanet_model_retinanet_9_test_retinanet_resnet101_retinanet_9_cpu"), (xfail_issue_43208, "test_onnx_model_zoo_text_machine_comprehension_gpt_2_model_gpt2_10_GPT2_model_cpu"), (xfail_issue_43209, "test_onnx_model_zoo_text_machine_comprehension_gpt_2_model_gpt2_lm_head_10_GPT_2_LM_HEAD_model_cpu"), - (xfail_issue_40957, "test_onnx_model_zoo_text_machine_comprehension_bert_squad_model_bertsquad_10_download_sample_10_bertsquad10_cpu"), (xfail_issue_40957, "test_onnx_model_zoo_text_machine_comprehension_roberta_model_roberta_base_11_roberta_base_11_roberta_base_11_cpu"), (xfail_issue_40957, "test_onnx_model_zoo_text_machine_comprehension_bert_squad_model_bertsquad_8_download_sample_8_bertsquad8_cpu"), (xfail_issue_39669, "test_onnx_model_zoo_text_machine_comprehension_t5_model_t5_encoder_12_t5_encoder_cpu"), (xfail_issue_38084, "test_onnx_model_zoo_vision_object_detection_segmentation_mask_rcnn_model_MaskRCNN_10_mask_rcnn_R_50_FPN_1x_cpu"), (xfail_issue_38084, "test_onnx_model_zoo_vision_object_detection_segmentation_faster_rcnn_model_FasterRCNN_10_faster_rcnn_R_50_FPN_1x_cpu"), + (xfail_issue_47430, "test_onnx_model_zoo_vision_object_detection_segmentation_fcn_model_fcn_resnet50_11_fcn_resnet50_11_model_cpu"), + (xfail_issue_47430, "test_onnx_model_zoo_vision_object_detection_segmentation_fcn_model_fcn_resnet101_11_fcn_resnet101_11_model_cpu"), # Model MSFT (xfail_issue_37973, "test_MSFT_opset7_tf_inception_v2_model_cpu"), @@ -199,7 +202,7 @@ if len(zoo_models) > 0: (xfail_issue_38084, "test_MSFT_opset10_faster_rcnn_faster_rcnn_R_50_FPN_1x_cpu"), (xfail_issue_39669, "test_MSFT_opset9_cgan_cgan_cpu"), - (xfail_issue_40957, "test_MSFT_opset10_BERT_Squad_bertsquad10_cpu"), + (xfail_issue_47495, "test_MSFT_opset10_BERT_Squad_bertsquad10_cpu"), (xfail_issue_45457, "test_MSFT_opset10_mlperf_ssd_resnet34_1200_ssd_resnet34_mAP_20.2_cpu"), ]