From 4601b31bd0efc1eadaed06ba0bbbfeaba3667ddf Mon Sep 17 00:00:00 2001 From: Ruslan Nugmanov Date: Fri, 13 Jan 2023 11:29:50 +0100 Subject: [PATCH] Keras to tflite version of tests (#15042) * keras to tflite version of tests * Update tests/layer_tests/common/tf2_layer_test_class.py Co-authored-by: Evgenya Stepyreva * moved out tf utility functions from modules with tf_layer_test classes to tf_utils module * moved out tf utility functions from modules with tf_layer_test classes to tf_utils module and tflite_utils modules Co-authored-by: Evgenya Stepyreva --- tests/layer_tests/common/layer_test_class.py | 5 +- .../common/tf2_layer_test_class.py | 15 +++- .../layer_tests/common/tf_layer_test_class.py | 81 +------------------ tests/layer_tests/common/utils/tf_utils.py | 32 ++++++++ .../layer_tests/common/utils/tflite_utils.py | 60 ++++++++++++++ .../tensorflow2_keras_tests/conftest.py | 1 + 6 files changed, 113 insertions(+), 81 deletions(-) create mode 100644 tests/layer_tests/common/utils/tflite_utils.py diff --git a/tests/layer_tests/common/layer_test_class.py b/tests/layer_tests/common/layer_test_class.py index c1c26cd8fc6..f51764b54ff 100644 --- a/tests/layer_tests/common/layer_test_class.py +++ b/tests/layer_tests/common/layer_test_class.py @@ -77,9 +77,10 @@ class CommonLayerTest: # assert flag, '\n'.join(resp) config = None - # GPU default execution precision is FP16, so if we want to check FP32 inference we need to set explicit precision hint + # GPU default execution precision is FP16, so if we want to check FP32 inference + # we need to set explicit precision hint if ie_device == 'GPU' and precision == 'FP32': - config = {'INFERENCE_PRECISION_HINT' : 'f32'} + config = {'INFERENCE_PRECISION_HINT': 'f32'} if self.use_old_api: ie_engine = IEInfer(model=path_to_xml, diff --git a/tests/layer_tests/common/tf2_layer_test_class.py b/tests/layer_tests/common/tf2_layer_test_class.py index 9a411173740..e37b2042b31 100644 --- a/tests/layer_tests/common/tf2_layer_test_class.py +++ b/tests/layer_tests/common/tf2_layer_test_class.py @@ -4,6 +4,7 @@ import os from common.layer_test_class import CommonLayerTest +from common.utils.tflite_utils import get_tflite_results, save_tf2_saved_model_to_tflite def save_to_tf2_savedmodel(tf2_model, path_to_saved_tf2_model): @@ -19,9 +20,21 @@ class CommonTF2LayerTest(CommonLayerTest): input_model_key = "saved_model_dir" def produce_model_path(self, framework_model, save_path): - return save_to_tf2_savedmodel(framework_model, save_path) + if not getattr(self, 'tflite', False): + return save_to_tf2_savedmodel(framework_model, save_path) + else: + self.input_model_key = 'input_model' + tf2_saved_model = save_to_tf2_savedmodel(framework_model, save_path) + return save_tf2_saved_model_to_tflite(tf2_saved_model) def get_framework_results(self, inputs_dict, model_path): + if not getattr(self, 'tflite', False): + return self.get_tf2_keras_results(inputs_dict, model_path) + else: + # get results from tflite + return get_tflite_results(self.use_new_frontend, self.use_old_api, inputs_dict, model_path) + + def get_tf2_keras_results(self, inputs_dict, model_path): import tensorflow as tf import numpy as np diff --git a/tests/layer_tests/common/tf_layer_test_class.py b/tests/layer_tests/common/tf_layer_test_class.py index 63e7d022d60..e75c4652efd 100644 --- a/tests/layer_tests/common/tf_layer_test_class.py +++ b/tests/layer_tests/common/tf_layer_test_class.py @@ -1,59 +1,11 @@ # Copyright (C) 2018-2022 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -import os - from common.layer_test_class import CommonLayerTest from common.utils.tf_utils import summarize_graph - -def transpose_nchw_to_nhwc(data, use_new_frontend, use_old_api): - if use_new_frontend or not use_old_api: - return data - - if len(data.shape) == 4: # reshaping for 4D tensors - return data.transpose(0, 2, 3, 1) - elif len(data.shape) == 5: # reshaping for 5D tensors - return data.transpose(0, 2, 3, 4, 1) - else: - return data - - -def transpose_nhwc_to_nchw(data, use_new_frontend, use_old_api): - if use_new_frontend or not use_old_api: - return data - - if len(data.shape) == 4: # reshaping for 4D tensors - return data.transpose(0, 3, 1, 2) # 2, 0, 1 - elif len(data.shape) == 5: # reshaping for 5D tensors - return data.transpose(0, 4, 1, 2, 3) # 3, 0, 1, 2 - else: - return data - - -def save_to_pb(tf_model, path_to_saved_tf_model): - import tensorflow as tf - tf.io.write_graph(tf_model, path_to_saved_tf_model, 'model.pb', False) - assert os.path.isfile(os.path.join(path_to_saved_tf_model, 'model.pb')), "model.pb haven't been saved " \ - "here: {}".format(path_to_saved_tf_model) - return os.path.join(path_to_saved_tf_model, 'model.pb') - - -def save_pb_to_tflite(pb_model): - import tensorflow as tf - - graph_summary = summarize_graph(pb_model) - inputs = [k for k in graph_summary['inputs'].keys()] - outputs = graph_summary['outputs'] - - converter = tf.compat.v1.lite.TFLiteConverter.from_frozen_graph(pb_model, inputs, outputs) - tflite_model = converter.convert() - - tflite_model_path = os.path.join(os.path.dirname(pb_model), 'model.tflite') - with tf.io.gfile.GFile(tflite_model_path, 'wb') as f: - f.write(tflite_model) - - return tflite_model_path +from common.utils.tflite_utils import get_tflite_results, save_pb_to_tflite +from common.utils.tf_utils import save_to_pb, transpose_nhwc_to_nchw, transpose_nchw_to_nhwc class CommonTFLayerTest(CommonLayerTest): @@ -99,33 +51,6 @@ class CommonTFLayerTest(CommonLayerTest): self.use_old_api) return result - def get_tflite_results(self, inputs_dict, model_path): - import tensorflow as tf - interpreter = tf.compat.v1.lite.Interpreter(model_path=model_path) - interpreter.allocate_tensors() - input_details = interpreter.get_input_details() - output_details = interpreter.get_output_details() - - input_name_to_id_mapping = {input['name']: input['index'] for input in input_details} - - for layer, data in inputs_dict.items(): - tensor_index = input_name_to_id_mapping[layer] - tensor_id = next(i for i, tensor in enumerate(input_details) if tensor['index'] == tensor_index) - interpreter.set_tensor(input_details[tensor_id]['index'], data) - - interpreter.invoke() - tf_result = dict() - for output in output_details: - tf_result[output['name']] = interpreter.get_tensor(output['index']) - - result = dict() - for out in tf_result.keys(): - _tf_res = tf_result[out] - result[out] = transpose_nhwc_to_nchw(_tf_res, self.use_new_frontend, - self.use_old_api) - - return tf_result - def get_framework_results(self, inputs_dict, model_path): if not getattr(self, 'tflite', False): # prepare inputs @@ -134,4 +59,4 @@ class CommonTFLayerTest(CommonLayerTest): return self.get_tf_results(inputs_dict, model_path) else: # get results from tflite - return self.get_tflite_results(inputs_dict, model_path) + return get_tflite_results(self.use_new_frontend, self.use_old_api, inputs_dict, model_path) diff --git a/tests/layer_tests/common/utils/tf_utils.py b/tests/layer_tests/common/utils/tf_utils.py index 5f40713baf8..44aa8086f4c 100644 --- a/tests/layer_tests/common/utils/tf_utils.py +++ b/tests/layer_tests/common/utils/tf_utils.py @@ -9,6 +9,7 @@ import tensorflow as tf from openvino.tools.mo.ops.op import PermuteAttrs + os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' @@ -135,3 +136,34 @@ def permute_nchw_to_nhwc(shape, use_new_frontend=False): def permute_axis(axis, permutation_inv): return permutation_inv[axis] + + +def transpose_nchw_to_nhwc(data, use_new_frontend, use_old_api): + if use_new_frontend or not use_old_api: + return data + + if len(data.shape) == 4: # reshaping for 4D tensors + return data.transpose(0, 2, 3, 1) + elif len(data.shape) == 5: # reshaping for 5D tensors + return data.transpose(0, 2, 3, 4, 1) + else: + return data + + +def transpose_nhwc_to_nchw(data, use_new_frontend, use_old_api): + if use_new_frontend or not use_old_api: + return data + + if len(data.shape) == 4: # reshaping for 4D tensors + return data.transpose(0, 3, 1, 2) # 2, 0, 1 + elif len(data.shape) == 5: # reshaping for 5D tensors + return data.transpose(0, 4, 1, 2, 3) # 3, 0, 1, 2 + else: + return data + + +def save_to_pb(tf_model, path_to_saved_tf_model): + tf.io.write_graph(tf_model, path_to_saved_tf_model, 'model.pb', False) + assert os.path.isfile(os.path.join(path_to_saved_tf_model, 'model.pb')), "model.pb haven't been saved " \ + "here: {}".format(path_to_saved_tf_model) + return os.path.join(path_to_saved_tf_model, 'model.pb') diff --git a/tests/layer_tests/common/utils/tflite_utils.py b/tests/layer_tests/common/utils/tflite_utils.py new file mode 100644 index 00000000000..2c10d8437b1 --- /dev/null +++ b/tests/layer_tests/common/utils/tflite_utils.py @@ -0,0 +1,60 @@ +import os + +import tensorflow as tf +from common.utils.tf_utils import summarize_graph, transpose_nhwc_to_nchw + + +def save_pb_to_tflite(pb_model): + graph_summary = summarize_graph(pb_model) + inputs = [k for k in graph_summary['inputs'].keys()] + outputs = graph_summary['outputs'] + + converter = tf.compat.v1.lite.TFLiteConverter.from_frozen_graph(pb_model, inputs, outputs) + tflite_model = converter.convert() + + tflite_model_path = os.path.join(os.path.dirname(pb_model), 'model.tflite') + with tf.io.gfile.GFile(tflite_model_path, 'wb') as f: + f.write(tflite_model) + + return tflite_model_path + + +def get_tflite_results(use_new_frontend, use_old_api, inputs_dict, model_path): + interpreter = tf.compat.v1.lite.Interpreter(model_path=model_path) + interpreter.allocate_tensors() + input_details = interpreter.get_input_details() + output_details = interpreter.get_output_details() + + input_name_to_id_mapping = {input['name']: input['index'] for input in input_details} + + for layer, data in inputs_dict.items(): + tensor_index = input_name_to_id_mapping[layer] + tensor_id = next(i for i, tensor in enumerate(input_details) if tensor['index'] == tensor_index) + interpreter.set_tensor(input_details[tensor_id]['index'], data) + + interpreter.invoke() + tf_lite_result = dict() + for output in output_details: + tf_lite_result[output['name']] = interpreter.get_tensor(output['index']) + + result = dict() + for out in tf_lite_result.keys(): + _tf_res = tf_lite_result[out] + result[out] = transpose_nhwc_to_nchw(_tf_res, use_new_frontend, + use_old_api) + + return tf_lite_result + + +def save_tf2_saved_model_to_tflite(savedmodel): + import tensorflow as tf + + converter = tf.compat.v1.lite.TFLiteConverter.from_saved_model(savedmodel) + tflite_model = converter.convert() + + tflite_model_path = os.path.join(os.path.dirname(savedmodel), 'model.tflite') + with tf.io.gfile.GFile(tflite_model_path, 'wb') as f: + f.write(tflite_model) + + return tflite_model_path + diff --git a/tests/layer_tests/tensorflow2_keras_tests/conftest.py b/tests/layer_tests/tensorflow2_keras_tests/conftest.py index 72a10903444..a3237480d9f 100644 --- a/tests/layer_tests/tensorflow2_keras_tests/conftest.py +++ b/tests/layer_tests/tensorflow2_keras_tests/conftest.py @@ -12,6 +12,7 @@ from common.utils.common_utils import copy_files_by_pattern def pytest_generate_tests(metafunc): test_gen_attrs_names = list(inspect.signature(get_params).parameters) params = get_params() + setattr(metafunc.cls, 'tflite', metafunc.config.getoption('tflite')) metafunc.parametrize(test_gen_attrs_names, params, scope="function")