diff --git a/.gitmodules b/.gitmodules index c08c4d21f2e..893a0a4ab7e 100644 --- a/.gitmodules +++ b/.gitmodules @@ -14,10 +14,6 @@ path = thirdparty/pugixml url = https://github.com/zeux/pugixml.git ignore = dirty -[submodule "thirdparty/ade"] - path = thirdparty/ade - url = https://github.com/opencv/ade.git - ignore = dirty [submodule "thirdparty/gflags/gflags"] path = thirdparty/gflags/gflags url = https://github.com/gflags/gflags.git diff --git a/cmake/features.cmake b/cmake/features.cmake index a4d19f116cb..c1e843ae64d 100644 --- a/cmake/features.cmake +++ b/cmake/features.cmake @@ -103,8 +103,6 @@ endif() ov_dependent_option (ENABLE_TBBBIND_2_5 "Enable TBBBind_2_5 static usage in OpenVINO runtime" ${ENABLE_TBBBIND_2_5_DEFAULT} "THREADING MATCHES TBB; NOT APPLE" OFF) ov_dependent_option (ENABLE_TBB_RELEASE_ONLY "Only Release TBB libraries are linked to the OpenVINO Runtime binaries" ON "THREADING MATCHES TBB;LINUX" OFF) -ov_dependent_option (ENABLE_GAPI_PREPROCESSING "Enables G-API preprocessing" ON "NOT MINGW64" OFF) - ov_option (ENABLE_MULTI "Enables MULTI Device Plugin" ON) ov_option (ENABLE_AUTO "Enables AUTO Device Plugin" ON) ov_option (ENABLE_AUTO_BATCH "Enables Auto-Batching Plugin" ON) @@ -113,8 +111,6 @@ ov_option (ENABLE_TEMPLATE "Enable template plugin" ON) ov_dependent_option (ENABLE_PLUGINS_XML "Generate plugins.xml configuration file or not" OFF "BUILD_SHARED_LIBS" OFF) -ov_dependent_option (GAPI_TEST_PERF "if GAPI unit tests should examine performance" OFF "ENABLE_TESTS;ENABLE_GAPI_PREPROCESSING" OFF) - ov_dependent_option (ENABLE_FUNCTIONAL_TESTS "functional tests" ON "ENABLE_TESTS" OFF) ov_option (ENABLE_SAMPLES "console samples are part of OpenVINO Runtime package" ON) diff --git a/cmake/templates/OpenVINOConfig.cmake.in b/cmake/templates/OpenVINOConfig.cmake.in index 6cf044d3927..03befe18c3f 100644 --- a/cmake/templates/OpenVINOConfig.cmake.in +++ b/cmake/templates/OpenVINOConfig.cmake.in @@ -333,17 +333,6 @@ macro(_ov_find_itt) unset(_ENABLE_SYSTEM_ITTAPI) endmacro() -macro(_ov_find_ade) - set(_OV_ENABLE_GAPI_PREPROCESSING "@ENABLE_GAPI_PREPROCESSING@") - # whether 'ade' is found via find_package - set(_ENABLE_SYSTEM_ADE "@ade_FOUND@") - if(_OV_ENABLE_GAPI_PREPROCESSING AND _ENABLE_SYSTEM_ADE) - _ov_find_dependency(ade @ade_VERSION@) - endif() - unset(_OV_ENABLE_GAPI_PREPROCESSING) - unset(_ENABLE_SYSTEM_ADE) -endmacro() - macro(_ov_find_intel_cpu_dependencies) set(_OV_ENABLE_CPU_ACL "@DNNL_USE_ACL@") if(_OV_ENABLE_CPU_ACL) @@ -493,9 +482,6 @@ if(NOT _OV_ENABLE_OPENVINO_BUILD_SHARED) _ov_find_itt() _ov_find_pugixml() - # preprocessing dependencies - _ov_find_ade() - # frontend dependencies _ov_find_protobuf_frontend_dependency() _ov_find_tensorflow_frontend_dependencies() diff --git a/docs/articles_en/documentation/openvino_legacy_features/api_2_0_transition_guide/deployment_migration.rst b/docs/articles_en/documentation/openvino_legacy_features/api_2_0_transition_guide/deployment_migration.rst index ee4461bd00d..af71f1d3aa5 100644 --- a/docs/articles_en/documentation/openvino_legacy_features/api_2_0_transition_guide/deployment_migration.rst +++ b/docs/articles_en/documentation/openvino_legacy_features/api_2_0_transition_guide/deployment_migration.rst @@ -205,7 +205,6 @@ OpenVINO 2022.1 introduced a reorganization of the libraries, to make deployment Below are detailed comparisons of the library structure between OpenVINO 2022.1 and the previous versions: * Starting with 2022.1 release, a single core library with all the functionalities (``openvino`` for C++ Runtime, ``openvino_c`` for Inference Engine API C interface) is used, instead of the previous core libraries which contained ``inference_engine``, ``ngraph``, ``inference_engine_transformations`` and ``inference_engine_lp_transformations``. -* The optional ``inference_engine_preproc`` preprocessing library (if `InferenceEngine::PreProcessInfo::setColorFormat `__ or `InferenceEngine::PreProcessInfo::setResizeAlgorithm `__ is used) has been renamed to ``openvino_gapi_preproc`` and deprecated in 2022.1. For more details, see the :doc:`Preprocessing capabilities of OpenVINO API 2.0 `. * The libraries of plugins have been renamed as follows: @@ -234,8 +233,6 @@ Older versions of OpenVINO had several core libraries and plugin modules: Now, the modularity is more clear: - A single core library with all the functionality ``openvino`` for C++ runtime - ``openvino_c`` with Inference Engine API C interface -- **Deprecated** Optional ``openvino_gapi_preproc`` preprocessing library (if ``InferenceEngine::PreProcessInfo::setColorFormat`` or ``InferenceEngine::PreProcessInfo::setResizeAlgorithm`` are used) - - Use :doc:`preprocessing capabilities of OpenVINO API 2.0 ` - Plugin libraries with clear names: - ``openvino_intel_cpu_plugin`` - ``openvino_intel_gpu_plugin`` diff --git a/docs/articles_en/openvino_workflow/deployment_intro/conditional_compilation_deployment.rst b/docs/articles_en/openvino_workflow/deployment_intro/conditional_compilation_deployment.rst index 0f0e88070b9..0a2b28dc6f6 100644 --- a/docs/articles_en/openvino_workflow/deployment_intro/conditional_compilation_deployment.rst +++ b/docs/articles_en/openvino_workflow/deployment_intro/conditional_compilation_deployment.rst @@ -278,7 +278,7 @@ Generate final optimal binaries size of OpenVINO package md build cd build - cmake -G "Visual Studio 16 2019" -A x64 -DENABLE_CPPLINT=OFF -DCMAKE_VERBOSE_MAKEFILE=ON -DCMAKE_COMPILE_WARNING_AS_ERROR=OFF -DCMAKE_BUILD_TYPE=Release -DENABLE_FASTER_BUILD=ON -DENABLE_PROFILING_ITT=OFF -DSELECTIVE_BUILD=ON -DENABLE_INTEL_GPU=OFF -DENABLE_MULTI=OFF -DENABLE_AUTO=OFF -DENABLE_AUTO_BATCH=OFF -DENABLE_HETERO=OFF -DENABLE_TEMPLATE=OFF -DENABLE_OV_ONNX_FRONTEND=OFF -DENABLE_OV_PADDLE_FRONTEND=OFF -DENABLE_OV_PYTORCH_FRONTEND=OFF -DENABLE_OV_TF_FRONTEND=OFF -DSELECTIVE_BUILD_STAT=%OPENVINO_HOME%\cc_data\*.csv -DBUILD_SHARED_LIBS=OFF -DENABLE_LTO=ON -DENABLE_ONEDNN_FOR_GPU=OFF -DENABLE_GAPI_PREPROCESSING=OFF -DENABLE_OV_TF_LITE_FRONTEND=OFF -DENABLE_PROFILING_FIRST_INFERENCE=OFF .. + cmake -G "Visual Studio 16 2019" -A x64 -DENABLE_CPPLINT=OFF -DCMAKE_VERBOSE_MAKEFILE=ON -DCMAKE_COMPILE_WARNING_AS_ERROR=OFF -DCMAKE_BUILD_TYPE=Release -DENABLE_FASTER_BUILD=ON -DENABLE_PROFILING_ITT=OFF -DSELECTIVE_BUILD=ON -DENABLE_INTEL_GPU=OFF -DENABLE_MULTI=OFF -DENABLE_AUTO=OFF -DENABLE_AUTO_BATCH=OFF -DENABLE_HETERO=OFF -DENABLE_TEMPLATE=OFF -DENABLE_OV_ONNX_FRONTEND=OFF -DENABLE_OV_PADDLE_FRONTEND=OFF -DENABLE_OV_PYTORCH_FRONTEND=OFF -DENABLE_OV_TF_FRONTEND=OFF -DSELECTIVE_BUILD_STAT=%OPENVINO_HOME%\cc_data\*.csv -DBUILD_SHARED_LIBS=OFF -DENABLE_LTO=ON -DENABLE_ONEDNN_FOR_GPU=OFF -DENABLE_OV_TF_LITE_FRONTEND=OFF -DENABLE_PROFILING_FIRST_INFERENCE=OFF .. cmake --build . --config Release diff --git a/docs/dev/cmake_options_for_custom_compilation.md b/docs/dev/cmake_options_for_custom_compilation.md index 36ba866b32f..725dfa940d0 100644 --- a/docs/dev/cmake_options_for_custom_compilation.md +++ b/docs/dev/cmake_options_for_custom_compilation.md @@ -44,8 +44,6 @@ This document provides description and default values for CMake options that can * `ON` is default. * `ENABLE_OV_IR_FRONTEND` enables OpenVINO Intermediate Representation frontend plugin for OpenVINO Runtime: * `ON` is default. -* `ENABLE_GAPI_PREPROCESSING` enables G-API preprocessing: - * `ON` is default. * `OPENVINO_EXTRA_MODULES` specifies path to add extra OpenVINO modules to the build. * See [OpenVINO Contrib] to add extra modules from. * `ENABLE_SAMPLES` enables OpenVINO Runtime samples build: diff --git a/docs/dev/static_libaries.md b/docs/dev/static_libaries.md index 7b00f9e5499..3570b4532d9 100644 --- a/docs/dev/static_libaries.md +++ b/docs/dev/static_libaries.md @@ -47,7 +47,6 @@ cmake -DENABLE_INTEL_GPU=OFF \ -DENABLE_OV_TF_FRONTEND=OFF \ -DENABLE_OV_TF_LITE_FRONTEND=OFF \ -DENABLE_OV_PYTORCH_FRONTEND=OFF \ - -DENABLE_GAPI_PREPROCESSING=OFF \ -DENABLE_INTEL_CPU=ON \ -DENABLE_OV_IR_FRONTEND=ON ``` @@ -150,4 +149,4 @@ cmake -DCMAKE_TOOLCHAIN_FILE=/cmake/toolchains/mt.runtime.w * [OpenVINO Developer Documentation](index.md) * [How to Build OpenVINO](build.md) - \ No newline at end of file + diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 41551270393..67ed291ff62 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -18,8 +18,6 @@ add_subdirectory(frontends) add_subdirectory(plugins) add_subdirectory(inference) include(cmake/openvino.cmake) -# preprocessing has dependency on `openvino` for static build -add_subdirectory(common/preprocessing) add_subdirectory(bindings) if(ENABLE_TESTS) diff --git a/src/cmake/openvino.cmake b/src/cmake/openvino.cmake index 44660671e60..dec7fcbe7e5 100644 --- a/src/cmake/openvino.cmake +++ b/src/cmake/openvino.cmake @@ -108,11 +108,7 @@ add_library(openvino::runtime::dev ALIAS openvino_runtime_dev) target_include_directories(openvino_runtime_dev INTERFACE $ - $ - $>) - -target_compile_definitions(openvino_runtime_dev INTERFACE - $) + $) target_link_libraries(openvino_runtime_dev INTERFACE ${TARGET_NAME} openvino::core::dev) diff --git a/src/cmake/ov_parallel.cmake b/src/cmake/ov_parallel.cmake index 510b207689a..8ac29116b43 100644 --- a/src/cmake/ov_parallel.cmake +++ b/src/cmake/ov_parallel.cmake @@ -265,7 +265,7 @@ function(ov_set_threading_interface_for TARGET_NAME) set(LINK_TYPE "PRIVATE") set(COMPILE_DEF_TYPE "PRIVATE") elseif(target_type STREQUAL "STATIC_LIBRARY") - # Affected libraries: inference_engine_s, openvino_gapi_preproc_s + # Affected libraries: inference_engine_s # they don't have TBB in public headers => PRIVATE set(LINK_TYPE "PRIVATE") set(COMPILE_DEF_TYPE "PUBLIC") diff --git a/src/common/preprocessing/CMakeLists.txt b/src/common/preprocessing/CMakeLists.txt deleted file mode 100644 index c004aa8ec62..00000000000 --- a/src/common/preprocessing/CMakeLists.txt +++ /dev/null @@ -1,11 +0,0 @@ -# Copyright (C) 2018-2023 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 -# - -ov_deprecated_no_errors() - -add_subdirectory(src) - -if(ENABLE_TESTS AND ENABLE_GAPI_PREPROCESSING) - add_subdirectory(tests) -endif() diff --git a/src/common/preprocessing/src/CMakeLists.txt b/src/common/preprocessing/src/CMakeLists.txt deleted file mode 100644 index 80eafe970a6..00000000000 --- a/src/common/preprocessing/src/CMakeLists.txt +++ /dev/null @@ -1,224 +0,0 @@ -# Copyright (C) 2018-2023 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 -# - -set (TARGET_NAME "openvino_gapi_preproc") - -file(GLOB LIBRARY_SRC ${CMAKE_CURRENT_SOURCE_DIR}/*.cpp) -file(GLOB LIBRARY_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/*.hpp) - -# -# Provides compilation flags for AVX512F, AVX512BW, AVX512DQ -# instructions support depending on an OS and a compiler -# -macro(ie_avx512_core_optimization_flags flags) - if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") - set(${flags} /arch:AVX512) - elseif(CMAKE_CXX_COMPILER_ID STREQUAL "Intel") - if(WIN32) - set(${flags} /QxCORE-AVX512) - else() - set(${flags} -xCORE-AVX512) - endif() - elseif(CMAKE_COMPILER_IS_GNUCXX OR OV_COMPILER_IS_CLANG) - set(${flags} -mavx512f -mavx512bw -mavx512dq -mfma) - if(CMAKE_COMPILER_IS_GNUCXX AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 12) - list(APPEND ${flags} -Wno-error=maybe-uninitialized -Wno-maybe-uninitialized) - endif() - else() - message(WARNING "Unsupported CXX compiler ${CMAKE_CXX_COMPILER_ID}") - endif() -endmacro() - -if(ENABLE_SSE42) - file(GLOB SSE_SRC ${CMAKE_CURRENT_SOURCE_DIR}/cpu_x86_sse42/*.cpp) - file(GLOB SSE_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/cpu_x86_sse42/*.hpp) - - list(APPEND LIBRARY_HEADERS ${SSE_HEADERS}) - list(APPEND LIBRARY_SRC ${SSE_SRC}) - - ov_sse42_optimization_flags(sse4_2_flags) - set_source_files_properties(${SSE_SRC} PROPERTIES COMPILE_OPTIONS "${sse4_2_flags}") - add_definitions(-DHAVE_SSE=1) -endif() - -if(ENABLE_AVX2) - file(GLOB AVX2_SRC ${CMAKE_CURRENT_SOURCE_DIR}/cpu_x86_avx2/*.cpp) - file(GLOB AVX2_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/cpu_x86_avx2/*.hpp) - - list(APPEND LIBRARY_HEADERS ${AVX2_HEADERS}) - list(APPEND LIBRARY_SRC ${AVX2_SRC}) - - ov_avx2_optimization_flags(avx2_flags) - set_source_files_properties(${AVX2_SRC} PROPERTIES COMPILE_OPTIONS "${avx2_flags}") - add_definitions(-DHAVE_AVX2=1) -endif() - - -# Workaround for GCC version 5.4 and 5.5 bugs in Debug configuration. -if (CMAKE_COMPILER_IS_GNUCXX AND - (CMAKE_CXX_COMPILER_VERSION VERSION_LESS_EQUAL 5.5) AND - (CMAKE_BUILD_TYPE STREQUAL Debug)) - set(GNU_5_DEBUG_CASE ON) -endif() - -if(ENABLE_AVX512F AND NOT GNU_5_DEBUG_CASE) - file(GLOB AVX512_SRC ${CMAKE_CURRENT_SOURCE_DIR}/cpu_x86_avx512/*.cpp) - file(GLOB AVX512_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/cpu_x86_avx512/*.hpp) - - list(APPEND LIBRARY_HEADERS ${AVX512_HEADERS}) - list(APPEND LIBRARY_SRC ${AVX512_SRC}) - - ie_avx512_core_optimization_flags(avx512_flags) - set_source_files_properties(${AVX512_SRC} PROPERTIES COMPILE_OPTIONS "${avx512_flags}") - add_definitions(-DHAVE_AVX512=1) -endif() - -if(ARM OR AARCH64) - ov_arm_neon_optimization_flags(neon_flags) - - file(GLOB NEON_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/arm_neon/*.hpp) - file(GLOB NEON_SRC ${CMAKE_CURRENT_SOURCE_DIR}/arm_neon/*.cpp) - - list(APPEND LIBRARY_HEADERS ${NEON_HEADERS}) - list(APPEND LIBRARY_SRC ${NEON_SRC}) - - if(neon_flags) - set_source_files_properties(${NEON_SRC} PROPERTIES COMPILE_OPTIONS "${neon_flags}") - endif() - - add_definitions(-DHAVE_NEON=1) -endif() - -# Create object library - -if(ENABLE_GAPI_PREPROCESSING) - add_library(${TARGET_NAME}_obj OBJECT - ${LIBRARY_SRC} - ${LIBRARY_HEADERS}) - ov_build_target_faster(${TARGET_NAME}_obj UNITY) - - target_compile_definitions(${TARGET_NAME}_obj PRIVATE - IMPLEMENT_INFERENCE_ENGINE_PLUGIN - $ - $ - ENABLE_GAPI_PREPROCESSING) - - target_include_directories(${TARGET_NAME}_obj SYSTEM PRIVATE - $ - $ - $) - - target_include_directories(${TARGET_NAME}_obj PRIVATE - "${CMAKE_CURRENT_SOURCE_DIR}" - $ - $) - - ov_set_threading_interface_for(${TARGET_NAME}_obj) - - add_cpplint_target(${TARGET_NAME}_obj_cpplint FOR_TARGETS ${TARGET_NAME}_obj) - - set(library_sources $) -endif() - -# Create module library file from object library - -if(ENABLE_GAPI_PREPROCESSING) - if(BUILD_SHARED_LIBS) - set(library_type MODULE) - else() - set(library_type STATIC) - endif() -else() - set(library_type INTERFACE) -endif() - -add_library(${TARGET_NAME} ${library_type} ${library_sources}) - -ov_set_threading_interface_for(${TARGET_NAME}) - -if(ENABLE_GAPI_PREPROCESSING) - target_compile_definitions(${TARGET_NAME} PUBLIC ENABLE_GAPI_PREPROCESSING) - target_link_libraries(${TARGET_NAME} PRIVATE fluid openvino::itt openvino::util) - - if(BUILD_SHARED_LIBS) - target_link_libraries(${TARGET_NAME} PRIVATE openvino::runtime) - # make plugin to depend on preprocessing library - foreach(plugin IN LISTS PLUGIN_FILES) - string(REPLACE ":" ";" plugin "${plugin}") - list(REMOVE_AT plugin 0) - add_dependencies(${plugin} openvino_gapi_preproc) - endforeach() - add_dependencies(ov_plugins openvino_gapi_preproc) - else() - # for static linkage the dependencies are in opposite order - target_link_libraries(openvino PRIVATE ${TARGET_NAME}) - endif() - - # Workaround to avoid warnings caused with bug in the avx512intrin.h of GCC5 - if((CMAKE_CXX_COMPILER_ID STREQUAL "GNU") AND - (CMAKE_CXX_COMPILER_VERSION VERSION_LESS_EQUAL 5.5)) - set_target_properties(${TARGET_NAME} PROPERTIES LINK_FLAGS_RELEASE "-Wno-error=maybe-uninitialized -Wno-maybe-uninitialized") - endif() - - if(WIN32 AND NOT library_type STREQUAL "INTERFACE") - set_target_properties(${TARGET_NAME} PROPERTIES COMPILE_PDB_NAME ${TARGET_NAME}) - endif() - - # must be called after all target_link_libraries - ov_add_api_validator_post_build_step(TARGET ${TARGET_NAME}) - - ov_add_vs_version_file(NAME ${TARGET_NAME} - FILEDESCRIPTION "OpenVINO Preprocessing plugin") -endif() - -target_include_directories(${TARGET_NAME} INTERFACE - $ - $) - -# Static library used for unit tests which are always built - -if(ENABLE_GAPI_PREPROCESSING) - add_library(${TARGET_NAME}_s STATIC EXCLUDE_FROM_ALL - $) - - ov_set_threading_interface_for(${TARGET_NAME}_s) - - target_include_directories(${TARGET_NAME}_s INTERFACE "${CMAKE_CURRENT_SOURCE_DIR}") - - if(WIN32) - set_target_properties(${TARGET_NAME}_s PROPERTIES COMPILE_PDB_NAME ${TARGET_NAME}_s) - endif() - - target_link_libraries(${TARGET_NAME}_s PRIVATE fluid openvino::itt openvino::runtime) - - target_compile_definitions(${TARGET_NAME}_s INTERFACE USE_STATIC_IE) - - set_target_properties(${TARGET_NAME}_s PROPERTIES EXCLUDE_FROM_ALL ON) -endif() - -# LTO - -if(TARGET ${TARGET_NAME}_obj) - set_target_properties(${TARGET_NAME}_obj ${TARGET_NAME}_s - PROPERTIES INTERPROCEDURAL_OPTIMIZATION_RELEASE ${ENABLE_LTO}) -endif() - -if(NOT library_type STREQUAL "INTERFACE") - set_target_properties(${TARGET_NAME} - PROPERTIES INTERPROCEDURAL_OPTIMIZATION_RELEASE ${ENABLE_LTO}) -endif() - -# developer package - -ov_developer_package_export_targets(TARGET ${TARGET_NAME}) - -# install - -if(BUILD_SHARED_LIBS) - install(TARGETS ${TARGET_NAME} - LIBRARY DESTINATION ${OV_CPACK_PLUGINSDIR} COMPONENT ${OV_CPACK_COMP_CORE} - ${OV_CPACK_COMP_CORE_EXCLUDE_ALL}) -else() - ov_install_static_lib(${TARGET_NAME} ${OV_CPACK_COMP_CORE}) -endif() diff --git a/src/common/preprocessing/src/arm_neon/ie_preprocess_gapi_kernels_neon.cpp b/src/common/preprocessing/src/arm_neon/ie_preprocess_gapi_kernels_neon.cpp deleted file mode 100644 index 01835008025..00000000000 --- a/src/common/preprocessing/src/arm_neon/ie_preprocess_gapi_kernels_neon.cpp +++ /dev/null @@ -1,651 +0,0 @@ -// Copyright (C) 2018-2023 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#include "ie_preprocess_gapi_kernels.hpp" -#include "ie_preprocess_gapi_kernels_impl.hpp" -#include "ie_preprocess_gapi_kernels_neon.hpp" - -#include - -#ifdef CV_NEON -#undef CV_NEON -#endif - -#define CV_NEON 1 - -#ifdef CV_SIMD128 -#undef CV_SIMD128 -#endif - -#define CV_SIMD128 1 - -#include "opencv_hal_intrin.hpp" -#include "ie_preprocess_gapi_kernels_simd_impl.hpp" - -using namespace cv; - -namespace InferenceEngine { -namespace gapi { -namespace kernels { -namespace neon { - -template -CV_ALWAYS_INLINE void channels2planes_store(std::array, chanNum>& dst, - const uchar* src, const int width, - const int line) { - constexpr int nlanes = static_cast(v_uint8::nlanes); - GAPI_DbgAssert(width >= nlanes); - - v_uint8 chan; - int x = 0; - for (;;) { - for (; x <= width - nlanes && x >= 0; x += nlanes) { - for (int c = 0; c < chanNum; ++c) { - v_gather_channel(chan, &src[chanNum * x], c); - vx_store(&dst[c][line][x], chan); - } - } - - if (x < width) { - x = width - nlanes; - continue; - } - break; - } -} - -CV_ALWAYS_INLINE void vertical_anyLPI(const uchar* src0, const uchar* src1, - uchar* tmp, const int inLength, - const short beta) { - constexpr int nlanes = static_cast(v_uint8::nlanes); - GAPI_DbgAssert(inLength >= nlanes); - - const int half_nlanes = nlanes/2; - int w = 0; - for (;;) { - for (; w <= inLength - nlanes; w += nlanes) { - v_int16 s0 = v_reinterpret_as_s16(vx_load_expand(&src0[w])); - v_int16 s1 = v_reinterpret_as_s16(vx_load_expand(&src1[w])); - v_int16 s2 = v_reinterpret_as_s16(vx_load_expand(&src0[w + half_nlanes])); - v_int16 s3 = v_reinterpret_as_s16(vx_load_expand(&src1[w + half_nlanes])); - v_int16 res1 = v_mulhrs(s0 - s1, beta) + s1; - v_int16 res2 = v_mulhrs(s2 - s3, beta) + s3; - - vx_store(tmp + w, v_pack_u(res1, res2)); - } - - if (w < inLength) { - w = inLength - nlanes; - continue; - } - break; - } -} - -template -CV_ALWAYS_INLINE void horizontal_anyLPI(std::array, chanNum>& dst, - const uchar* src, const short mapsx[], - const short alpha[], const int width, - const int line) { - constexpr int nlanes = static_cast(v_uint8::nlanes); - const int half_nlanes = nlanes/2; - GAPI_DbgAssert(width >= half_nlanes); - - v_int16 t0, t1;//, t2, t3; - int x = 0; - for (;;) { - for (; x <= width - half_nlanes && x >= 0; x += half_nlanes) { - v_int16 a0 = vx_load(&alpha[x]); - for (int c = 0; c < chanNum; ++c) { - v_gather_channel(t0, src, &mapsx[x], c, 0); - v_gather_channel(t1, src, &mapsx[x], c, 1); - //v_gather_channel(t2, src, &mapsx[x + half_nlanes], c, 0); - //v_gather_channel(t3, src, &mapsx[x + half_nlanes], c, 1); - v_int16 res1 = v_mulhrs(t0 - t1, a0) + t1; - //v_int16 res2 = v_mulhrs(t2 - t3, a0) + t3; - //vx_store(&dst[c][line][x], v_pack_u(res1, res2)); - v_pack_u_store(&dst[c][line][x], res1); - } - } - - if (x < width) { - //x = width - nlanes; - x = width - half_nlanes; - continue; - } - break; - } -} - -CV_ALWAYS_INLINE void vertical_4LPI(const uint8_t* src0[], const uint8_t* src1[], - uchar tmp[], const short beta[], const int length) { - constexpr int nlanes = static_cast(v_uint8::nlanes); - constexpr int half_nlanes = nlanes / 2; - GAPI_Assert(length >= half_nlanes); - - v_int16 b0 = vx_setall_s16(beta[0]); - v_int16 b1 = vx_setall_s16(beta[1]); - v_int16 b2 = vx_setall_s16(beta[2]); - v_int16 b3 = vx_setall_s16(beta[3]); - - v_int16 lo1, hi1, lo2, hi2; - v_int32 res1_s32, res2_s32; - int w = 0; - for (;;) { - for (; w <= length - half_nlanes; w += half_nlanes) { - v_int16 val0_0 = v_reinterpret_as_s16(vx_load_expand(&src0[0][w])); - v_int16 val0_1 = v_reinterpret_as_s16(vx_load_expand(&src0[1][w])); - v_int16 val0_2 = v_reinterpret_as_s16(vx_load_expand(&src0[2][w])); - v_int16 val0_3 = v_reinterpret_as_s16(vx_load_expand(&src0[3][w])); - - v_int16 val1_0 = v_reinterpret_as_s16(vx_load_expand(&src1[0][w])); - v_int16 val1_1 = v_reinterpret_as_s16(vx_load_expand(&src1[1][w])); - v_int16 val1_2 = v_reinterpret_as_s16(vx_load_expand(&src1[2][w])); - v_int16 val1_3 = v_reinterpret_as_s16(vx_load_expand(&src1[3][w])); - - v_int16 t0 = v_mulhrs(v_sub_wrap(val0_0, val1_0), b0); - v_int16 t1 = v_mulhrs(v_sub_wrap(val0_1, val1_1), b1); - v_int16 t2 = v_mulhrs(v_sub_wrap(val0_2, val1_2), b2); - v_int16 t3 = v_mulhrs(v_sub_wrap(val0_3, val1_3), b3); - - v_int16 r0 = v_add_wrap(val1_0, t0); - v_int16 r1 = v_add_wrap(val1_1, t1); - v_int16 r2 = v_add_wrap(val1_2, t2); - v_int16 r3 = v_add_wrap(val1_3, t3); - - v_interleave(r0, r1, lo1, hi1); - v_interleave(r2, r3, lo2, hi2); - - v_int32 lo1_s32 = v_reinterpret_as_s32(lo1); - v_int32 hi1_s32 = v_reinterpret_as_s32(hi1); - v_int32 lo2_s32 = v_reinterpret_as_s32(lo2); - v_int32 hi2_s32 = v_reinterpret_as_s32(hi2); - - v_interleave(lo1_s32, lo2_s32, res1_s32, res2_s32); - - v_int16 res1 = v_reinterpret_as_s16(res1_s32); - v_int16 res2 = v_reinterpret_as_s16(res2_s32); - - v_pack_u_store(&tmp[4 * w + 0], res1); - v_pack_u_store(&tmp[4 * w + half_nlanes], res2); - - v_interleave(hi1_s32, hi2_s32, res1_s32, res2_s32); - - v_int16 res3 = v_reinterpret_as_s16(res1_s32); - v_int16 res4 = v_reinterpret_as_s16(res2_s32); - - v_pack_u_store(&tmp[4 * w + 2*half_nlanes], res3); - v_pack_u_store(&tmp[4 * w + 3*half_nlanes], res4); - } - - if (w < length) { - w = length - half_nlanes; - continue; - } - break; - } -} - -template -CV_ALWAYS_INLINE void horizontal_4LPI(std::array, chanNum>& dst, - const uchar* tmp, const short mapsx[], const uchar _mask_horizontal[], - const short clone[], - const int length) { - constexpr int nlanes = static_cast(v_uint8::nlanes); - constexpr int half_nlanes = nlanes / 2; - GAPI_DbgAssert(length >= half_nlanes); - - const int shift = static_cast(half_nlanes / 4); - - v_uint8 hmask = vx_load(_mask_horizontal); - - v_uint8 val_0, val_1, val_2, val_3; - - int x = 0; - for (;;) { - for (; x <= length - half_nlanes && x >= 0; x += half_nlanes) { - v_int16 a10 = vx_load(&clone[4 * x]); - v_int16 a32 = vx_load(&clone[4 * (x + 2)]); - v_int16 a54 = vx_load(&clone[4 * (x + 4)]); - v_int16 a76 = vx_load(&clone[4 * (x + 6)]); - - for (int c = 0; c < chanNum; ++c) { - v_gather_channel(val_0, tmp, &mapsx[x], chanNum, c, 0); - v_gather_channel(val_1, tmp, &mapsx[x], chanNum, c, shift); - v_gather_channel(val_2, tmp, &mapsx[x], chanNum, c, shift * 2); - v_gather_channel(val_3, tmp, &mapsx[x], chanNum, c, shift * 3); - - v_int16 val0_0 = v_reinterpret_as_s16(v_expand_low(val_0)); - v_int16 val0_1 = v_reinterpret_as_s16(v_expand_low(val_1)); - v_int16 val0_2 = v_reinterpret_as_s16(v_expand_low(val_2)); - v_int16 val0_3 = v_reinterpret_as_s16(v_expand_low(val_3)); - - v_int16 val1_0 = v_reinterpret_as_s16(v_expand_high(val_0)); - v_int16 val1_1 = v_reinterpret_as_s16(v_expand_high(val_1)); - v_int16 val1_2 = v_reinterpret_as_s16(v_expand_high(val_2)); - v_int16 val1_3 = v_reinterpret_as_s16(v_expand_high(val_3)); - - v_int16 t0 = v_mulhrs(v_sub_wrap(val0_0, val1_0), a10); - v_int16 t1 = v_mulhrs(v_sub_wrap(val0_1, val1_1), a32); - v_int16 t2 = v_mulhrs(v_sub_wrap(val0_2, val1_2), a54); - v_int16 t3 = v_mulhrs(v_sub_wrap(val0_3, val1_3), a76); - - v_int16 r0 = v_add_wrap(val1_0, t0); - v_int16 r1 = v_add_wrap(val1_1, t1); - v_int16 r2 = v_add_wrap(val1_2, t2); - v_int16 r3 = v_add_wrap(val1_3, t3); - - v_uint8 q0 = v_pack_u(r0, r1); - v_uint8 q1 = v_pack_u(r2, r3); - - v_uint8 q2 = v_shuffle(q0, hmask); - v_uint8 q3 = v_shuffle(q1, hmask); - - v_uint8 q4 = v_blend<0xCC /*0b11001100*/>(q2, v_shift_left<4>(q3)); - v_uint8 q5 = v_blend<0xCC /*0b11001100*/>(v_shift_right<4>(q2), q3); - - v_store_low(&dst[c][0][x], q4); - v_store_high(&dst[c][1][x], q4); - v_store_low(&dst[c][2][x], q5); - v_store_high(&dst[c][3][x], q5); - } - } - - if (x < length) { - x = length - half_nlanes; - continue; - } - break; - } -} - -template -CV_ALWAYS_INLINE bool calcRowLinear_8UC_Impl(neon_tag, - std::array, chanNum>& dst, - const uint8_t* src0[], - const uint8_t* src1[], - const short alpha[], - const short clone[], // 4 clones of alpha - const short mapsx[], - const short beta[], - uint8_t tmp[], - const Size& inSz, - const Size& outSz, - const int lpi) { - static_assert(v_uint8::nlanes == 16, - "The wide of NEON vector is 128 bits, so one vector contains 16 uchars"); - constexpr int nlanes = static_cast(v_uint8::nlanes); - if ((inSz.width * chanNum < nlanes) || (outSz.width < nlanes)) - return false; - - bool xRatioEq = inSz.width == outSz.width; - bool yRatioEq = inSz.height == outSz.height; - - if (!xRatioEq && !yRatioEq) { - uchar _mask_horizontal[nlanes] = { 0, 4, 8, 12, 2, 6, 10, 14, - 1, 5, 9, 13, 3, 7, 11, 15 }; - if (4 == lpi) { - // vertical pass - vertical_4LPI(src0, src1, tmp, beta, inSz.width * chanNum); - - // horizontal pass - horizontal_4LPI(dst, tmp, mapsx, _mask_horizontal, clone, outSz.width); - } else { // if any lpi - int inLength = inSz.width * chanNum; - - for (int l = 0; l < lpi; ++l) { - short beta0 = beta[l]; - const uchar* s0 = src0[l]; - const uchar* s1 = src1[l]; - - // vertical pass - vertical_anyLPI(s0, s1, tmp, inLength, beta0); - - // horizontal pass - horizontal_anyLPI(dst, tmp, mapsx, alpha, outSz.width, l); - } - } - } else if (!xRatioEq) { - GAPI_DbgAssert(yRatioEq); - uchar _mask_horizontal[nlanes] = { 0, 4, 8, 12, 2, 6, 10, 14, - 1, 5, 9, 13, 3, 7, 11, 15 }; - - if (4 == lpi) { - int inLength = inSz.width * chanNum; - - // vertical pass - GAPI_DbgAssert(inLength >= nlanes); - v_uint8 s0, s1, s2, s3; - int w = 0; - for (;;) { - for (; w <= inLength - nlanes; w += nlanes) { - s0 = vx_load(&src0[0][w]); - s1 = vx_load(&src0[1][w]); - s2 = vx_load(&src0[2][w]); - s3 = vx_load(&src0[3][w]); - v_store_interleave(&tmp[lpi * w], s0, s1, s2, s3); - } - - if (w < inLength) { - w = inLength - nlanes; - continue; - } - break; - } - - // horizontal pass - horizontal_4LPI(dst, tmp, mapsx, _mask_horizontal, clone, outSz.width); - } else { // any LPI - for (int l = 0; l < lpi; ++l) { - const uchar* src = src0[l]; - - // horizontal pass - horizontal_anyLPI(dst, src, mapsx, alpha, outSz.width, l); - } - } - } else if (!yRatioEq) { - GAPI_DbgAssert(xRatioEq); - int inLength = inSz.width*chanNum; // == outSz.width - - for (int l = 0; l < lpi; ++l) { - short beta0 = beta[l]; - const uchar* s0 = src0[l]; - const uchar* s1 = src1[l]; - - // vertical pass - vertical_anyLPI(s0, s1, tmp, inLength, beta0); - - //split channels to planes and store - channels2planes_store(dst, tmp, outSz.width, l); - } - } else { - GAPI_DbgAssert(xRatioEq && yRatioEq); - - //split channels to planes and store - for (int l = 0; l < lpi; ++l) { - const uchar* src = src0[l]; - channels2planes_store(dst, src, outSz.width, l); - } - } - return true; -} - -CV_ALWAYS_INLINE void horizontal_4LPI(uint8_t* dst[], - const uchar* tmp, const short mapsx[], - const short clone[], const int length) { - constexpr int nlanes = static_cast(v_uint8::nlanes); - constexpr int half_nlanes = nlanes / 2; - GAPI_DbgAssert(length >= half_nlanes); - - uchar _mask_horizontal[nlanes] = { 0, 4, 8, 12, 2, 6, 10, 14, - 1, 5, 9, 13, 3, 7, 11, 15 }; - v_uint8 hmask = vx_load(_mask_horizontal); - int x = 0; - for (;;) { - for (; x <= length - half_nlanes; x += half_nlanes) { - v_int16 a10 = vx_load(&clone[4 * x]); - v_int16 a32 = vx_load(&clone[4 * (x + 2)]); - v_int16 a54 = vx_load(&clone[4 * (x + 4)]); - v_int16 a76 = vx_load(&clone[4 * (x + 6)]); - - v_uint8 val_0 = v_gather_lines(tmp, &mapsx[x]); - v_uint8 val_1 = v_gather_lines(tmp, &mapsx[x + 2]); - v_uint8 val_2 = v_gather_lines(tmp, &mapsx[x + 4]); - v_uint8 val_3 = v_gather_lines(tmp, &mapsx[x + 6]); - - v_int16 val0_0 = v_reinterpret_as_s16(v_expand_low(val_0)); - v_int16 val0_1 = v_reinterpret_as_s16(v_expand_low(val_1)); - v_int16 val0_2 = v_reinterpret_as_s16(v_expand_low(val_2)); - v_int16 val0_3 = v_reinterpret_as_s16(v_expand_low(val_3)); - - v_int16 val1_0 = v_reinterpret_as_s16(v_expand_high(val_0)); - v_int16 val1_1 = v_reinterpret_as_s16(v_expand_high(val_1)); - v_int16 val1_2 = v_reinterpret_as_s16(v_expand_high(val_2)); - v_int16 val1_3 = v_reinterpret_as_s16(v_expand_high(val_3)); - - v_int16 t0 = v_mulhrs(v_sub_wrap(val0_0, val1_0), a10); - v_int16 t1 = v_mulhrs(v_sub_wrap(val0_1, val1_1), a32); - v_int16 t2 = v_mulhrs(v_sub_wrap(val0_2, val1_2), a54); - v_int16 t3 = v_mulhrs(v_sub_wrap(val0_3, val1_3), a76); - - v_int16 r0 = v_add_wrap(val1_0, t0); - v_int16 r1 = v_add_wrap(val1_1, t1); - v_int16 r2 = v_add_wrap(val1_2, t2); - v_int16 r3 = v_add_wrap(val1_3, t3); - - v_uint8 q0 = v_pack_u(r0, r1); - v_uint8 q1 = v_pack_u(r2, r3); - - v_uint8 q2 = v_shuffle(q0, hmask); - v_uint8 q3 = v_shuffle(q1, hmask); - - v_uint8 q4 = v_blend<0xCC /*0b11001100*/>(q2, v_shift_left<4>(q3)); - v_uint8 q5 = v_blend<0xCC /*0b11001100*/>(v_shift_right<4>(q2), q3); - - v_store_low(&dst[0][x], q4); - v_store_high(&dst[1][x], q4); - v_store_low(&dst[2][x], q5); - v_store_high(&dst[3][x], q5); - } - - if (x < length) { - x = length - half_nlanes; - continue; - } - break; - } -} - -CV_ALWAYS_INLINE void horizontal_anyLPI(uint8_t* dst, - const uchar* src, const short mapsx[], - const short alpha[], const int length) { - constexpr int nlanes = static_cast(v_uint8::nlanes); - constexpr int half_nlanes = nlanes / 2; - GAPI_DbgAssert(length >= half_nlanes); - - v_int16 t0, t1; - int x = 0; - for (;;) { - for (; x <= length - half_nlanes; x += half_nlanes) { - v_int16 a0 = vx_load(&alpha[x]); - v_uint8 t = v_gather_pairs(src, &mapsx[x]); - - v_deinterleave_expand(t, t0, t1); - v_int16 d = v_mulhrs(t0 - t1, a0) + t1; - v_pack_u_store(&dst[x], d); - } - - if (x < length) { - x = length - half_nlanes; - continue; - } - break; - } -} -} // namespace neon - -template<> -bool calcRowLinear8UC3C4Impl(neon_tag, - std::array, 3> &dst, - const uint8_t* src0[], - const uint8_t* src1[], - const short alpha[], - const short clone[], // 4 clones of alpha - const short mapsx[], - const short beta[], - uint8_t tmp[], - const Size& inSz, - const Size& outSz, - const int lpi, - const int ) { - constexpr int chanNum = 3; - return neon::calcRowLinear_8UC_Impl(neon_tag{}, dst, src0, src1, alpha, clone, mapsx, beta, tmp, inSz, outSz, lpi); -} - -// Resize (bi-linear, 8UC4) -template<> -bool calcRowLinear8UC3C4Impl(neon_tag, - std::array, 4> &dst, - const uint8_t* src0[], - const uint8_t* src1[], - const short alpha[], - const short clone[], // 4 clones of alpha - const short mapsx[], - const short beta[], - uint8_t tmp[], - const Size& inSz, - const Size& outSz, - const int lpi, - const int ) { - constexpr int chanNum = 4; - return neon::calcRowLinear_8UC_Impl(neon_tag{}, dst, src0, src1, alpha, clone, mapsx, beta, tmp, inSz, outSz, lpi); -} - -// 8UC1 Resize (bi-linear) -template<> -bool calcRowLinear8UC1Impl(neon_tag, - uint8_t* dst[], - const uint8_t* src0[], - const uint8_t* src1[], - const short alpha[], - const short clone[], // 4 clones of alpha - const short mapsx[], - const short beta[], - uint8_t tmp[], - const Size& inSz, - const Size& outSz, - const int lpi, - const int) { - static_assert(v_uint8::nlanes == 16, - "The wide of NEON vector is 128 bits, so one vector contains 16 uchars"); - - constexpr int nlanes = static_cast(v_uint8::nlanes); - constexpr int half_nlanes = v_uint8::nlanes / 2; - - if (inSz.width < nlanes || outSz.width < half_nlanes) - return false; - - bool xRatioEq = inSz.width == outSz.width; - bool yRatioEq = inSz.height == outSz.height; - - if (!xRatioEq && !yRatioEq) { - if (4 == lpi) { - // vertical pass - neon::vertical_4LPI(src0, src1, tmp, beta, inSz.width); - - // horizontal pass - neon::horizontal_4LPI(dst, tmp, mapsx, clone, outSz.width); - } else { // if any lpi - for (int l = 0; l < lpi; ++l) { - short beta0 = beta[l]; - const uchar* s0 = src0[l]; - const uchar* s1 = src1[l]; - uchar* _dst = dst[l]; - - // vertical pass - neon::vertical_anyLPI(s0, s1, tmp, inSz.width, beta0); - - // horizontal pass - neon::horizontal_anyLPI(_dst, tmp, mapsx, alpha, outSz.width); - } - } // if lpi == 4 - - } else if (!xRatioEq) { - GAPI_DbgAssert(yRatioEq); - GAPI_DbgAssert(inSz.width >= nlanes); - - if (4 == lpi) { - // vertical pass - int w = 0; - for (;;) { - for (; w <= inSz.width - nlanes; w += nlanes) { - v_uint8 s0 = vx_load(&src0[0][w]); - v_uint8 s1 = vx_load(&src0[1][w]); - v_uint8 s2 = vx_load(&src0[2][w]); - v_uint8 s3 = vx_load(&src0[3][w]); - v_store_interleave(&tmp[4 * w], s0, s1, s2, s3); - } - - if (w < inSz.width) { - w = inSz.width - nlanes; - continue; - } - break; - } - - // horizontal pass - neon::horizontal_4LPI(dst, tmp, mapsx, clone, outSz.width); - - } else { // any LPI - for (int l = 0; l < lpi; ++l) { - const uchar* src = src0[l]; - uchar* _dst = dst[l]; - - // horizontal pass - neon::horizontal_anyLPI(_dst, src, mapsx, alpha, outSz.width); - } - } - - } else if (!yRatioEq) { - GAPI_DbgAssert(xRatioEq); - int length = inSz.width; // == outSz.width - - for (int l = 0; l < lpi; ++l) { - short beta0 = beta[l]; - const uchar* s0 = src0[l]; - const uchar* s1 = src1[l]; - - // vertical pass - neon::vertical_anyLPI(s0, s1, dst[l], length, beta0); - } - - } else { - GAPI_DbgAssert(xRatioEq && yRatioEq); - int length = inSz.width; // == outSz.width - - for (int l = 0; l < lpi; ++l) { - memcpy(dst[l], src0[l], length); - } - } - return true; -} - -template void chanToPlaneRowImpl(neon_tag, const uint8_t* in, int chan, int chs, uint8_t* out, const int length); -template void chanToPlaneRowImpl(neon_tag, const float* in, int chan, int chs, float * out, const int length); - -template void nv12ToRgbRowImpl(neon_tag, const uint8_t** y_rows, const uint8_t* uv_row, uint8_t** out_rows, const int buf_width); - -template void i420ToRgbRowImpl(neon_tag, const uint8_t** y_rows, const uint8_t* u_row, - const uint8_t* v_row, uint8_t** out_rows, const int buf_width); - -template void splitRowImpl(neon_tag, const uint8_t* in, std::array& outs, const int length); -template void splitRowImpl(neon_tag, const float* in, std::array& outs, const int length); -template void splitRowImpl(neon_tag, const uint8_t* in, std::array& outs, const int length); -template void splitRowImpl(neon_tag, const float* in, std::array& outs, const int length); -template void splitRowImpl(neon_tag, const uint8_t* in, std::array& outs, const int length); -template void splitRowImpl(neon_tag, const float* in, std::array& outs, const int length); - -template void mergeRowImpl(neon_tag, const std::array& ins, uint8_t* out, const int length); -template void mergeRowImpl(neon_tag, const std::array& ins, float* out, const int length); -template void mergeRowImpl(neon_tag, const std::array& ins, uint8_t* out, const int length); -template void mergeRowImpl(neon_tag, const std::array& ins, float* out, const int length); -template void mergeRowImpl(neon_tag, const std::array& ins, uint8_t* out, const int length); -template void mergeRowImpl(neon_tag, const std::array& ins, float* out, const int length); - -template void calcRowLinear32FC1Impl(neon_tag, float* dst[], const float* src0[], const float* src1[], - const float alpha[], const int mapsx[], const float beta[], - const Size& inSz, const Size& outSz, const int lpi, const int l); - -template void calcRowAreaImpl(neon_tag, uint8_t dst[], const uint8_t* src[], const Size& inSz, - const Size& outSz, Q0_16 yalpha, const MapperUnit8U &ymap, - int xmaxdf, const short xindex[], const Q0_16 xalpha[], - Q8_8 vbuf[]); - -template void calcRowAreaImpl(neon_tag, float dst[], const float *src[], const Size& inSz, - const Size& outSz, float yalpha, const MapperUnit32F& ymap, - int xmaxdf, const int xindex[], const float xalpha[], - float vbuf[]); -} // namespace kernels -} // namespace gapi -} // namespace InferenceEngine diff --git a/src/common/preprocessing/src/arm_neon/ie_preprocess_gapi_kernels_neon.hpp b/src/common/preprocessing/src/arm_neon/ie_preprocess_gapi_kernels_neon.hpp deleted file mode 100644 index deb76e5c8f8..00000000000 --- a/src/common/preprocessing/src/arm_neon/ie_preprocess_gapi_kernels_neon.hpp +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright (C) 2018-2023 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#pragma once - -#include "ie_preprocess_gapi_kernels.hpp" -#include "ie_preprocess_gapi_kernels_impl.hpp" -#include - -namespace InferenceEngine { -namespace gapi { -namespace kernels { -namespace neon { - -using C3 = std::integral_constant; -using C4 = std::integral_constant; -//----------------------------------------------------------------------------- - -typedef MapperUnit MapperUnit32F; -typedef MapperUnit MapperUnit8U; - -void calcRowArea_8U(uchar dst[], const uchar *src[], const Size &inSz, const Size &outSz, - Q0_16 yalpha, const MapperUnit8U& ymap, int xmaxdf, const short xindex[], - const Q0_16 xalpha[], Q8_8 vbuf[]); - -void calcRowArea_32F(float dst[], const float *src[], const Size &inSz, const Size &outSz, - float yalpha, const MapperUnit32F& ymap, int xmaxdf, const int xindex[], - const float xalpha[], float vbuf[]); -} // namespace neon - -template -void chanToPlaneRowImpl(isa_tag_t, const T* in, const int chan, const int chs, T* out, const int length); - -template -void nv12ToRgbRowImpl(isa_tag_t, const uint8_t** y_rows, const uint8_t* uv_row, uint8_t** out_rows, const int buf_width); - -template -void i420ToRgbRowImpl(isa_tag_t, const uint8_t** y_rows, const uint8_t* u_row, - const uint8_t* v_row, uint8_t** out_rows, const int buf_width); - -template -void splitRowImpl(isa_tag_t, const T* in, std::array& outs, const int length); - -template -void mergeRowImpl(isa_tag_t, const std::array& ins, T* out, const int length); - -template -bool calcRowLinear8UC1Impl(isa_tag_t, uint8_t* dst[], const uint8_t* src0[], const uint8_t* src1[], - const short alpha[], const short clone[], const short mapsx[], - const short beta[], uint8_t tmp[], const Size& inSz, - const Size& outSz, const int lpi, const int l); - -template -void calcRowLinear32FC1Impl(isa_tag_t, float* dst[], const float* src0[], const float* src1[], - const float alpha[], const int mapsx[], - const float beta[], const Size& inSz, const Size& outSz, - const int lpi, const int l); - -template -bool calcRowLinear8UC3C4Impl(isa_tag_t, std::array, chs>& dst, - const uint8_t* src0[], const uint8_t* src1[], - const short alpha[], const short clone[], const short mapsx[], - const short beta[], uint8_t tmp[], const Size& inSz, - const Size& outSz, const int lpi, const int l); - -template -void calcRowAreaImpl(isa_tag_t, T dst[], const T* src[], const Size& inSz, - const Size& outSz, A yalpha, const MapperUnit& ymap, - int xmaxdf, const I xindex[], const A xalpha[], W vbuf[]); -} // namespace kernels -} // namespace gapi -} // namespace InferenceEngine diff --git a/src/common/preprocessing/src/cpu_x86_avx2/ie_preprocess_gapi_kernels_avx2.cpp b/src/common/preprocessing/src/cpu_x86_avx2/ie_preprocess_gapi_kernels_avx2.cpp deleted file mode 100644 index 26c5dd6d676..00000000000 --- a/src/common/preprocessing/src/cpu_x86_avx2/ie_preprocess_gapi_kernels_avx2.cpp +++ /dev/null @@ -1,529 +0,0 @@ -// Copyright (C) 2018-2023 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#include -#include - -#include "ie_preprocess_gapi_kernels_avx2.hpp" - -#include - -#ifdef CV_AVX2 -#undef CV_AVX2 -#endif - -#define CV_AVX2 1 - -#define CV_CPU_HAS_SUPPORT_SSE2 1 - -#ifdef CV_SIMD256 -#undef CV_SIMD256 -#endif - -#define CV_SIMD256 1 - -#include "opencv_hal_intrin.hpp" -#include "ie_preprocess_gapi_kernels_simd_impl.hpp" - -#if !CV_SIMD256 -#error CV_SIMD256 is required! -#endif - -#include - -#if defined __GNUC__ -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wstrict-overflow" -#endif - -using namespace cv; - -namespace InferenceEngine { - -namespace gapi { - -namespace kernels { - -namespace avx { - -CV_ALWAYS_INLINE void main_computation_horizontalPass_lpi4(const v_uint8& val_0, - const v_uint8& val_1, - const v_uint8& val_2, - const v_uint8& val_3, - const v_int16& a10, - const v_int16& a32, - const v_int16& a54, - const v_int16& a76, - v_uint8& shuf_mask1, - v_uint8& shuf_mask2, - v_uint8& res1, v_uint8& res2) { - v_int16 val0_0 = v_reinterpret_as_s16(v_expand_low(val_0)); - v_int16 val0_1 = v_reinterpret_as_s16(v_expand_low(val_1)); - v_int16 val0_2 = v_reinterpret_as_s16(v_expand_low(val_2)); - v_int16 val0_3 = v_reinterpret_as_s16(v_expand_low(val_3)); - - v_int16 val1_0 = v_reinterpret_as_s16(v_expand_high(val_0)); - v_int16 val1_1 = v_reinterpret_as_s16(v_expand_high(val_1)); - v_int16 val1_2 = v_reinterpret_as_s16(v_expand_high(val_2)); - v_int16 val1_3 = v_reinterpret_as_s16(v_expand_high(val_3)); - - v_int16 t0 = v_mulhrs(v_sub_wrap(val0_0, val1_0), a10); - v_int16 t1 = v_mulhrs(v_sub_wrap(val0_1, val1_1), a32); - v_int16 t2 = v_mulhrs(v_sub_wrap(val0_2, val1_2), a54); - v_int16 t3 = v_mulhrs(v_sub_wrap(val0_3, val1_3), a76); - - v_int16 r0 = v_add_wrap(val1_0, t0); - v_int16 r1 = v_add_wrap(val1_1, t1); - v_int16 r2 = v_add_wrap(val1_2, t2); - v_int16 r3 = v_add_wrap(val1_3, t3); - - v_uint8 q0 = v_packus(r0, r1); - v_uint8 q1 = v_packus(r2, r3); - - v_uint8 q2 = v_shuffle_s8(q0, shuf_mask1); - v_uint8 q3 = v_shuffle_s8(q1, shuf_mask1); - - v_uint8 q4 = v_blend_shiftleft<0xCC /*0b11001100*/, 4>(q2, q3); - v_uint8 q5 = v_blend_shiftright<0xCC /*0b11001100*/, 4>(q2, q3); - - v_uint8 q6 = v256_permute4x64<0xD8>(q4); - v_uint8 q7 = v256_permute4x64<0xD8>(q5); - - res1 = v_shuffle_s8(q6, shuf_mask2); - res2 = v_shuffle_s8(q7, shuf_mask2); -} - -CV_ALWAYS_INLINE void verticalPass_lpi4_8U(const uint8_t* src0[], const uint8_t* src1[], - uint8_t tmp[], const short beta[], - const int& length) { - constexpr int half_nlanes = (v_uint8::nlanes / 2); - GAPI_DbgAssert(length >= half_nlanes); - - v_int16 b0 = vx_setall_s16(beta[0]); - v_int16 b1 = vx_setall_s16(beta[1]); - v_int16 b2 = vx_setall_s16(beta[2]); - v_int16 b3 = vx_setall_s16(beta[3]); - - v_uint8 shuf_mask = v_setr_s8(0, 8, 4, 12, 1, 9, 5, 13, - 2, 10, 6, 14, 3, 11, 7, 15, - 0, 8, 4, 12, 1, 9, 5, 13, - 2, 10, 6, 14, 3, 11, 7, 15); - for (int w = 0; w < length; ) { - for (; w <= length - half_nlanes; w += half_nlanes) { - v_int16 val0_0 = v_load_ccache_expand(&src0[0][w]); - v_int16 val0_1 = v_load_ccache_expand(&src0[1][w]); - v_int16 val0_2 = v_load_ccache_expand(&src0[2][w]); - v_int16 val0_3 = v_load_ccache_expand(&src0[3][w]); - - v_int16 val1_0 = v_load_ccache_expand(&src1[0][w]); - v_int16 val1_1 = v_load_ccache_expand(&src1[1][w]); - v_int16 val1_2 = v_load_ccache_expand(&src1[2][w]); - v_int16 val1_3 = v_load_ccache_expand(&src1[3][w]); - - v_int16 t0 = v_mulhrs(v_sub_wrap(val0_0, val1_0), b0); - v_int16 t1 = v_mulhrs(v_sub_wrap(val0_1, val1_1), b1); - v_int16 t2 = v_mulhrs(v_sub_wrap(val0_2, val1_2), b2); - v_int16 t3 = v_mulhrs(v_sub_wrap(val0_3, val1_3), b3); - - v_int16 r0 = v_add_wrap(val1_0, t0); - v_int16 r1 = v_add_wrap(val1_1, t1); - v_int16 r2 = v_add_wrap(val1_2, t2); - v_int16 r3 = v_add_wrap(val1_3, t3); - - v_uint8 q0 = v_packus(r0, r1); - v_uint8 q1 = v_packus(r2, r3); - - v_uint8 q2 = v_blend_shiftleft<0xCC /*0b11001100*/, 4>(q0, q1); - v_uint8 q3 = v_blend_shiftright<0xCC /*0b11001100*/, 4>(q0, q1); - - v_uint8 q4 = v_shuffle_s8(q2, shuf_mask); - v_uint8 q5 = v_shuffle_s8(q3, shuf_mask); - - v_uint8 q6 = v256_permute2x128<0x20>(q4, q5); - v_uint8 q7 = v256_permute2x128<0x31>(q4, q5); - - vx_store(&tmp[4 * w + 0], q6); - vx_store(&tmp[4 * w + 2 * half_nlanes], q7); - } - - if (w < length) { - w = length - half_nlanes; - } - } -} - -CV_ALWAYS_INLINE v_uint8 setHorizontalShufMask1() { - return v_setr_s8(0, 4, 8, 12, 2, 6, 10, 14, - 1, 5, 9, 13, 3, 7, 11, 15, - 0, 4, 8, 12, 2, 6, 10, 14, - 1, 5, 9, 13, 3, 7, 11, 15); -} - -CV_ALWAYS_INLINE v_uint8 setHorizontalShufMask2() { - return v_setr_s8(0, 1, 8, 9, 2, 3, 10, 11, - 4, 5, 12, 13, 6, 7, 14, 15, - 0, 1, 8, 9, 2, 3, 10, 11, - 4, 5, 12, 13, 6, 7, 14, 15); -} - -CV_ALWAYS_INLINE void verticalPass_anylpi_8U(const uint8_t* src0[], const uint8_t* src1[], - uint8_t tmp[], const int& beta0, - const int l, const int length1, const int length2) { - constexpr int half_nlanes = (v_uint8::nlanes / 2); - GAPI_DbgAssert(length1 >= half_nlanes); - - for (int w = 0; w < length2; ) { - for (; w <= length1 - half_nlanes; w += half_nlanes) { - v_int16 s0 = v_reinterpret_as_s16(vx_load_expand(&src0[l][w])); - v_int16 s1 = v_reinterpret_as_s16(vx_load_expand(&src1[l][w])); - v_int16 t = v_mulhrs(s0 - s1, beta0) + s1; - v_pack_u_store(tmp + w, t); - } - - if (w < length1) { - w = length1 - half_nlanes; - } - } -} - -template -CV_ALWAYS_INLINE bool calcRowLinear_8UC_Impl(avx2_tag, - std::array, chanNum> &dst, - const uint8_t* src0[], - const uint8_t* src1[], - const short alpha[], - const short clone[], // 4 clones of alpha - const short mapsx[], - const short beta[], - uint8_t tmp[], - const Size& inSz, - const Size& outSz, - const int lpi) { - constexpr int half_nlanes = (v_uint8::nlanes / 2); - if ((inSz.width * chanNum < half_nlanes) || (outSz.width < half_nlanes)) - return false; - - const int shift = (half_nlanes / 4); - - if (4 == lpi) { - GAPI_DbgAssert(inSz.width*chanNum >= half_nlanes); - verticalPass_lpi4_8U(src0, src1, tmp, beta, - inSz.width*chanNum); - - // horizontal pass - GAPI_DbgAssert(outSz.width >= half_nlanes); - //This variables are here to initialize them once. This variant don't affect performance. - v_uint8 val_0, val_1, val_2, val_3, res1, res2; - - v_uint8 shuf_mask1 = setHorizontalShufMask1(); - v_uint8 shuf_mask2 = setHorizontalShufMask2(); - - for (int x = 0; x < outSz.width; ) { - for (; x <= outSz.width - half_nlanes && x >= 0; x += half_nlanes) { - v_int16 a10 = vx_load(&clone[4 * x]); - v_int16 a32 = vx_load(&clone[4 * (x + 4)]); - v_int16 a54 = vx_load(&clone[4 * (x + 8)]); - v_int16 a76 = vx_load(&clone[4 * (x + 12)]); - - for (int c = 0; c < chanNum; ++c) { - v_gather_channel(val_0, tmp, mapsx, chanNum, c, x, 0); - v_gather_channel(val_1, tmp, mapsx, chanNum, c, x, shift); - v_gather_channel(val_2, tmp, mapsx, chanNum, c, x, shift * 2); - v_gather_channel(val_3, tmp, mapsx, chanNum, c, x, shift * 3); - - main_computation_horizontalPass_lpi4(val_0, val_1, val_2, val_3, - a10, a32, a54, a76, - shuf_mask1, shuf_mask2, - res1, res2); - - v_store_low(&dst[c][0][x], res1); - v_store_high(&dst[c][1][x], res1); - v_store_low(&dst[c][2][x], res2); - v_store_high(&dst[c][3][x], res2); - } - } - - if (x < outSz.width) { - x = outSz.width - half_nlanes; - } - } - } else { // if any lpi - for (int l = 0; l < lpi; ++l) { - short beta0 = beta[l]; - - // vertical pass - verticalPass_anylpi_8U(src0, src1, tmp, beta0, l, - inSz.width*chanNum, inSz.width*chanNum); - - // horizontal pass - GAPI_DbgAssert(outSz.width >= half_nlanes); - for (int x = 0; x < outSz.width; ) { - for (; x <= outSz.width - half_nlanes && x >= 0; x += half_nlanes) { - for (int c = 0; c < chanNum; ++c) { - v_int16 a0 = vx_load(&alpha[x]); // as signed Q1.1.14 - v_int16 sx = vx_load(&mapsx[x]); // as integer (int16) - v_int16 t0 = v_gather_chan(tmp, sx, c, 0); - v_int16 t1 = v_gather_chan(tmp, sx, c, 1); - v_int16 d = v_mulhrs(t0 - t1, a0) + t1; - v_pack_u_store(&dst[c][l][x], d); - } - } - - if (x < outSz.width) { - x = outSz.width - half_nlanes; - } - } - } - } - return true; -} - - -CV_ALWAYS_INLINE void horizontalPass_lpi4_8UC1(const short clone[], const short mapsx[], - uint8_t tmp[], uint8_t* dst[], const int& length) { - constexpr int half_nlanes = (v_uint8::nlanes / 2); - GAPI_DbgAssert(length >= half_nlanes); - - v_uint8 val_0, val_1, val_2, val_3, res1, res2; - constexpr int shift = 4; - v_uint8 shuf_mask1 = avx::setHorizontalShufMask1(); - v_uint8 shuf_mask2 = avx::setHorizontalShufMask2(); - - v_uint32 idxs = v_setr_s32(0, 2, 4, 6, 1, 3, 5, 7); - - for (int x = 0; x < length; ) { - for (; x <= length - half_nlanes; x += half_nlanes) { - v_int16 a10 = vx_load(&clone[4 * x]); - v_int16 a32 = vx_load(&clone[4 * (x + 4)]); - v_int16 a54 = vx_load(&clone[4 * (x + 8)]); - v_int16 a76 = vx_load(&clone[4 * (x + 12)]); - - v_setr64(val_0, val_1, val_2, val_3, mapsx, tmp, x, shift); - val_0 = v_permute32(val_0, idxs); - val_1 = v_permute32(val_1, idxs); - val_2 = v_permute32(val_2, idxs); - val_3 = v_permute32(val_3, idxs); - - avx::main_computation_horizontalPass_lpi4(val_0, val_1, val_2, val_3, - a10, a32, a54, a76, - shuf_mask1, shuf_mask2, - res1, res2); - - v_store_low(&dst[0][x], res1); - v_store_high(&dst[1][x], res1); - v_store_low(&dst[2][x], res2); - v_store_high(&dst[3][x], res2); - } - - if (x < length) { - x = length - half_nlanes; - } - } -} - -CV_ALWAYS_INLINE void horizontalPass_anylpi_8U(const short alpha[], const short mapsx[], - uint8_t* dst[], const uchar tmp[], const int l, - const int length) { - constexpr int half_nlanes = (v_uint8::nlanes / 2); - GAPI_DbgAssert(length >= half_nlanes); - - v_int16 t0, t1; - for (int x = 0; x < length; ) { - for (; x <= length - half_nlanes; x += half_nlanes) { - v_int16 a0 = vx_load(&alpha[x]); // as signed Q1.1.14 - v_int16 sx = vx_load(&mapsx[x]); // as integer (int16) - v_uint8 t = v_gather_pairs(tmp, sx); // 16 pairs of src0 pixels - - v_deinterleave_expand(t, t0, t1); // tmp pixels as int16 - v_int16 d = v_mulhrs(t0 - t1, a0) + t1; - v_pack_u_store(&dst[l][x], d); - } - - if (x < length) { - x = length - half_nlanes; - } - } -} -} // namespace avx - -// Resize (bi-linear, 8UC3) -template<> -bool calcRowLinear8UC3C4Impl(avx2_tag, - std::array, 3> &dst, - const uint8_t* src0[], - const uint8_t* src1[], - const short alpha[], - const short clone[], // 4 clones of alpha - const short mapsx[], - const short beta[], - uint8_t tmp[], - const Size& inSz, - const Size& outSz, - const int lpi, - const int ) { - constexpr int chanNum = 3; - return avx::calcRowLinear_8UC_Impl(avx2_tag{}, dst, src0, src1, alpha, clone, mapsx, beta, tmp, inSz, outSz, lpi); -} - -// Resize (bi-linear, 8UC4) -template<> -bool calcRowLinear8UC3C4Impl(avx2_tag, - std::array, 4> &dst, - const uint8_t* src0[], - const uint8_t* src1[], - const short alpha[], - const short clone[], // 4 clones of alpha - const short mapsx[], - const short beta[], - uint8_t tmp[], - const Size& inSz, - const Size& outSz, - const int lpi, - const int ) { - constexpr int chanNum = 4; - return avx::calcRowLinear_8UC_Impl(avx2_tag{}, dst, src0, src1, alpha, clone, mapsx, beta, tmp, inSz, outSz, lpi); -} - -// 8UC1 Resize (bi-linear) -template<> -bool calcRowLinear8UC1Impl(avx2_tag, - uint8_t* dst[], - const uint8_t* src0[], - const uint8_t* src1[], - const short alpha[], - const short clone[], // 4 clones of alpha - const short mapsx[], - const short beta[], - uint8_t tmp[], - const Size& inSz, - const Size& outSz, - const int lpi, - const int) { - constexpr int nlanes = v_uint8::nlanes; - constexpr int half_nlanes = (v_uint8::nlanes / 2); - - if (inSz.width < nlanes || outSz.width < half_nlanes) - return false; - - bool xRatioEq = inSz.width == outSz.width; - bool yRatioEq = inSz.height == outSz.height; - - if (!xRatioEq && !yRatioEq) { - if (4 == lpi) { - // vertical pass - avx::verticalPass_lpi4_8U(src0, src1, tmp, beta, inSz.width); - - // horizontal pass - avx::horizontalPass_lpi4_8UC1(clone, mapsx, tmp, dst, outSz.width); - - } else { // if any lpi - int inLength = inSz.width; - int outLength = outSz.width; - for (int l = 0; l < lpi; ++l) { - short beta0 = beta[l]; - - // vertical pass - avx::verticalPass_anylpi_8U(src0, src1, tmp, beta0, l, inLength, inLength); - - // horizontal pass - avx::horizontalPass_anylpi_8U(alpha, mapsx, dst, tmp, l, outLength); - } - } // if lpi == 4 - - } else if (!xRatioEq) { - GAPI_DbgAssert(yRatioEq); - - if (4 == lpi) { - // vertical pass - GAPI_DbgAssert(inSz.width >= nlanes); - v_uint8 s0, s1, s2, s3; - for (int w = 0; w < inSz.width; ) { - for (; w <= inSz.width - nlanes; w += nlanes) { - s0 = vx_load(&src0[0][w]); - s1 = vx_load(&src0[1][w]); - s2 = vx_load(&src0[2][w]); - s3 = vx_load(&src0[3][w]); - v_store_interleave(&tmp[4 * w], s0, s1, s2, s3); - } - - if (w < inSz.width) { - w = inSz.width - nlanes; - } - } - - // horizontal pass - avx::horizontalPass_lpi4_8UC1(clone, mapsx, tmp, dst, outSz.width); - - } else { // any LPI - for (int l = 0; l < lpi; ++l) { - const uchar* src = src0[l]; - - // horizontal pass - avx::horizontalPass_anylpi_8U(alpha, mapsx, dst, src, l, outSz.width); - } - } - - } else if (!yRatioEq) { - GAPI_DbgAssert(xRatioEq); - int inLength = inSz.width; - int outLength = outSz.width; - - for (int l = 0; l < lpi; ++l) { - short beta0 = beta[l]; - - // vertical pass - avx::verticalPass_anylpi_8U(src0, src1, dst[l], beta0, l, inLength, outLength); - } - - } else { - GAPI_DbgAssert(xRatioEq && yRatioEq); - int length = inSz.width; - - for (int l = 0; l < lpi; ++l) { - memcpy(dst[l], src0[l], length); - } - } - return true; -} - -template void chanToPlaneRowImpl(avx2_tag, const uint8_t* in, const int chan, const int chs, uint8_t* out, const int length); -template void chanToPlaneRowImpl(avx2_tag, const float* in, const int chan, const int chs, float* out, const int length); - -template void nv12ToRgbRowImpl(avx2_tag, const uint8_t** y_rows, const uint8_t* uv_row, - uint8_t** out_rows, const int buf_width); - -template void i420ToRgbRowImpl(avx2_tag, const uint8_t** y_rows, const uint8_t* u_row, - const uint8_t* v_row, uint8_t** out_rows, const int buf_width); - -template void splitRowImpl(avx2_tag, const uint8_t* in, std::array& outs, const int length); -template void splitRowImpl(avx2_tag, const float* in, std::array& outs, const int length); -template void splitRowImpl(avx2_tag, const uint8_t* in, std::array& outs, const int length); -template void splitRowImpl(avx2_tag, const float* in, std::array& outs, const int length); -template void splitRowImpl(avx2_tag, const uint8_t* in, std::array& outs, const int length); -template void splitRowImpl(avx2_tag, const float* in, std::array& outs, const int length); - -template void mergeRowImpl(avx2_tag, const std::array& ins, uint8_t* out, const int length); -template void mergeRowImpl(avx2_tag, const std::array& ins, float* out, const int length); -template void mergeRowImpl(avx2_tag, const std::array& ins, uint8_t* out, const int length); -template void mergeRowImpl(avx2_tag, const std::array& ins, float* out, const int length); -template void mergeRowImpl(avx2_tag, const std::array& ins, uint8_t* out, const int length); -template void mergeRowImpl(avx2_tag, const std::array& ins, float* out, const int length); - -template void calcRowLinear32FC1Impl(avx2_tag, float* dst[], const float* src0[], const float* src1[], - const float alpha[], const int mapsx[], - const float beta[], const Size& inSz, const Size& outSz, - const int lpi, const int l); - -template void calcRowAreaImpl(avx2_tag, uint8_t dst[], const uint8_t* src[], const Size& inSz, - const Size& outSz, Q0_16 yalpha, const MapperUnit8U &ymap, - int xmaxdf, const short xindex[], const Q0_16 xalpha[], - Q8_8 vbuf[]); - -template void calcRowAreaImpl(avx2_tag, float dst[], const float *src[], const Size& inSz, - const Size& outSz, float yalpha, const MapperUnit32F& ymap, - int xmaxdf, const int xindex[], const float xalpha[], - float vbuf[]); -} // namespace kernels -} // namespace gapi -} // namespace InferenceEngine diff --git a/src/common/preprocessing/src/cpu_x86_avx2/ie_preprocess_gapi_kernels_avx2.hpp b/src/common/preprocessing/src/cpu_x86_avx2/ie_preprocess_gapi_kernels_avx2.hpp deleted file mode 100644 index 1f856b66b75..00000000000 --- a/src/common/preprocessing/src/cpu_x86_avx2/ie_preprocess_gapi_kernels_avx2.hpp +++ /dev/null @@ -1,86 +0,0 @@ -// Copyright (C) 2018-2023 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#pragma once - -#include "ie_preprocess_gapi_kernels.hpp" -#include "ie_preprocess_gapi_kernels_impl.hpp" -#include - -namespace InferenceEngine { -namespace gapi { -namespace kernels { -namespace avx { - -using C3 = std::integral_constant; -using C4 = std::integral_constant; -//---------------------------------------------------------------------- - -void calcRowArea_8U(uchar dst[], const uchar *src[], const Size &inSz, const Size &outSz, - Q0_16 yalpha, const MapperUnit8U& ymap, int xmaxdf, const short xindex[], - const Q0_16 xalpha[], Q8_8 vbuf[]); - -void calcRowArea_32F(float dst[], const float *src[], const Size &inSz, const Size &outSz, - float yalpha, const MapperUnit32F& ymap, int xmaxdf, const int xindex[], - const float xalpha[], float vbuf[]); - -#if USE_CVKL -void calcRowArea_CVKL_U8_SSE42(const uchar * src[], - uchar dst[], - const Size & inSz, - const Size & outSz, - int y, - const uint16_t xsi[], - const uint16_t ysi[], - const uint16_t xalpha[], - const uint16_t yalpha[], - int x_max_count, - int y_max_count, - uint16_t vert_sum[]); -#endif -} // namespace avx - -template -void chanToPlaneRowImpl(isa_tag_t, const T* in, const int chan, const int chs, T* out, const int length); - -template -void nv12ToRgbRowImpl(isa_tag_t, const uint8_t** y_rows, const uint8_t* uv_row, - uint8_t** out_rows, const int buf_width); - -template -void i420ToRgbRowImpl(isa_tag_t, const uint8_t** y_rows, const uint8_t* u_row, - const uint8_t* v_row, uint8_t** out_rows, const int buf_width); - -template -void splitRowImpl(isa_tag_t, const T* in, std::array& outs, const int length); - -template -void mergeRowImpl(isa_tag_t, const std::array& ins, T* out, const int length); - -template -bool calcRowLinear8UC1Impl(isa_tag_t, uint8_t* dst[], const uint8_t* src0[], const uint8_t* src1[], - const short alpha[], const short clone[], const short mapsx[], - const short beta[], uint8_t tmp[], const Size& inSz, - const Size& outSz, const int lpi, const int l); - -template -void calcRowLinear32FC1Impl(isa_tag_t, float* dst[], const float* src0[], const float* src1[], - const float alpha[], const int mapsx[], - const float beta[], const Size& inSz, const Size& outSz, - const int lpi, const int l); - -template -bool calcRowLinear8UC3C4Impl(isa_tag_t, std::array, chs>& dst, - const uint8_t* src0[], const uint8_t* src1[], - const short alpha[], const short clone[], const short mapsx[], - const short beta[], uint8_t tmp[], const Size& inSz, - const Size& outSz, const int lpi, const int l); - -template -void calcRowAreaImpl(isa_tag_t, T dst[], const T* src[], const Size& inSz, - const Size& outSz, A yalpha, const MapperUnit& ymap, - int xmaxdf, const I xindex[], const A xalpha[], W vbuf[]); -} // namespace kernels -} // namespace gapi -} // namespace InferenceEngine diff --git a/src/common/preprocessing/src/cpu_x86_avx512/ie_preprocess_gapi_kernels_avx512.cpp b/src/common/preprocessing/src/cpu_x86_avx512/ie_preprocess_gapi_kernels_avx512.cpp deleted file mode 100644 index 34772f7ee17..00000000000 --- a/src/common/preprocessing/src/cpu_x86_avx512/ie_preprocess_gapi_kernels_avx512.cpp +++ /dev/null @@ -1,557 +0,0 @@ -// Copyright (C) 2018-2023 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#include -#include - -#include "ie_preprocess_gapi_kernels_avx512.hpp" - -#include - -#ifdef CV_AVX512_SKX -#undef CV_AVX512_SKX -#endif - -#define CV_AVX512_SKX 1 - -#define CV_CPU_HAS_SUPPORT_SSE2 1 - -#ifdef CV_SIMD512 -#undef CV_SIMD512 -#endif - -#define CV_SIMD512 1 - -#include "opencv_hal_intrin.hpp" -#include "ie_preprocess_gapi_kernels_simd_impl.hpp" - -using namespace cv; - -#if defined __GNUC__ -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wstrict-overflow" -#endif - -namespace InferenceEngine { - -namespace gapi { - -namespace kernels { - -namespace avx512 { - -CV_ALWAYS_INLINE void verticalPass_lpi4_8U(const uint8_t* src0[], const uint8_t* src1[], - uint8_t tmp[], const short beta[], const v_uint8& shuf_mask, - const int width) { - constexpr int half_nlanes = (v_uint8::nlanes / 2); - GAPI_DbgAssert(width >= half_nlanes); - - v_int16 b0 = vx_setall_s16(beta[0]); - v_int16 b1 = vx_setall_s16(beta[1]); - v_int16 b2 = vx_setall_s16(beta[2]); - v_int16 b3 = vx_setall_s16(beta[3]); - - v_uint32 permute_idxs1 = v_set_s32(23, 21, 7, 5, 22, 20, 6, 4, 19, 17, 3, 1, 18, 16, 2, 0); - v_uint32 permute_idxs2 = v_set_s32(31, 29, 15, 13, 30, 28, 14, 12, 27, 25, 11, 9, 26, 24, 10, 8); - - for (int w = 0; w < width; ) { - for (; w <= width - half_nlanes; w += half_nlanes) { - v_int16 val0_0 = v_load_ccache_expand(&src0[0][w]); - v_int16 val0_1 = v_load_ccache_expand(&src0[1][w]); - v_int16 val0_2 = v_load_ccache_expand(&src0[2][w]); - v_int16 val0_3 = v_load_ccache_expand(&src0[3][w]); - - v_int16 val1_0 = v_load_ccache_expand(&src1[0][w]); - v_int16 val1_1 = v_load_ccache_expand(&src1[1][w]); - v_int16 val1_2 = v_load_ccache_expand(&src1[2][w]); - v_int16 val1_3 = v_load_ccache_expand(&src1[3][w]); - - v_int16 t0 = v_mulhrs(v_sub_wrap(val0_0, val1_0), b0); - v_int16 t1 = v_mulhrs(v_sub_wrap(val0_1, val1_1), b1); - v_int16 t2 = v_mulhrs(v_sub_wrap(val0_2, val1_2), b2); - v_int16 t3 = v_mulhrs(v_sub_wrap(val0_3, val1_3), b3); - - v_int16 r0 = v_add_wrap(val1_0, t0); - v_int16 r1 = v_add_wrap(val1_1, t1); - v_int16 r2 = v_add_wrap(val1_2, t2); - v_int16 r3 = v_add_wrap(val1_3, t3); - - v_uint8 q0 = v_packus(r0, r1); - v_uint8 q1 = v_packus(r2, r3); - - v_uint8 q2 = v_permutex2_s32(q0, q1, permute_idxs1); - v_uint8 q3 = v_permutex2_s32(q0, q1, permute_idxs2); - - v_uint8 q4 = v_shuffle_s8(q2, shuf_mask); - v_uint8 q5 = v_shuffle_s8(q3, shuf_mask); - - vx_store(&tmp[4 * w + 0], q4); - vx_store(&tmp[4 * w + 2 * half_nlanes], q5); - } - - if (w < width) { - w = width - half_nlanes; - } - } -} - -CV_ALWAYS_INLINE void main_computation_horizontalPass_lpi4(const v_uint8& val_0, - const v_uint8& val_1, - const v_uint8& val_2, - const v_uint8& val_3, - const v_int16& a10, - const v_int16& a32, - const v_int16& a54, - const v_int16& a76, - v_uint8& shuf_mask1, - v_uint8& shuf_mask2, - v_uint32& idxs1, - v_uint32& idxs2, - v_uint8& res1, v_uint8& res2) { - v_int16 val0_0 = v_reinterpret_as_s16(v_expand_low(val_0)); - v_int16 val0_1 = v_reinterpret_as_s16(v_expand_low(val_1)); - v_int16 val0_2 = v_reinterpret_as_s16(v_expand_low(val_2)); - v_int16 val0_3 = v_reinterpret_as_s16(v_expand_low(val_3)); - - v_int16 val1_0 = v_reinterpret_as_s16(v_expand_high(val_0)); - v_int16 val1_1 = v_reinterpret_as_s16(v_expand_high(val_1)); - v_int16 val1_2 = v_reinterpret_as_s16(v_expand_high(val_2)); - v_int16 val1_3 = v_reinterpret_as_s16(v_expand_high(val_3)); - - v_int16 t0 = v_mulhrs(v_sub_wrap(val0_0, val1_0), a10); - v_int16 t1 = v_mulhrs(v_sub_wrap(val0_1, val1_1), a32); - v_int16 t2 = v_mulhrs(v_sub_wrap(val0_2, val1_2), a54); - v_int16 t3 = v_mulhrs(v_sub_wrap(val0_3, val1_3), a76); - - v_int16 r0 = v_add_wrap(val1_0, t0); - v_int16 r1 = v_add_wrap(val1_1, t1); - v_int16 r2 = v_add_wrap(val1_2, t2); - v_int16 r3 = v_add_wrap(val1_3, t3); - - v_uint8 q0 = v_packus(r0, r1); - v_uint8 q1 = v_packus(r2, r3); - - v_uint8 q2 = v_shuffle_s8(q0, shuf_mask1); - v_uint8 q3 = v_shuffle_s8(q1, shuf_mask1); - - v_uint8 q4 = v_permutex2_s32(q2, q3, idxs1); - v_uint8 q5 = v_permutex2_s32(q2, q3, idxs2); - - res1 = v_shuffle_s8(q4, shuf_mask2); - res2 = v_shuffle_s8(q5, shuf_mask2); -} - -CV_ALWAYS_INLINE void verticalPass_anylpi_8U(const uint8_t* src0[], const uint8_t* src1[], - uint8_t tmp[], const int beta0, - const int l, const int length1, const int length2) { - constexpr int half_nlanes = (v_uint8::nlanes / 2); - GAPI_DbgAssert(length1 >= half_nlanes); - - for (int w = 0; w < length2; ) { - for (; w <= length1 - half_nlanes; w += half_nlanes) { - v_int16 s0 = v_reinterpret_as_s16(vx_load_expand(&src0[l][w])); - v_int16 s1 = v_reinterpret_as_s16(vx_load_expand(&src1[l][w])); - v_int16 t = v_mulhrs(s0 - s1, beta0) + s1; - v_pack_u_store(tmp + w, t); - } - - if (w < length1) { - w = length1 - half_nlanes; - } - } -} - -// Resize (bi-linear, 8U, generic number of channels) -template -CV_ALWAYS_INLINE bool calcRowLinear_8UC_Impl(avx512_tag, - std::array, chanNum> &dst, - const uint8_t* src0[], - const uint8_t* src1[], - const short alpha[], - const short clone[], // 4 clones of alpha - const short mapsx[], - const short beta[], - uint8_t tmp[], - const Size& inSz, - const Size& outSz, - const int lpi) { - constexpr int half_nlanes = (v_uint8::nlanes / 2); - if ((inSz.width * chanNum < half_nlanes) || (outSz.width < half_nlanes)) - return false; - - constexpr int shift = (half_nlanes / 4); - - if (4 == lpi) { - GAPI_DbgAssert(inSz.width >= half_nlanes); - - v_uint8 shuf_mask1 = v_setr_s8(0, 4, 8, 12, 1, 5, 9, 13, - 2, 6, 10, 14, 3, 7, 11, 15, - 0, 4, 8, 12, 1, 5, 9, 13, - 2, 6, 10, 14, 3, 7, 11, 15, - 0, 4, 8, 12, 1, 5, 9, 13, - 2, 6, 10, 14, 3, 7, 11, 15, - 0, 4, 8, 12, 1, 5, 9, 13, - 2, 6, 10, 14, 3, 7, 11, 15); - - // vertical pass - verticalPass_lpi4_8U(src0, src1, tmp, beta, - shuf_mask1, inSz.width*chanNum); - - // horizontal pass - v_uint8 val_0, val_1, val_2, val_3, res1, res2; - v_uint8 shuf_mask2 = v_setr_s8(0, 1, 4, 5, 8, 9, 12, 13, - 2, 3, 6, 7, 10, 11, 14, 15, - 0, 1, 4, 5, 8, 9, 12, 13, - 2, 3, 6, 7, 10, 11, 14, 15, - 0, 1, 4, 5, 8, 9, 12, 13, - 2, 3, 6, 7, 10, 11, 14, 15, - 0, 1, 4, 5, 8, 9, 12, 13, - 2, 3, 6, 7, 10, 11, 14, 15); - - v_uint32 idxs3 = v_set_s32(29, 25, 21, 17, 13, 9, 5, 1, 28, 24, 20, 16, 12, 8, 4, 0); - v_uint32 idxs4 = v_set_s32(31, 27, 23, 19, 15, 11, 7, 3, 30, 26, 22, 18, 14, 10, 6, 2); - - GAPI_DbgAssert(outSz.width >= half_nlanes); - for (int x = 0; x < outSz.width; ) { - for (; x <= outSz.width - half_nlanes && x >= 0; x += half_nlanes) { - v_int16 a10 = vx_load(&clone[4 * x]); - v_int16 a32 = vx_load(&clone[4 * (x + 8)]); - v_int16 a54 = vx_load(&clone[4 * (x + 16)]); - v_int16 a76 = vx_load(&clone[4 * (x + 24)]); - - for (int c = 0; c < chanNum; ++c) { - v_gather_channel(val_0, tmp, mapsx, chanNum, c, x, 0); - v_gather_channel(val_1, tmp, mapsx, chanNum, c, x, shift); - v_gather_channel(val_2, tmp, mapsx, chanNum, c, x, shift * 2); - v_gather_channel(val_3, tmp, mapsx, chanNum, c, x, shift * 3); - - main_computation_horizontalPass_lpi4(val_0, val_1, val_2, val_3, - a10, a32, a54, a76, - shuf_mask1, shuf_mask2, - idxs3, idxs4, - res1, res2); - - v_store_low(&dst[c][0][x], res1); - v_store_high(&dst[c][1][x], res1); - v_store_low(&dst[c][2][x], res2); - v_store_high(&dst[c][3][x], res2); - } - } - - if (x < outSz.width) { - x = outSz.width - half_nlanes; - } - } - } else { // if any lpi - for (int l = 0; l < lpi; ++l) { - short beta0 = beta[l]; - - // vertical pass - GAPI_DbgAssert(inSz.width*chanNum >= half_nlanes); - verticalPass_anylpi_8U(src0, src1, tmp, beta0, l, - inSz.width*chanNum, inSz.width*chanNum); - - // horizontal pass - GAPI_DbgAssert(outSz.width >= half_nlanes); - for (int x = 0; x < outSz.width; ) { - for (; x <= outSz.width - half_nlanes && x >= 0; x += half_nlanes) { - for (int c = 0; c < chanNum; ++c) { - v_int16 a0 = vx_load(&alpha[x]); // as signed Q1.1.14 - v_int16 sx = vx_load(&mapsx[x]); // as integer (int16) - v_int16 t0 = v_gather_chan(tmp, sx, c, 0); - v_int16 t1 = v_gather_chan(tmp, sx, c, 1); - v_int16 d = v_mulhrs(t0 - t1, a0) + t1; - v_pack_u_store(&dst[c][l][x], d); - } - } - - if (x < outSz.width) { - x = outSz.width - half_nlanes; - } - } - } - } - return true; -} - -CV_ALWAYS_INLINE void horizontalPass_lpi4_U8C1(const short clone[], const short mapsx[], - uint8_t tmp[], uint8_t* dst[], - v_uint8& shuf_mask1, - const int width) { - constexpr int half_nlanes = (v_uint8::nlanes / 2); - GAPI_DbgAssert(width >= half_nlanes); - - v_uint8 shuf_mask2 = v_setr_s8(0, 1, 4, 5, 8, 9, 12, 13, - 2, 3, 6, 7, 10, 11, 14, 15, - 0, 1, 4, 5, 8, 9, 12, 13, - 2, 3, 6, 7, 10, 11, 14, 15, - 0, 1, 4, 5, 8, 9, 12, 13, - 2, 3, 6, 7, 10, 11, 14, 15, - 0, 1, 4, 5, 8, 9, 12, 13, - 2, 3, 6, 7, 10, 11, 14, 15); - - v_uint32 permute_idxs1 = v_set_s32(15, 13, 11, 9, 7, 5, 3, 1, 14, 12, 10, 8, 6, 4, 2, 0); - v_uint32 permute_idxs2 = v_set_s32(29, 25, 21, 17, 13, 9, 5, 1, 28, 24, 20, 16, 12, 8, 4, 0); - v_uint32 permute_idxs3 = v_set_s32(31, 27, 23, 19, 15, 11, 7, 3, 30, 26, 22, 18, 14, 10, 6, 2); - - v_uint8 val_0, val_1, val_2, val_3, res1, res2; - const int shift = half_nlanes / 4; - - for (int x = 0; x < width; ) { - for (; x <= width - half_nlanes; x += half_nlanes) { - v_int16 a10 = vx_load(&clone[4 * x]); - v_int16 a32 = vx_load(&clone[4 * (x + 8)]); - v_int16 a54 = vx_load(&clone[4 * (x + 16)]); - v_int16 a76 = vx_load(&clone[4 * (x + 24)]); - - v_set(val_0, val_1, val_2, val_3, tmp, mapsx, x, shift); - - val_0 = v_permute32(val_0, permute_idxs1); - val_1 = v_permute32(val_1, permute_idxs1); - val_2 = v_permute32(val_2, permute_idxs1); - val_3 = v_permute32(val_3, permute_idxs1); - - main_computation_horizontalPass_lpi4(val_0, val_1, val_2, val_3, - a10, a32, a54, a76, - shuf_mask1, shuf_mask2, - permute_idxs2, permute_idxs3, - res1, res2); - v_store_low(&dst[0][x], res1); - v_store_high(&dst[1][x], res1); - v_store_low(&dst[2][x], res2); - v_store_high(&dst[3][x], res2); - } - - if (x < width) { - x = width - half_nlanes; - } - } -} - -CV_ALWAYS_INLINE void horizontalPass_anylpi_8U(const short alpha[], const short mapsx[], - uint8_t* dst[], const uchar tmp[], const int l, - const int length) { - constexpr int half_nlanes = (v_uint8::nlanes / 2); - GAPI_DbgAssert(length >= half_nlanes); - - v_int16 t0, t1; - for (int x = 0; x < length; ) { - for (; x <= length - half_nlanes; x += half_nlanes) { - v_int16 a0 = vx_load(&alpha[x]); // as signed Q1.1.14 - v_int16 sx = vx_load(&mapsx[x]); // as integer (int16) - v_uint8 t = v_gather_pairs(tmp, sx); - - v_deinterleave_expand(t, t0, t1); // tmp pixels as int16 - v_int16 d = v_mulhrs(t0 - t1, a0) + t1; - v_pack_u_store(&dst[l][x], d); - } - - if (x < length) { - x = length - half_nlanes; - } - } -} -} // namespace avx512 - -// Resize (bi-linear, 8UC3) -template<> -bool calcRowLinear8UC3C4Impl(avx512_tag, - std::array, 3> &dst, - const uint8_t* src0[], - const uint8_t* src1[], - const short alpha[], - const short clone[], // 4 clones of alpha - const short mapsx[], - const short beta[], - uint8_t tmp[], - const Size& inSz, - const Size& outSz, - const int lpi, - const int ) { - constexpr int chanNum = 3; - return avx512::calcRowLinear_8UC_Impl(avx512_tag{}, dst, src0, src1, alpha, clone, mapsx, beta, tmp, inSz, outSz, lpi); -} - -// Resize (bi-linear, 8UC4) -template<> -bool calcRowLinear8UC3C4Impl(avx512_tag, - std::array, 4> &dst, - const uint8_t* src0[], - const uint8_t* src1[], - const short alpha[], - const short clone[], // 4 clones of alpha - const short mapsx[], - const short beta[], - uint8_t tmp[], - const Size& inSz, - const Size& outSz, - const int lpi, - const int ) { - constexpr int chanNum = 4; - return avx512::calcRowLinear_8UC_Impl(avx512_tag{}, dst, src0, src1, alpha, clone, mapsx, beta, tmp, inSz, outSz, lpi); -} - -// 8UC1 Resize (bi-linear) -template<> -bool calcRowLinear8UC1Impl(avx512_tag, - uint8_t* dst[], - const uint8_t* src0[], - const uint8_t* src1[], - const short alpha[], - const short clone[], // 4 clones of alpha - const short mapsx[], - const short beta[], - uint8_t tmp[], - const Size& inSz, - const Size& outSz, - const int lpi, - const int) { - constexpr int nlanes = v_uint8::nlanes; - constexpr int half_nlanes = (v_uint8::nlanes / 2); - - if (inSz.width < nlanes || outSz.width < half_nlanes) - return false; - - bool xRatioEq = inSz.width == outSz.width; - bool yRatioEq = inSz.height == outSz.height; - - if (!xRatioEq && !yRatioEq) { - if (4 == lpi) { - v_uint8 shuf_mask1 = v_setr_s8(0, 4, 8, 12, 1, 5, 9, 13, - 2, 6, 10, 14, 3, 7, 11, 15, - 0, 4, 8, 12, 1, 5, 9, 13, - 2, 6, 10, 14, 3, 7, 11, 15, - 0, 4, 8, 12, 1, 5, 9, 13, - 2, 6, 10, 14, 3, 7, 11, 15, - 0, 4, 8, 12, 1, 5, 9, 13, - 2, 6, 10, 14, 3, 7, 11, 15); - // vertical pass - avx512::verticalPass_lpi4_8U(src0, src1, tmp, beta, shuf_mask1, inSz.width); - - // horizontal pass - avx512::horizontalPass_lpi4_U8C1(clone, mapsx, tmp, dst, shuf_mask1, - outSz.width); - - } else { // if any lpi - int inLength = inSz.width; - int outLength = outSz.width; - - for (int l = 0; l < lpi; ++l) { - short beta0 = beta[l]; - - // vertical pass - avx512::verticalPass_anylpi_8U(src0, src1, tmp, beta0, l, inLength, inLength); - - // horizontal pass - avx512::horizontalPass_anylpi_8U(alpha, mapsx, dst, tmp, l, outLength); - } - } // if lpi == 4 - - } else if (!xRatioEq) { - GAPI_DbgAssert(yRatioEq); - - if (4 == lpi) { - // vertical pass - GAPI_DbgAssert(inSz.width >= nlanes); - for (int w = 0; w < inSz.width; ) { - for (; w <= inSz.width - nlanes; w += nlanes) { - v_uint8 s0, s1, s2, s3; - s0 = vx_load(&src0[0][w]); - s1 = vx_load(&src0[1][w]); - s2 = vx_load(&src0[2][w]); - s3 = vx_load(&src0[3][w]); - v_store_interleave(&tmp[4 * w], s0, s1, s2, s3); - } - - if (w < inSz.width) { - w = inSz.width - nlanes; - } - } - - // horizontal pass - v_uint8 shuf_mask1 = v_setr_s8(0, 4, 8, 12, 1, 5, 9, 13, - 2, 6, 10, 14, 3, 7, 11, 15, - 0, 4, 8, 12, 1, 5, 9, 13, - 2, 6, 10, 14, 3, 7, 11, 15, - 0, 4, 8, 12, 1, 5, 9, 13, - 2, 6, 10, 14, 3, 7, 11, 15, - 0, 4, 8, 12, 1, 5, 9, 13, - 2, 6, 10, 14, 3, 7, 11, 15); - - avx512::horizontalPass_lpi4_U8C1(clone, mapsx, tmp, dst, shuf_mask1, - outSz.width); - - } else { // any LPI - for (int l = 0; l < lpi; ++l) { - const uchar* src = src0[l]; - - // horizontal pass - avx512::horizontalPass_anylpi_8U(alpha, mapsx, dst, src, l, outSz.width); - } - } - - } else if (!yRatioEq) { - GAPI_DbgAssert(xRatioEq); - int inLength = inSz.width; - int outLength = outSz.width; - - for (int l = 0; l < lpi; ++l) { - short beta0 = beta[l]; - - // vertical pass - avx512::verticalPass_anylpi_8U(src0, src1, dst[l], beta0, l, inLength, outLength); - } - - } else { - GAPI_DbgAssert(xRatioEq && yRatioEq); - int length = inSz.width; - - for (int l = 0; l < lpi; ++l) { - memcpy(dst[l], src0[l], length); - } - } - return true; -} - -template void chanToPlaneRowImpl(avx512_tag, const uint8_t* in, const int chan, const int chs, uint8_t* out, const int length); -template void chanToPlaneRowImpl(avx512_tag, const float* in, const int chan, const int chs, float* out, const int length); - -template void nv12ToRgbRowImpl(avx512_tag, const uint8_t** y_rows, const uint8_t* uv_row, uint8_t** out_rows, const int buf_width); - -template void i420ToRgbRowImpl(avx512_tag, const uint8_t** y_rows, const uint8_t* u_row, - const uint8_t* v_row, uint8_t** out_rows, const int buf_width); - -template void splitRowImpl(avx512_tag, const uint8_t* in, std::array& outs, const int length); -template void splitRowImpl(avx512_tag, const float* in, std::array& outs, const int length); -template void splitRowImpl(avx512_tag, const uint8_t* in, std::array& outs, const int length); -template void splitRowImpl(avx512_tag, const float* in, std::array& outs, const int length); -template void splitRowImpl(avx512_tag, const uint8_t* in, std::array& outs, const int length); -template void splitRowImpl(avx512_tag, const float* in, std::array& outs, const int length); - -template void mergeRowImpl(avx512_tag, const std::array& ins, uint8_t* out, const int length); -template void mergeRowImpl(avx512_tag, const std::array& ins, float* out, const int length); -template void mergeRowImpl(avx512_tag, const std::array& ins, uint8_t* out, const int length); -template void mergeRowImpl(avx512_tag, const std::array& ins, float* out, const int length); -template void mergeRowImpl(avx512_tag, const std::array& ins, uint8_t* out, const int length); -template void mergeRowImpl(avx512_tag, const std::array& ins, float* out, const int length); - -template void calcRowLinear32FC1Impl(avx512_tag, float* dst[], const float* src0[], - const float* src1[], const float alpha[], - const int mapsx[], const float beta[], - const Size& inSz, const Size& outSz, - const int lpi, const int l); - -template void calcRowAreaImpl(avx512_tag, uint8_t dst[], const uint8_t* src[], const Size& inSz, - const Size& outSz, Q0_16 yalpha, const MapperUnit8U &ymap, - int xmaxdf, const short xindex[], const Q0_16 xalpha[], - Q8_8 vbuf[]); - -template void calcRowAreaImpl(avx512_tag, float dst[], const float *src[], const Size& inSz, - const Size& outSz, float yalpha, const MapperUnit32F& ymap, - int xmaxdf, const int xindex[], const float xalpha[], - float vbuf[]); -} // namespace kernels -} // namespace gapi -} // namespace InferenceEngine - diff --git a/src/common/preprocessing/src/cpu_x86_avx512/ie_preprocess_gapi_kernels_avx512.hpp b/src/common/preprocessing/src/cpu_x86_avx512/ie_preprocess_gapi_kernels_avx512.hpp deleted file mode 100644 index ffb32c1df36..00000000000 --- a/src/common/preprocessing/src/cpu_x86_avx512/ie_preprocess_gapi_kernels_avx512.hpp +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright (C) 2018-2023 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#pragma once - -#include "ie_preprocess_gapi_kernels.hpp" -#include "ie_preprocess_gapi_kernels_impl.hpp" -#include - -namespace InferenceEngine { -namespace gapi { -namespace kernels { -namespace avx512 { - -using C3 = std::integral_constant; -using C4 = std::integral_constant; -//----------------------------------------------------------------------------- - -void calcRowArea_8U(uchar dst[], const uchar *src[], const Size &inSz, const Size &outSz, - Q0_16 yalpha, const MapperUnit8U& ymap, int xmaxdf, const short xindex[], - const Q0_16 xalpha[], Q8_8 vbuf[]); - -void calcRowArea_32F(float dst[], const float *src[], const Size &inSz, const Size &outSz, - float yalpha, const MapperUnit32F& ymap, int xmaxdf, const int xindex[], - const float xalpha[], float vbuf[]); - -#if USE_CVKL -void calcRowArea_CVKL_U8(const uchar * src[], - uchar dst[], - const Size & inSz, - const Size & outSz, - int y, - const uint16_t xsi[], - const uint16_t ysi[], - const uint16_t xalpha[], - const uint16_t yalpha[], - int x_max_count, - int y_max_count, - uint16_t vert_sum[]); -#endif -} // namespace avx512 - -template -void chanToPlaneRowImpl(isa_tag_t, const T* in, const int chan, const int chs, T* out, const int length); - -template -void nv12ToRgbRowImpl(isa_tag_t, const uint8_t** y_rows, const uint8_t* uv_row, uint8_t** out_rows, const int buf_width); - -template -void i420ToRgbRowImpl(isa_tag_t, const uint8_t** y_rows, const uint8_t* u_row, - const uint8_t* v_row, uint8_t** out_rows, const int buf_width); - -template -void splitRowImpl(isa_tag_t, const T* in, std::array& outs, const int length); - -template -void mergeRowImpl(isa_tag_t, const std::array& ins, T* out, const int length); - -template -bool calcRowLinear8UC1Impl(isa_tag_t, uint8_t* dst[], const uint8_t* src0[], const uint8_t* src1[], - const short alpha[], const short clone[], const short mapsx[], - const short beta[], uint8_t tmp[], const Size& inSz, - const Size& outSz, const int lpi, const int l); - -template -void calcRowLinear32FC1Impl(isa_tag_t, float* dst[], const float* src0[], const float* src1[], - const float alpha[], const int mapsx[], - const float beta[], const Size& inSz, const Size& outSz, - const int lpi, const int l); - -template -bool calcRowLinear8UC3C4Impl(isa_tag_t, std::array, chs>& dst, - const uint8_t* src0[], const uint8_t* src1[], - const short alpha[], const short clone[], const short mapsx[], - const short beta[], uint8_t tmp[], const Size& inSz, - const Size& outSz, const int lpi, const int l); - -template -void calcRowAreaImpl(isa_tag_t, T dst[], const T* src[], const Size& inSz, - const Size& outSz, A yalpha, const MapperUnit& ymap, - int xmaxdf, const I xindex[], const A xalpha[], W vbuf[]); -} // namespace kernels -} // namespace gapi -} // namespace InferenceEngine diff --git a/src/common/preprocessing/src/cpu_x86_sse42/ie_preprocess_gapi_kernels_sse42.cpp b/src/common/preprocessing/src/cpu_x86_sse42/ie_preprocess_gapi_kernels_sse42.cpp deleted file mode 100644 index 5c8d0ca7812..00000000000 --- a/src/common/preprocessing/src/cpu_x86_sse42/ie_preprocess_gapi_kernels_sse42.cpp +++ /dev/null @@ -1,1298 +0,0 @@ -// Copyright (C) 2018-2023 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#include -#include - -#include "openvino/core/visibility.hpp" -#include "ie_preprocess_gapi_kernels.hpp" -#include "ie_preprocess_gapi_kernels_impl.hpp" -#include "ie_preprocess_gapi_kernels_sse42.hpp" - -// NB: include this before opencv_hal_sse.hpp -#include "nmmintrin.h" - -// NB: define these before opencv_hal_sse.hpp -#ifdef CV_SSE4_2 - #undef CV_SSE4_2 - #undef CV_SSE4_1 - #undef CV_SSSE3 - #undef CV_SSE3 - #undef CV_SSE2 - #undef CV_SSE -#endif -#define CV_SSE4_2 1 -#define CV_SSE4_1 1 -#define CV_SSSE3 1 -#define CV_SSE3 1 -#define CV_SSE2 1 -#define CV_SSE 1 -#define CV_CPU_HAS_SUPPORT_SSE2 1 - -#ifdef CV_SIMD128 -#undef CV_SIMD128 -#endif - -#define CV_SIMD128 1 - -#include "opencv_hal_intrin.hpp" -#include "ie_preprocess_gapi_kernels_simd_impl.hpp" - -#if !CV_SIMD128 -#error CV_SIMD128 is required! -#endif - -#if defined __GNUC__ -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wstrict-overflow" -#endif - -#include - -using namespace cv; - -namespace InferenceEngine { -namespace gapi { -namespace kernels { - -// 8UC1 Resize (bi-linear) -template<> -bool calcRowLinear8UC1Impl(sse42_tag, - uint8_t *dst[], - const uint8_t *src0[], - const uint8_t *src1[], - const short alpha[], - const short clone[], // 4 clones of alpha - const short mapsx[], - const short beta[], - uint8_t tmp[], - const Size& inSz, - const Size& outSz, - const int lpi, - const int) { - constexpr int nlanes = v_uint8::nlanes; - constexpr int half_nlanes = (v_uint8::nlanes / 2); - - if (inSz.width < nlanes || outSz.width < half_nlanes) - return false; - - bool xRatioEq1 = inSz.width == outSz.width; - bool yRatioEq1 = inSz.height == outSz.height; - - if (!xRatioEq1 && !yRatioEq1) { - if (4 == lpi) { - // vertical pass - GAPI_DbgAssert(inSz.width >= 8); - - __m128i b0 = _mm_set1_epi16(beta[0]); - __m128i b1 = _mm_set1_epi16(beta[1]); - __m128i b2 = _mm_set1_epi16(beta[2]); - __m128i b3 = _mm_set1_epi16(beta[3]); - - for (int w = 0; w < inSz.width; ) { - for (; w <= inSz.width - 8; w += 8) { - #if USE_CVKL - //-------------------------------------------- - // reworked from: ie_preprocess_data_sse42.cpp - // function: resize_bilinear_u8 - // label: vertical_pass - //-------------------------------------------- -#ifdef OPENVINO_ARCH_X86 - __m128i val0lo = _mm_castpd_si128(_mm_loadh_pd( - _mm_load_sd(reinterpret_cast(&src0[0][w])), - reinterpret_cast(&src0[1][w]))); - __m128i val0hi = _mm_castpd_si128(_mm_loadh_pd( - _mm_load_sd(reinterpret_cast(&src0[2][w])), - reinterpret_cast(&src0[3][w]))); - __m128i val1lo = _mm_castpd_si128(_mm_loadh_pd( - _mm_load_sd(reinterpret_cast(&src1[0][w])), - reinterpret_cast(&src1[1][w]))); - __m128i val1hi = _mm_castpd_si128(_mm_loadh_pd( - _mm_load_sd(reinterpret_cast(&src1[2][w])), - reinterpret_cast(&src1[3][w]))); -#else - __m128i val0lo = _mm_insert_epi64(_mm_loadl_epi64(reinterpret_cast(&src0[0][w])), - *reinterpret_cast(&src0[1][w]), 1); - __m128i val0hi = _mm_insert_epi64(_mm_loadl_epi64(reinterpret_cast(&src0[2][w])), - *reinterpret_cast(&src0[3][w]), 1); - __m128i val1lo = _mm_insert_epi64(_mm_loadl_epi64(reinterpret_cast(&src1[0][w])), - *reinterpret_cast(&src1[1][w]), 1); - __m128i val1hi = _mm_insert_epi64(_mm_loadl_epi64(reinterpret_cast(&src1[2][w])), - *reinterpret_cast(&src1[3][w]), 1); -#endif - __m128i val0_0 = _mm_cvtepu8_epi16(val0lo); - __m128i val0_2 = _mm_cvtepu8_epi16(val0hi); - __m128i val1_0 = _mm_cvtepu8_epi16(val1lo); - __m128i val1_2 = _mm_cvtepu8_epi16(val1hi); - - __m128i val0_1 = _mm_unpackhi_epi8(val0lo, _mm_setzero_si128()); - __m128i val0_3 = _mm_unpackhi_epi8(val0hi, _mm_setzero_si128()); - __m128i val1_1 = _mm_unpackhi_epi8(val1lo, _mm_setzero_si128()); - __m128i val1_3 = _mm_unpackhi_epi8(val1hi, _mm_setzero_si128()); - - __m128i t0 = _mm_mulhrs_epi16(_mm_sub_epi16(val0_0, val1_0), b0); - __m128i t1 = _mm_mulhrs_epi16(_mm_sub_epi16(val0_1, val1_1), b1); - __m128i t2 = _mm_mulhrs_epi16(_mm_sub_epi16(val0_2, val1_2), b2); - __m128i t3 = _mm_mulhrs_epi16(_mm_sub_epi16(val0_3, val1_3), b3); - - __m128i r0 = _mm_add_epi16(val1_0, t0); - __m128i r1 = _mm_add_epi16(val1_1, t1); - __m128i r2 = _mm_add_epi16(val1_2, t2); - __m128i r3 = _mm_add_epi16(val1_3, t3); - - __m128i q0 = _mm_packus_epi16(r0, r1); - __m128i q1 = _mm_packus_epi16(r2, r3); - - __m128i q2 = _mm_blend_epi16(q0, _mm_slli_si128(q1, 4), 0xCC /*0b11001100*/); - __m128i q3 = _mm_blend_epi16(_mm_srli_si128(q0, 4), q1, 0xCC /*0b11001100*/); - - __m128i q4 = _mm_shuffle_epi8(q2, _mm_setr_epi8(0, 8, 4, 12, 1, 9, 5, 13, 2, 10, 6, 14, 3, 11, 7, 15)); - __m128i q5 = _mm_shuffle_epi8(q3, _mm_setr_epi8(0, 8, 4, 12, 1, 9, 5, 13, 2, 10, 6, 14, 3, 11, 7, 15)); - - _mm_storeu_si128(reinterpret_cast<__m128i *>(&tmp[4*w + 0]), q4); - _mm_storeu_si128(reinterpret_cast<__m128i *>(&tmp[4*w + 16]), q5); - - #else - // let: t[i] = src0[i][w]*beta0[i] + src1[i][w]*beta1 - // here: beta0[i] = beta[i], beta1 = 1 - beta0[i] - v_int16x8 t0, t1, t2, t3; - { - v_int16x8 s0, s1; - - s0 = v_reinterpret_as_s16(v_load_expand(&src0[0][w])); - s1 = v_reinterpret_as_s16(v_load_expand(&src1[0][w])); - t0 = v_mulhrs(s0 - s1, beta[0]) + s1; - - s0 = v_reinterpret_as_s16(v_load_expand(&src0[1][w])); - s1 = v_reinterpret_as_s16(v_load_expand(&src1[1][w])); - t1 = v_mulhrs(s0 - s1, beta[1]) + s1; - - s0 = v_reinterpret_as_s16(v_load_expand(&src0[2][w])); - s1 = v_reinterpret_as_s16(v_load_expand(&src1[2][w])); - t2 = v_mulhrs(s0 - s1, beta[2]) + s1; - - s0 = v_reinterpret_as_s16(v_load_expand(&src0[3][w])); - s1 = v_reinterpret_as_s16(v_load_expand(&src1[3][w])); - t3 = v_mulhrs(s0 - s1, beta[3]) + s1; - } - // store as groups of 4 pixels: each group to have a pixel per row - { - v_uint8x16 a0, a1, a2, a3; - a0 = v_pack_u(t0, v_setall_s16(0)); - a1 = v_pack_u(t1, v_setall_s16(0)); - a2 = v_pack_u(t2, v_setall_s16(0)); - a3 = v_pack_u(t3, v_setall_s16(0)); - - v_int16x8 b0, b1; - b0 = v_reinterpret_as_s16(v_interleave_low(a0, a1)); // 0th, 1st - b1 = v_reinterpret_as_s16(v_interleave_low(a2, a3)); // 2nd, 3rd - - v_uint8x16 d0, d1; - d0 = v_reinterpret_as_u8(v_interleave_low(b0, b1)); - d1 = v_reinterpret_as_u8(v_interleave_high(b0, b1)); - - v_store(&tmp[4*w + 0], d0); - v_store(&tmp[4*w + 16], d1); - } - #endif - } - - if (w < inSz.width) { - w = inSz.width - 8; - } - } - - // horizontal pass - GAPI_DbgAssert(outSz.width >= 8); - for (int x = 0; x < outSz.width; ) { - for (; x <= outSz.width - 8; x += 8) { - #if USE_CVKL - //-------------------------------------------- - // reworked from: ie_preprocess_data_sse42.cpp - // function: resize_bilinear_u8 - // label: horizontal_pass - //-------------------------------------------- - - #if 1 - __m128i a10 = _mm_loadu_si128(reinterpret_cast(&clone[4 * x])); - __m128i a32 = _mm_loadu_si128(reinterpret_cast(&clone[4 * (x + 2)])); - __m128i a54 = _mm_loadu_si128(reinterpret_cast(&clone[4 * (x + 4)])); - __m128i a76 = _mm_loadu_si128(reinterpret_cast(&clone[4 * (x + 6)])); - #else - // provided alpha[x..x+7] = { a0, a1, a2, a3, a4, a5, a6, a7}, - // clone each a[i] 4 times - one item per each of LPI rows, - // so that a10 = {a0, a0, a0, a0, a1, a1, a1, a1}, etc. - __m128i a10, a32, a54, a76; - __m128i alpha0 = _mm_loadu_si128(reinterpret_cast(&alpha[x])); - a10 = _mm_unpacklo_epi16(alpha0, alpha0); // {a0, a0, a1, a1, a2, a2, a3, a3} - a32 = _mm_unpackhi_epi16(a10, a10); // {a2, a2, a2, a2, a3, a3, a3, a3} - a10 = _mm_unpacklo_epi16(a10, a10); // {a0, a0, a0, a0, a1, a1, a1, a1} - a54 = _mm_unpackhi_epi16(alpha0, alpha0); // {a4, a4, a5, a5, a6, a6, a7, a7} - a76 = _mm_unpackhi_epi16(a54, a54); // {a6, a6, a6, a6, a7, a7, a7, a7} - a54 = _mm_unpacklo_epi16(a54, a54); // {a4, a4, a4, a4, a5, a5, a5, a5} - #endif - - __m128d val0d, val1d, val2d, val3d; - val0d = _mm_load_sd(/****/ reinterpret_cast(&tmp[4 * mapsx[x + 0]])); - val0d = _mm_loadh_pd(val0d, reinterpret_cast(&tmp[4 * mapsx[x + 1]])); - val1d = _mm_load_sd(/****/ reinterpret_cast(&tmp[4 * mapsx[x + 2]])); - val1d = _mm_loadh_pd(val1d, reinterpret_cast(&tmp[4 * mapsx[x + 3]])); - val2d = _mm_load_sd(/****/ reinterpret_cast(&tmp[4 * mapsx[x + 4]])); - val2d = _mm_loadh_pd(val2d, reinterpret_cast(&tmp[4 * mapsx[x + 5]])); - val3d = _mm_load_sd(/****/ reinterpret_cast(&tmp[4 * mapsx[x + 6]])); - val3d = _mm_loadh_pd(val3d, reinterpret_cast(&tmp[4 * mapsx[x + 7]])); - - __m128i val_0 = _mm_castpd_si128(val0d); - __m128i val_1 = _mm_castpd_si128(val1d); - __m128i val_2 = _mm_castpd_si128(val2d); - __m128i val_3 = _mm_castpd_si128(val3d); - - val_0 = _mm_shuffle_epi32(val_0, _MM_SHUFFLE(3, 1, 2, 0)); - val_1 = _mm_shuffle_epi32(val_1, _MM_SHUFFLE(3, 1, 2, 0)); - val_2 = _mm_shuffle_epi32(val_2, _MM_SHUFFLE(3, 1, 2, 0)); - val_3 = _mm_shuffle_epi32(val_3, _MM_SHUFFLE(3, 1, 2, 0)); - - __m128i val0_0 = _mm_cvtepu8_epi16(val_0); - __m128i val0_1 = _mm_cvtepu8_epi16(val_1); - __m128i val0_2 = _mm_cvtepu8_epi16(val_2); - __m128i val0_3 = _mm_cvtepu8_epi16(val_3); - - __m128i val1_0 = _mm_unpackhi_epi8(val_0, _mm_setzero_si128()); - __m128i val1_1 = _mm_unpackhi_epi8(val_1, _mm_setzero_si128()); - __m128i val1_2 = _mm_unpackhi_epi8(val_2, _mm_setzero_si128()); - __m128i val1_3 = _mm_unpackhi_epi8(val_3, _mm_setzero_si128()); - - __m128i t0 = _mm_mulhrs_epi16(_mm_sub_epi16(val0_0, val1_0), a10); - __m128i t1 = _mm_mulhrs_epi16(_mm_sub_epi16(val0_1, val1_1), a32); - __m128i t2 = _mm_mulhrs_epi16(_mm_sub_epi16(val0_2, val1_2), a54); - __m128i t3 = _mm_mulhrs_epi16(_mm_sub_epi16(val0_3, val1_3), a76); - - __m128i r0 = _mm_add_epi16(val1_0, t0); - __m128i r1 = _mm_add_epi16(val1_1, t1); - __m128i r2 = _mm_add_epi16(val1_2, t2); - __m128i r3 = _mm_add_epi16(val1_3, t3); - - __m128i q0 = _mm_packus_epi16(r0, r1); - __m128i q1 = _mm_packus_epi16(r2, r3); - - __m128i q2 = _mm_shuffle_epi8(q0, _mm_setr_epi8(0, 4, 8, 12, 2, 6, 10, 14, 1, 5, 9, 13, 3, 7, 11, 15)); - __m128i q3 = _mm_shuffle_epi8(q1, _mm_setr_epi8(0, 4, 8, 12, 2, 6, 10, 14, 1, 5, 9, 13, 3, 7, 11, 15)); - - __m128i q4 = _mm_blend_epi16(q2, _mm_slli_si128(q3, 4), 0xCC /*0b11001100*/); - __m128i q5 = _mm_blend_epi16(_mm_srli_si128(q2, 4), q3, 0xCC /*0b11001100*/); - - _mm_storel_epi64(reinterpret_cast<__m128i*>(&dst[0][x]), q4); - _mm_storel_epi64(reinterpret_cast<__m128i*>(&dst[1][x]), _mm_srli_si128(q4, 8)); - _mm_storel_epi64(reinterpret_cast<__m128i*>(&dst[2][x]), q5); - _mm_storel_epi64(reinterpret_cast<__m128i*>(&dst[3][x]), _mm_srli_si128(q5, 8)); - - #else - // let: t be 2 pairs of groups of 4 pixels (each group is for 4 dst rows) - // each pair of gorups corresponds to pixels indexed as sx0 and sx1=sx0+1 - // so: low part of t0 is 2x4 pixels corresponding to sx0=mapsx[x+0], etc. - v_uint8x16 t0, t1, t2, t3; - { - t0.val = _mm_insert_epi64(_mm_loadl_epi64(reinterpret_cast<__m128i*>(&tmp[4 * mapsx[x + 0]])), - *reinterpret_cast(&tmp[4 * mapsx[x + 1]]), 1); - t1.val = _mm_insert_epi64(_mm_loadl_epi64(reinterpret_cast<__m128i*>(&tmp[4 * mapsx[x + 2]])), - *reinterpret_cast(&tmp[4 * mapsx[x + 3]]), 1); - t2.val = _mm_insert_epi64(_mm_loadl_epi64(reinterpret_cast<__m128i*>(&tmp[4 * mapsx[x + 4]])), - *reinterpret_cast(&tmp[4 * mapsx[x + 5]]), 1); - t3.val = _mm_insert_epi64(_mm_loadl_epi64(reinterpret_cast<__m128i*>(&tmp[4 * mapsx[x + 6]])), - *reinterpret_cast(&tmp[4 * mapsx[x + 7]]), 1); - } - - // let: r0 be pixels for 0th row, etc - v_uint8x16 r0, r1, r2, r3; - v_deinterleave(t0, t1, t2, t3, r0, r1, r2, r3); - - // let: dl be resulting 8 pixels for l'th row - // dl = alpha0*s0l + alpha1*s1l - // note that alpha0 + alpha1 = 1 - { - v_int16x8 s0, s1, d, alpha0; - - alpha0 = v_load(&alpha[x]); // 8 coefficients - - v_deinterleave_expand(r0, s0, s1); - d = v_mulhrs(s0 - s1, alpha0) + s1; - v_pack_u_store(&dst[0][x], d); - - v_deinterleave_expand(r1, s0, s1); - d = v_mulhrs(s0 - s1, alpha0) + s1; - v_pack_u_store(&dst[1][x], d); - - v_deinterleave_expand(r2, s0, s1); - d = v_mulhrs(s0 - s1, alpha0) + s1; - v_pack_u_store(&dst[2][x], d); - - v_deinterleave_expand(r3, s0, s1); - d = v_mulhrs(s0 - s1, alpha0) + s1; - v_pack_u_store(&dst[3][x], d); - } - #endif - } - - if (x < outSz.width) { - x = outSz.width - 8; - } - } - - } else { // if any lpi - for (int l = 0; l < lpi; l++) { - short beta0 = beta[l]; - // short beta1 = saturate_cast(ONE - beta[l]); - - // vertical pass - GAPI_DbgAssert(inSz.width >= 8); - for (int w = 0; w < inSz.width; ) { - for (; w <= inSz.width - 8; w += 8) { - v_int16x8 s0 = v_reinterpret_as_s16(v_load_expand(&src0[l][w])); - v_int16x8 s1 = v_reinterpret_as_s16(v_load_expand(&src1[l][w])); - v_int16x8 t = v_mulhrs(s0 - s1, beta0) + s1; - v_pack_u_store(tmp + w, t); - } - - if (w < inSz.width) { - w = inSz.width - 8; - } - } - - // horizontal pass - GAPI_DbgAssert(outSz.width >= 8); - for (int x = 0; x < outSz.width; ) { - for (; x <= outSz.width - 8; x += 8) { - v_int16x8 a0 = v_load(&alpha[x]); // as signed Q1.1.14 - v_int16x8 sx = v_load(&mapsx[x]); // as integer (int16) - v_uint8x16 t = v_gather_pairs(tmp, sx); // 8 pairs of src0 pixels - v_int16x8 t0, t1; - v_deinterleave_expand(t, t0, t1); // tmp pixels as int16 - v_int16x8 d = v_mulhrs(t0 - t1, a0) + t1; - v_pack_u_store(&dst[l][x], d); - } - - if (x < outSz.width) { - x = outSz.width - 8; - } - } - } - } // if lpi == 4 - - } else if (!xRatioEq1) { - GAPI_DbgAssert(yRatioEq1); - - if (4 == lpi) { - // vertical pass - GAPI_DbgAssert(inSz.width >= 16); - for (int w = 0; w < inSz.width; ) { - for (; w <= inSz.width - 16; w += 16) { - v_uint8x16 s0, s1, s2, s3; - s0 = v_load(&src0[0][w]); - s1 = v_load(&src0[1][w]); - s2 = v_load(&src0[2][w]); - s3 = v_load(&src0[3][w]); - v_store_interleave(&tmp[4*w], s0, s1, s2, s3); - } - - if (w < inSz.width) { - w = inSz.width - 16; - } - } - - // horizontal pass - GAPI_DbgAssert(outSz.width >= 8); - for (int x = 0; x < outSz.width; ) { - for (; x <= outSz.width - 8; x += 8) { - v_uint8x16 t0, t1, t2, t3; -#ifdef OPENVINO_ARCH_X86 - t0.val = _mm_castpd_si128(_mm_loadh_pd( - _mm_load_sd(reinterpret_cast(&tmp[4 * mapsx[x + 0]])), - reinterpret_cast(&tmp[4 * mapsx[x + 1]]))); - t1.val = _mm_castpd_si128(_mm_loadh_pd( - _mm_load_sd(reinterpret_cast(&tmp[4 * mapsx[x + 2]])), - reinterpret_cast(&tmp[4 * mapsx[x + 3]]))); - t2.val = _mm_castpd_si128(_mm_loadh_pd( - _mm_load_sd(reinterpret_cast(&tmp[4 * mapsx[x + 4]])), - reinterpret_cast(&tmp[4 * mapsx[x + 5]]))); - t3.val = _mm_castpd_si128(_mm_loadh_pd( - _mm_load_sd(reinterpret_cast(&tmp[4 * mapsx[x + 6]])), - reinterpret_cast(&tmp[4 * mapsx[x + 7]]))); -#else - t0.val = _mm_insert_epi64(_mm_loadl_epi64(reinterpret_cast<__m128i*>(&tmp[4 * mapsx[x + 0]])), - *reinterpret_cast(&tmp[4 * mapsx[x + 1]]), 1); - t1.val = _mm_insert_epi64(_mm_loadl_epi64(reinterpret_cast<__m128i*>(&tmp[4 * mapsx[x + 2]])), - *reinterpret_cast(&tmp[4 * mapsx[x + 3]]), 1); - t2.val = _mm_insert_epi64(_mm_loadl_epi64(reinterpret_cast<__m128i*>(&tmp[4 * mapsx[x + 4]])), - *reinterpret_cast(&tmp[4 * mapsx[x + 5]]), 1); - t3.val = _mm_insert_epi64(_mm_loadl_epi64(reinterpret_cast<__m128i*>(&tmp[4 * mapsx[x + 6]])), - *reinterpret_cast(&tmp[4 * mapsx[x + 7]]), 1); -#endif - v_uint8x16 r0, r1, r2, r3; - v_deinterleave(t0, t1, t2, t3, r0, r1, r2, r3); - - v_int16x8 s0, s1, d, alpha0; - - alpha0 = v_load(&alpha[x]); // 8 coefficients - - v_deinterleave_expand(r0, s0, s1); - d = v_mulhrs(s0 - s1, alpha0) + s1; - v_pack_u_store(&dst[0][x], d); - - v_deinterleave_expand(r1, s0, s1); - d = v_mulhrs(s0 - s1, alpha0) + s1; - v_pack_u_store(&dst[1][x], d); - - v_deinterleave_expand(r2, s0, s1); - d = v_mulhrs(s0 - s1, alpha0) + s1; - v_pack_u_store(&dst[2][x], d); - - v_deinterleave_expand(r3, s0, s1); - d = v_mulhrs(s0 - s1, alpha0) + s1; - v_pack_u_store(&dst[3][x], d); - } - - if (x < outSz.width) { - x = outSz.width - 8; - } - } - - } else { // any LPI - for (int l = 0; l < lpi; l++) { - const uchar *src = src0[l]; - - // horizontal pass - GAPI_DbgAssert(outSz.width >= 8); - for (int x = 0; x < outSz.width; ) { - for (; x <= outSz.width - 8; x += 8) { - v_int16x8 a0 = v_load(&alpha[x]); // as signed Q1.1.14 - v_int16x8 sx = v_load(&mapsx[x]); // as integer (int16) - v_uint8x16 t = v_gather_pairs(src, sx); // 8 pairs of src0 pixels - v_int16x8 t0, t1; - v_deinterleave_expand(t, t0, t1); // tmp pixels as int16 - v_int16x8 d = v_mulhrs(t0 - t1, a0) + t1; - v_pack_u_store(&dst[l][x], d); - } - - if (x < outSz.width) { - x = outSz.width - 8; - } - } - } - } - - } else if (!yRatioEq1) { - GAPI_DbgAssert(xRatioEq1); - int length = inSz.width; // == outSz.width - - for (int l = 0; l < lpi; l++) { - short beta0 = beta[l]; - // short beta1 = saturate_cast(ONE - beta[l]); - - // vertical pass - GAPI_DbgAssert(inSz.width >= 8); - for (int w = 0; w < outSz.width; ) { - for (; w <= length - 8; w += 8) { - v_int16x8 s0 = v_reinterpret_as_s16(v_load_expand(src0[l] + w)); - v_int16x8 s1 = v_reinterpret_as_s16(v_load_expand(src1[l] + w)); - v_int16x8 t = v_mulhrs(s0 - s1, beta0) + s1; - v_pack_u_store(dst[l] + w, t); - } - - if (w < inSz.width) { - w = inSz.width - 8; - } - } - } - - } else { - GAPI_DbgAssert(xRatioEq1 && yRatioEq1); - int length = inSz.width; // == outSz.width - - for (int l = 0; l < lpi; l++) { - memcpy(dst[l], src0[l], length); - } - } - return true; -} - -// Resize 3C/4C universal intrinsic implementation for SSE42 version is a bit slower than original sometimes. -// Remove original implementation when I find a cause. -#if 1 -// Resize (bi-linear, 8U, generic number of channels) -template -CV_ALWAYS_INLINE bool calcRowLinear_8UC_Impl_(sse42_tag, - std::array, chanNum> &dst, - const uint8_t* src0[], - const uint8_t* src1[], - const short alpha[], - const short clone[], // 4 clones of alpha - const short mapsx[], - const short beta[], - uint8_t tmp[], - const Size& inSz, - const Size& outSz, - const int lpi) { - const int half_nlanes = (v_uint8::nlanes / 2); - if ((inSz.width < half_nlanes) || (outSz.width < half_nlanes)) - return false; - - if (4 == lpi) { - // vertical pass - GAPI_DbgAssert(inSz.width >= half_nlanes); - - __m128i b0 = _mm_set1_epi16(beta[0]); - __m128i b1 = _mm_set1_epi16(beta[1]); - __m128i b2 = _mm_set1_epi16(beta[2]); - __m128i b3 = _mm_set1_epi16(beta[3]); - - for (int w = 0; w < inSz.width*chanNum; ) { - for (; w <= inSz.width*chanNum - half_nlanes && w >= 0; w += half_nlanes) { - //-------------------------------------------- - // reworked from: ie_preprocess_data_sse42.cpp - // function: resize_bilinear_u8 - // label: vertical_pass - //-------------------------------------------- -#ifdef OPENVINO_ARCH_X86 - __m128i val0lo = _mm_castpd_si128(_mm_loadh_pd( - _mm_load_sd(reinterpret_cast(&src0[0][w])), - reinterpret_cast(&src0[1][w]))); - __m128i val0hi = _mm_castpd_si128(_mm_loadh_pd( - _mm_load_sd(reinterpret_cast(&src0[2][w])), - reinterpret_cast(&src0[3][w]))); - __m128i val1lo = _mm_castpd_si128(_mm_loadh_pd( - _mm_load_sd(reinterpret_cast(&src1[0][w])), - reinterpret_cast(&src1[1][w]))); - __m128i val1hi = _mm_castpd_si128(_mm_loadh_pd( - _mm_load_sd(reinterpret_cast(&src1[2][w])), - reinterpret_cast(&src1[3][w]))); -#else - __m128i val0lo = _mm_insert_epi64(_mm_loadl_epi64(reinterpret_cast(&src0[0][w])), - *reinterpret_cast(&src0[1][w]), 1); - __m128i val0hi = _mm_insert_epi64(_mm_loadl_epi64(reinterpret_cast(&src0[2][w])), - *reinterpret_cast(&src0[3][w]), 1); - __m128i val1lo = _mm_insert_epi64(_mm_loadl_epi64(reinterpret_cast(&src1[0][w])), - *reinterpret_cast(&src1[1][w]), 1); - __m128i val1hi = _mm_insert_epi64(_mm_loadl_epi64(reinterpret_cast(&src1[2][w])), - *reinterpret_cast(&src1[3][w]), 1); -#endif - __m128i val0_0 = _mm_cvtepu8_epi16(val0lo); - __m128i val0_2 = _mm_cvtepu8_epi16(val0hi); - __m128i val1_0 = _mm_cvtepu8_epi16(val1lo); - __m128i val1_2 = _mm_cvtepu8_epi16(val1hi); - - __m128i val0_1 = _mm_unpackhi_epi8(val0lo, _mm_setzero_si128()); - __m128i val0_3 = _mm_unpackhi_epi8(val0hi, _mm_setzero_si128()); - __m128i val1_1 = _mm_unpackhi_epi8(val1lo, _mm_setzero_si128()); - __m128i val1_3 = _mm_unpackhi_epi8(val1hi, _mm_setzero_si128()); - - __m128i t0 = _mm_mulhrs_epi16(_mm_sub_epi16(val0_0, val1_0), b0); - __m128i t1 = _mm_mulhrs_epi16(_mm_sub_epi16(val0_1, val1_1), b1); - __m128i t2 = _mm_mulhrs_epi16(_mm_sub_epi16(val0_2, val1_2), b2); - __m128i t3 = _mm_mulhrs_epi16(_mm_sub_epi16(val0_3, val1_3), b3); - - __m128i r0 = _mm_add_epi16(val1_0, t0); - __m128i r1 = _mm_add_epi16(val1_1, t1); - __m128i r2 = _mm_add_epi16(val1_2, t2); - __m128i r3 = _mm_add_epi16(val1_3, t3); - - __m128i q0 = _mm_packus_epi16(r0, r1); - __m128i q1 = _mm_packus_epi16(r2, r3); - - __m128i q2 = _mm_blend_epi16(q0, _mm_slli_si128(q1, 4), 0xCC /*0b11001100*/); - __m128i q3 = _mm_blend_epi16(_mm_srli_si128(q0, 4), q1, 0xCC /*0b11001100*/); - - __m128i q4 = _mm_shuffle_epi8(q2, _mm_setr_epi8(0, 8, 4, 12, 1, 9, 5, 13, 2, 10, 6, 14, 3, 11, 7, 15)); - __m128i q5 = _mm_shuffle_epi8(q3, _mm_setr_epi8(0, 8, 4, 12, 1, 9, 5, 13, 2, 10, 6, 14, 3, 11, 7, 15)); - - _mm_storeu_si128(reinterpret_cast<__m128i *>(&tmp[4*w + 0]), q4); - _mm_storeu_si128(reinterpret_cast<__m128i *>(&tmp[4*w + 16]), q5); - } - - if (w < inSz.width*chanNum) { - w = inSz.width*chanNum - half_nlanes; - } - } - - // horizontal pass - GAPI_DbgAssert(outSz.width >= half_nlanes); - for (int x = 0; x < outSz.width; ) { - for (; x <= outSz.width - half_nlanes && x >= 0; x += half_nlanes) { - //-------------------------------------------- - // reworked from: ie_preprocess_data_sse42.cpp - // function: resize_bilinear_u8 - // label: horizontal_pass - //-------------------------------------------- - - __m128i a10 = _mm_loadu_si128(reinterpret_cast(&clone[4 * x])); - __m128i a32 = _mm_loadu_si128(reinterpret_cast(&clone[4 * (x + 2)])); - __m128i a54 = _mm_loadu_si128(reinterpret_cast(&clone[4 * (x + 4)])); - __m128i a76 = _mm_loadu_si128(reinterpret_cast(&clone[4 * (x + 6)])); - - __m128i val_0 = _mm_setzero_si128(); - __m128i val_1 = _mm_setzero_si128(); - __m128i val_2 = _mm_setzero_si128(); - __m128i val_3 = _mm_setzero_si128(); - - for (int c = 0; c < chanNum; c++) { - val_0 = _mm_insert_epi32(val_0, *reinterpret_cast(&tmp[4 * (chanNum * mapsx[x + 0] + c)]), 0); - val_0 = _mm_insert_epi32(val_0, *reinterpret_cast(&tmp[4 * (chanNum * (mapsx[x + 0] + 1) + c)]), 1); - val_0 = _mm_insert_epi32(val_0, *reinterpret_cast(&tmp[4 * (chanNum * mapsx[x + 1] + c)]), 2); - val_0 = _mm_insert_epi32(val_0, *reinterpret_cast(&tmp[4 * (chanNum * (mapsx[x + 1] + 1) + c)]), 3); - - val_1 = _mm_insert_epi32(val_1, *reinterpret_cast(&tmp[4 * (chanNum * mapsx[x + 2] + c)]), 0); - val_1 = _mm_insert_epi32(val_1, *reinterpret_cast(&tmp[4 * (chanNum * (mapsx[x + 2] + 1) + c)]), 1); - val_1 = _mm_insert_epi32(val_1, *reinterpret_cast(&tmp[4 * (chanNum * mapsx[x + 3] + c)]), 2); - val_1 = _mm_insert_epi32(val_1, *reinterpret_cast(&tmp[4 * (chanNum * (mapsx[x + 3] + 1) + c)]), 3); - - val_2 = _mm_insert_epi32(val_2, *reinterpret_cast(&tmp[4 * (chanNum * mapsx[x + 4] + c)]), 0); - val_2 = _mm_insert_epi32(val_2, *reinterpret_cast(&tmp[4 * (chanNum * (mapsx[x + 4] + 1) + c)]), 1); - val_2 = _mm_insert_epi32(val_2, *reinterpret_cast(&tmp[4 * (chanNum * mapsx[x + 5] + c)]), 2); - val_2 = _mm_insert_epi32(val_2, *reinterpret_cast(&tmp[4 * (chanNum * (mapsx[x + 5] + 1) + c)]), 3); - - val_3 = _mm_insert_epi32(val_3, *reinterpret_cast(&tmp[4 * (chanNum * mapsx[x + 6] + c)]), 0); - val_3 = _mm_insert_epi32(val_3, *reinterpret_cast(&tmp[4 * (chanNum * (mapsx[x + 6] + 1) + c)]), 1); - val_3 = _mm_insert_epi32(val_3, *reinterpret_cast(&tmp[4 * (chanNum * mapsx[x + 7] + c)]), 2); - val_3 = _mm_insert_epi32(val_3, *reinterpret_cast(&tmp[4 * (chanNum * (mapsx[x + 7] + 1) + c)]), 3); - - val_0 = _mm_shuffle_epi32(val_0, _MM_SHUFFLE(3, 1, 2, 0)); - val_1 = _mm_shuffle_epi32(val_1, _MM_SHUFFLE(3, 1, 2, 0)); - val_2 = _mm_shuffle_epi32(val_2, _MM_SHUFFLE(3, 1, 2, 0)); - val_3 = _mm_shuffle_epi32(val_3, _MM_SHUFFLE(3, 1, 2, 0)); - - __m128i val0_0 = _mm_cvtepu8_epi16(val_0); - __m128i val0_1 = _mm_cvtepu8_epi16(val_1); - __m128i val0_2 = _mm_cvtepu8_epi16(val_2); - __m128i val0_3 = _mm_cvtepu8_epi16(val_3); - - __m128i val1_0 = _mm_unpackhi_epi8(val_0, _mm_setzero_si128()); - __m128i val1_1 = _mm_unpackhi_epi8(val_1, _mm_setzero_si128()); - __m128i val1_2 = _mm_unpackhi_epi8(val_2, _mm_setzero_si128()); - __m128i val1_3 = _mm_unpackhi_epi8(val_3, _mm_setzero_si128()); - - __m128i t0 = _mm_mulhrs_epi16(_mm_sub_epi16(val0_0, val1_0), a10); - __m128i t1 = _mm_mulhrs_epi16(_mm_sub_epi16(val0_1, val1_1), a32); - __m128i t2 = _mm_mulhrs_epi16(_mm_sub_epi16(val0_2, val1_2), a54); - __m128i t3 = _mm_mulhrs_epi16(_mm_sub_epi16(val0_3, val1_3), a76); - - __m128i r0 = _mm_add_epi16(val1_0, t0); - __m128i r1 = _mm_add_epi16(val1_1, t1); - __m128i r2 = _mm_add_epi16(val1_2, t2); - __m128i r3 = _mm_add_epi16(val1_3, t3); - - __m128i q0 = _mm_packus_epi16(r0, r1); - __m128i q1 = _mm_packus_epi16(r2, r3); - - __m128i q2 = _mm_shuffle_epi8(q0, _mm_setr_epi8(0, 4, 8, 12, 2, 6, 10, 14, 1, 5, 9, 13, 3, 7, 11, 15)); - __m128i q3 = _mm_shuffle_epi8(q1, _mm_setr_epi8(0, 4, 8, 12, 2, 6, 10, 14, 1, 5, 9, 13, 3, 7, 11, 15)); - - __m128i q4 = _mm_blend_epi16(q2, _mm_slli_si128(q3, 4), 0xCC /*0b11001100*/); - __m128i q5 = _mm_blend_epi16(_mm_srli_si128(q2, 4), q3, 0xCC /*0b11001100*/); - - _mm_storel_epi64(reinterpret_cast<__m128i*>(&dst[c][0][x]), q4); - _mm_storel_epi64(reinterpret_cast<__m128i*>(&dst[c][1][x]), _mm_srli_si128(q4, 8)); - _mm_storel_epi64(reinterpret_cast<__m128i*>(&dst[c][2][x]), q5); - _mm_storel_epi64(reinterpret_cast<__m128i*>(&dst[c][3][x]), _mm_srli_si128(q5, 8)); - } - } - - if (x < outSz.width) { - x = outSz.width - half_nlanes; - } - } - - } else { // if any lpi - for (int l = 0; l < lpi; l++) { - short beta0 = beta[l]; - - // vertical pass - GAPI_DbgAssert(inSz.width*chanNum >= half_nlanes); - for (int w = 0; w < inSz.width*chanNum; ) { - for (; w <= inSz.width*chanNum - half_nlanes; w += half_nlanes) { - v_int16 s0 = v_reinterpret_as_s16(vx_load_expand(&src0[l][w])); - v_int16 s1 = v_reinterpret_as_s16(vx_load_expand(&src1[l][w])); - v_int16 t = v_mulhrs(s0 - s1, beta0) + s1; - v_pack_u_store(tmp + w, t); - } - - if (w < inSz.width*chanNum) { - w = inSz.width*chanNum - half_nlanes; - } - } - - // horizontal pass - GAPI_DbgAssert(outSz.width >= half_nlanes); - for (int x = 0; x < outSz.width; ) { - for (; x <= outSz.width - half_nlanes && x >= 0; x += half_nlanes) { - for (int c = 0; c < chanNum; c++) { - v_int16 a0 = vx_load(&alpha[x]); // as signed Q1.1.14 - v_int16 sx = vx_load(&mapsx[x]); // as integer (int16) - v_int16 t0 = v_gather_chan(tmp, sx, c, 0); - v_int16 t1 = v_gather_chan(tmp, sx, c, 1); - v_int16 d = v_mulhrs(t0 - t1, a0) + t1; - v_pack_u_store(&dst[c][l][x], d); - } - } - - if (x < outSz.width) { - x = outSz.width - half_nlanes; - } - } - } - } - return true; -} -#else -// Resize 3C/4C universal intrinsic implementation for SSE42 version is a bit slower sometimes. -// Gonna turn it on when I find a cause. -template -void calcRowLinear_8UC_Impl_(std::array, chanNum> &dst, - const uint8_t *src0[], - const uint8_t *src1[], - const short alpha[], - const short clone[], // 4 clones of alpha - const short mapsx[], - const short beta[], - uint8_t tmp[], - const Size &inSz, - const Size &outSz, - int lpi) { - const int half_nlanes = (v_uint8::nlanes / 2); - - if (4 == lpi) { - // vertical pass - GAPI_DbgAssert(inSz.width >= half_nlanes); - - v_int16 b0 = vx_setall_s16(beta[0]); - v_int16 b1 = vx_setall_s16(beta[1]); - v_int16 b2 = vx_setall_s16(beta[2]); - v_int16 b3 = vx_setall_s16(beta[3]); - - for (int w = 0; w < inSz.width*chanNum; ) { - for (; w <= inSz.width*chanNum - half_nlanes && w >= 0; w += half_nlanes) { - v_int16 val0_0 = v_reinterpret_as_s16(vx_load_expand(&src0[0][w])); - v_int16 val0_1 = v_reinterpret_as_s16(vx_load_expand(&src0[1][w])); - v_int16 val0_2 = v_reinterpret_as_s16(vx_load_expand(&src0[2][w])); - v_int16 val0_3 = v_reinterpret_as_s16(vx_load_expand(&src0[3][w])); - - v_int16 val1_0 = v_reinterpret_as_s16(vx_load_expand(&src1[0][w])); - v_int16 val1_1 = v_reinterpret_as_s16(vx_load_expand(&src1[1][w])); - v_int16 val1_2 = v_reinterpret_as_s16(vx_load_expand(&src1[2][w])); - v_int16 val1_3 = v_reinterpret_as_s16(vx_load_expand(&src1[3][w])); - - v_int16 t0 = v_mulhrs(v_sub_wrap(val0_0, val1_0), b0); - v_int16 t1 = v_mulhrs(v_sub_wrap(val0_1, val1_1), b1); - v_int16 t2 = v_mulhrs(v_sub_wrap(val0_2, val1_2), b2); - v_int16 t3 = v_mulhrs(v_sub_wrap(val0_3, val1_3), b3); - - v_int16 r0 = v_add_wrap(val1_0, t0); - v_int16 r1 = v_add_wrap(val1_1, t1); - v_int16 r2 = v_add_wrap(val1_2, t2); - v_int16 r3 = v_add_wrap(val1_3, t3); - - v_uint8 q0 = v_packus(r0, r1); - v_uint8 q1 = v_packus(r2, r3); - - v_uint8 q2 = v_blend_shiftleft<0xCC /*0b11001100*/, 4>(q0, q1); - v_uint8 q3 = v_blend_shiftright<0xCC /*0b11001100*/, 4>(q0, q1); - - v_uint8 mask = v_setr_s8(0, 8, 4, 12, 1, 9, 5, 13, 2, 10, 6, 14, 3, 11, 7, 15); - - v_uint8 q4 = v_shuffle_s8(q2, mask); - v_uint8 q5 = v_shuffle_s8(q3, mask); - - vx_store(&tmp[4 * w + 0], q4); - vx_store(&tmp[4 * w + 2 * half_nlanes], q5); - } - - if (w < inSz.width*chanNum) { - w = inSz.width*chanNum - half_nlanes; - } - } - - // horizontal pass - v_uint8 val_0, val_1, val_2, val_3; - int shift = (half_nlanes / 4); - GAPI_DbgAssert(outSz.width >= half_nlanes); - for (int x = 0; x < outSz.width; ) { - for (; x <= outSz.width - half_nlanes && x >= 0; x += half_nlanes) { - v_int16 a10 = vx_load(&clone[4 * x]); - v_int16 a32 = vx_load(&clone[4 * (x + 2)]); - v_int16 a54 = vx_load(&clone[4 * (x + 4)]); - v_int16 a76 = vx_load(&clone[4 * (x + 6)]); - - for (int c = 0; c < chanNum; ++c) { - v_gather_channel(val_0, tmp, mapsx, chanNum, c, x, 0); - v_gather_channel(val_1, tmp, mapsx, chanNum, c, x, shift); - v_gather_channel(val_2, tmp, mapsx, chanNum, c, x, shift * 2); - v_gather_channel(val_3, tmp, mapsx, chanNum, c, x, shift * 3); - - v_int16 val0_0 = v_reinterpret_as_s16(v_expand_low(val_0)); - v_int16 val0_1 = v_reinterpret_as_s16(v_expand_low(val_1)); - v_int16 val0_2 = v_reinterpret_as_s16(v_expand_low(val_2)); - v_int16 val0_3 = v_reinterpret_as_s16(v_expand_low(val_3)); - - v_int16 val1_0 = v_reinterpret_as_s16(v_expand_high(val_0)); - v_int16 val1_1 = v_reinterpret_as_s16(v_expand_high(val_1)); - v_int16 val1_2 = v_reinterpret_as_s16(v_expand_high(val_2)); - v_int16 val1_3 = v_reinterpret_as_s16(v_expand_high(val_3)); - - v_int16 t0 = v_mulhrs(v_sub_wrap(val0_0, val1_0), a10); - v_int16 t1 = v_mulhrs(v_sub_wrap(val0_1, val1_1), a32); - v_int16 t2 = v_mulhrs(v_sub_wrap(val0_2, val1_2), a54); - v_int16 t3 = v_mulhrs(v_sub_wrap(val0_3, val1_3), a76); - - v_int16 r0 = v_add_wrap(val1_0, t0); - v_int16 r1 = v_add_wrap(val1_1, t1); - v_int16 r2 = v_add_wrap(val1_2, t2); - v_int16 r3 = v_add_wrap(val1_3, t3); - - v_uint8 q0 = v_packus(r0, r1); - v_uint8 q1 = v_packus(r2, r3); - - v_uint8 mask = v_setr_s8(0, 4, 8, 12, 2, 6, 10, 14, 1, 5, 9, 13, 3, 7, 11, 15); - - v_uint8 q2 = v_shuffle_s8(q0, mask); - v_uint8 q3 = v_shuffle_s8(q1, mask); - - v_uint8 q4 = v_blend_shiftleft<0xCC /*0b11001100*/, 4>(q2, q3); - v_uint8 q5 = v_blend_shiftright<0xCC /*0b11001100*/, 4>(q2, q3); - - v_store_low(&dst[c][0][x], q4); - v_store_high(&dst[c][1][x], q4); - v_store_low(&dst[c][2][x], q5); - v_store_high(&dst[c][3][x], q5); - } - } - - if (x < outSz.width) { - x = outSz.width - half_nlanes; - } - } - - } else { // if any lpi - for (int l = 0; l < lpi; ++l) { - short beta0 = beta[l]; - - // vertical pass - GAPI_DbgAssert(inSz.width*chanNum >= half_nlanes); - for (int w = 0; w < inSz.width*chanNum; ) { - for (; w <= inSz.width*chanNum - half_nlanes; w += half_nlanes) { - v_int16 s0 = v_reinterpret_as_s16(vx_load_expand(&src0[l][w])); - v_int16 s1 = v_reinterpret_as_s16(vx_load_expand(&src1[l][w])); - v_int16 t = v_mulhrs(s0 - s1, beta0) + s1; - v_pack_u_store(tmp + w, t); - } - - if (w < inSz.width*chanNum) { - w = inSz.width*chanNum - half_nlanes; - } - } - - // horizontal pass - GAPI_DbgAssert(outSz.width >= half_nlanes); - for (int x = 0; x < outSz.width; ) { - for (; x <= outSz.width - half_nlanes && x >= 0; x += half_nlanes) { - for (int c = 0; c < chanNum; ++c) { - v_int16 a0 = vx_load(&alpha[x]); // as signed Q1.1.14 - v_int16 sx = vx_load(&mapsx[x]); // as integer (int16) - v_int16 t0 = v_gather_chan(tmp, sx, c, 0); - v_int16 t1 = v_gather_chan(tmp, sx, c, 1); - v_int16 d = v_mulhrs(t0 - t1, a0) + t1; - v_pack_u_store(&dst[c][l][x], d); - } - } - - if (x < outSz.width) { - x = outSz.width - half_nlanes; - } - } - } - } -} -#endif - -// Resize (bi-linear, 8UC3) -template<> -bool calcRowLinear8UC3C4Impl(sse42_tag, - std::array, 3> &dst, - const uint8_t* src0[], - const uint8_t* src1[], - const short alpha[], - const short clone[], // 4 clones of alpha - const short mapsx[], - const short beta[], - uint8_t tmp[], - const Size& inSz, - const Size& outSz, - const int lpi, - const int ) { - constexpr int chanNum = 3; - return calcRowLinear_8UC_Impl_(sse42_tag{}, dst, src0, src1, alpha, clone, mapsx, beta, tmp, inSz, outSz, lpi); -} - -// Resize (bi-linear, 8UC4) -template<> -bool calcRowLinear8UC3C4Impl(sse42_tag, - std::array, 4> &dst, - const uint8_t* src0[], - const uint8_t* src1[], - const short alpha[], - const short clone[], // 4 clones of alpha - const short mapsx[], - const short beta[], - uint8_t tmp[], - const Size& inSz, - const Size& outSz, - const int lpi, - const int ) { - constexpr int chanNum = 4; - return calcRowLinear_8UC_Impl_(sse42_tag{}, dst, src0, src1, alpha, clone, mapsx, beta, tmp, inSz, outSz, lpi); -} - -//------------------------------------------------------------------------------ -#if USE_CVKL - -// from: ie_preprocess_data.hpp -static inline uint8_t saturateU32toU8(uint32_t v) { - return static_cast(v > UINT8_MAX ? UINT8_MAX : v); -} - -// from: ie_preprocess_data_sse42.cpp -static inline uint16_t mulq16(uint16_t a, uint16_t b) { - return static_cast(((uint32_t)a * (uint32_t)b) >> 16); -} - -// extracted from: ie_preprocess_data_sse42.cpp -// (and reworked for 1-channel and fluid's src) -void calcRowArea_CVKL_U8_SSE42(const uchar * src[], - uchar dst[], - const Size & inSz, - const Size & outSz, - int y, - const uint16_t xsi[], - const uint16_t ysi[], - const uint16_t xalpha[], - const uint16_t yalpha[], - int x_max_count, - int y_max_count, - uint16_t vert_sum[]) { - int dwidth = outSz.width; -// int dheight = outSz.height; - int swidth = inSz.width; - int sheight = inSz.height; - - int vest_sum_size = 2*swidth; -// uint16_t* vert_sum = yalpha + dheight*y_max_count; - uint16_t* alpha0 = vert_sum + vest_sum_size; - uint16_t* alpha1 = alpha0 + dwidth; - uint16_t* alpha2 = alpha1 + dwidth; - uint16_t* alpha3 = alpha2 + dwidth; - uint16_t* sxid0 = alpha3 + dwidth; - uint16_t* sxid1 = sxid0 + 4*dwidth; - uint16_t* sxid2 = sxid1 + 4*dwidth; - uint16_t* sxid3 = sxid2 + 4*dwidth; - - uint8_t * pdst_row = dst; - uint16_t* vert_sum_ = vert_sum; - - int ysi_row = ysi[y]; - - memset(vert_sum_, 0, swidth * sizeof(uint16_t)); - - for (int dy = 0; dy < y_max_count; dy++) { - if (ysi_row + dy >= sheight) - break; - - uint16_t yalpha_dy = yalpha[y * y_max_count + dy]; - const uint8_t *sptr_dy = src[dy]; - - int x = 0; - - __m128i yalpha_dy_sse = _mm_set1_epi16(yalpha_dy); - for (; x <= swidth - 16; x += 16) { - __m128i sval = _mm_loadu_si128(reinterpret_cast(sptr_dy + x)); - - // sptr_dy[x] << 8 - __m128i sval_Q16_lo = _mm_unpacklo_epi8(_mm_setzero_si128(), sval); - __m128i sval_Q16_hi = _mm_unpackhi_epi8(_mm_setzero_si128(), sval); - - __m128i vert_sum_lo = _mm_loadu_si128(reinterpret_cast(vert_sum_ + x + 0)); - __m128i vert_sum_hi = _mm_loadu_si128(reinterpret_cast(vert_sum_ + x + 8)); - - vert_sum_lo = _mm_add_epi16(vert_sum_lo, _mm_mulhi_epu16(yalpha_dy_sse, sval_Q16_lo)); - vert_sum_hi = _mm_add_epi16(vert_sum_hi, _mm_mulhi_epu16(yalpha_dy_sse, sval_Q16_hi)); - - _mm_storeu_si128(reinterpret_cast<__m128i*>(vert_sum_ + x + 0), vert_sum_lo); - _mm_storeu_si128(reinterpret_cast<__m128i*>(vert_sum_ + x + 8), vert_sum_hi); - } - - for (; x < swidth; x++) { - vert_sum_[x] += mulq16(yalpha_dy, static_cast(sptr_dy[x] << 8)); - } - } - - if (x_max_count == 2) { - int x = 0; - for (; x <= dwidth - 8; x += 8) { - __m128i res = _mm_set1_epi16(1 << (8 - 1)); - - int id0 = xsi[x]; - - __m128i chunk0 = _mm_loadu_si128(reinterpret_cast(vert_sum_ + id0)); - __m128i chunk1 = _mm_loadu_si128(reinterpret_cast(vert_sum_ + id0 + 8)); - - __m128i sx0_id0 = _mm_loadu_si128(reinterpret_cast(sxid0 + x * 2)); - __m128i sx0_id1 = _mm_loadu_si128(reinterpret_cast(sxid0 + x * 2 + 8)); - - __m128i sx1_id0 = _mm_loadu_si128(reinterpret_cast(sxid1 + x * 2)); - __m128i sx1_id1 = _mm_loadu_si128(reinterpret_cast(sxid1 + x * 2 + 8)); - - __m128i vert_sum0 = _mm_or_si128(_mm_shuffle_epi8(chunk0, sx0_id0), - _mm_shuffle_epi8(chunk1, sx0_id1)); - __m128i vert_sum1 = _mm_or_si128(_mm_shuffle_epi8(chunk0, sx1_id0), - _mm_shuffle_epi8(chunk1, sx1_id1)); - - res = _mm_add_epi16(res, _mm_mulhi_epu16(_mm_loadu_si128(reinterpret_cast(alpha0 + x)), vert_sum0)); - res = _mm_add_epi16(res, _mm_mulhi_epu16(_mm_loadu_si128(reinterpret_cast(alpha1 + x)), vert_sum1)); - - res = _mm_srli_epi16(res, 8); - res = _mm_packus_epi16(res, res); - _mm_storel_epi64(reinterpret_cast<__m128i*>(pdst_row + x), res); - } - - for (; x < dwidth; x++) { - uint16_t res = 1 << (8 - 1); - int id = xsi[x]; - res += mulq16(alpha0[x], vert_sum_[id + 0]); - res += mulq16(alpha1[x], vert_sum_[id + 1]); - pdst_row[x] = saturateU32toU8(res >> 8); - } - } else if (x_max_count == 3) { - int x = 0; - for (; x <= dwidth - 8; x += 8) { - __m128i res = _mm_set1_epi16(1 << (8 - 1)); - - int id0 = xsi[x]; - - __m128i chunk0 = _mm_loadu_si128(reinterpret_cast(vert_sum_ + id0)); - __m128i chunk1 = _mm_loadu_si128(reinterpret_cast(vert_sum_ + id0 + 8)); - __m128i chunk2 = _mm_loadu_si128(reinterpret_cast(vert_sum_ + id0 + 16)); - - __m128i sx0_id0 = _mm_loadu_si128(reinterpret_cast(sxid0 + x * 3)); - __m128i sx0_id1 = _mm_loadu_si128(reinterpret_cast(sxid0 + x * 3 + 8)); - __m128i sx0_id2 = _mm_loadu_si128(reinterpret_cast(sxid0 + x * 3 + 16)); - - __m128i sx1_id0 = _mm_loadu_si128(reinterpret_cast(sxid1 + x * 3)); - __m128i sx1_id1 = _mm_loadu_si128(reinterpret_cast(sxid1 + x * 3 + 8)); - __m128i sx1_id2 = _mm_loadu_si128(reinterpret_cast(sxid1 + x * 3 + 16)); - - __m128i sx2_id0 = _mm_loadu_si128(reinterpret_cast(sxid2 + x * 3)); - __m128i sx2_id1 = _mm_loadu_si128(reinterpret_cast(sxid2 + x * 3 + 8)); - __m128i sx2_id2 = _mm_loadu_si128(reinterpret_cast(sxid2 + x * 3 + 16)); - - __m128i vert_sum0 = _mm_or_si128(_mm_or_si128(_mm_shuffle_epi8(chunk0, sx0_id0), - _mm_shuffle_epi8(chunk1, sx0_id1)), - _mm_shuffle_epi8(chunk2, sx0_id2)); - __m128i vert_sum1 = _mm_or_si128(_mm_or_si128(_mm_shuffle_epi8(chunk0, sx1_id0), - _mm_shuffle_epi8(chunk1, sx1_id1)), - _mm_shuffle_epi8(chunk2, sx1_id2)); - __m128i vert_sum2 = _mm_or_si128(_mm_or_si128(_mm_shuffle_epi8(chunk0, sx2_id0), - _mm_shuffle_epi8(chunk1, sx2_id1)), - _mm_shuffle_epi8(chunk2, sx2_id2)); - - res = _mm_add_epi16(res, _mm_mulhi_epu16(_mm_loadu_si128(reinterpret_cast(alpha0 + x)), vert_sum0)); - res = _mm_add_epi16(res, _mm_mulhi_epu16(_mm_loadu_si128(reinterpret_cast(alpha1 + x)), vert_sum1)); - res = _mm_add_epi16(res, _mm_mulhi_epu16(_mm_loadu_si128(reinterpret_cast(alpha2 + x)), vert_sum2)); - - res = _mm_srli_epi16(res, 8); - res = _mm_packus_epi16(res, res); - _mm_storel_epi64(reinterpret_cast<__m128i*>(pdst_row + x), res); - } - - for (; x < dwidth; x++) { - uint16_t res = 1 << (8 - 1); - int id = xsi[x]; - res += mulq16(alpha0[x], vert_sum_[id + 0]); - res += mulq16(alpha1[x], vert_sum_[id + 1]); - res += mulq16(alpha2[x], vert_sum_[id + 2]); - pdst_row[x] = saturateU32toU8(res >> 8); - } - } else if (x_max_count == 4) { - int x = 0; - for (; x <= dwidth - 8; x += 8) { - __m128i res = _mm_set1_epi16(1 << (8 - 1)); - - int id0 = xsi[x]; - - __m128i chunk0 = _mm_loadu_si128(reinterpret_cast(vert_sum_ + id0)); - __m128i chunk1 = _mm_loadu_si128(reinterpret_cast(vert_sum_ + id0 + 8)); - __m128i chunk2 = _mm_loadu_si128(reinterpret_cast(vert_sum_ + id0 + 16)); - __m128i chunk3 = _mm_loadu_si128(reinterpret_cast(vert_sum_ + id0 + 24)); - - __m128i sx0_id0 = _mm_loadu_si128(reinterpret_cast(sxid0 + x * 4)); - __m128i sx0_id1 = _mm_loadu_si128(reinterpret_cast(sxid0 + x * 4 + 8)); - __m128i sx0_id2 = _mm_loadu_si128(reinterpret_cast(sxid0 + x * 4 + 16)); - __m128i sx0_id3 = _mm_loadu_si128(reinterpret_cast(sxid0 + x * 4 + 24)); - - __m128i sx1_id0 = _mm_loadu_si128(reinterpret_cast(sxid1 + x * 4)); - __m128i sx1_id1 = _mm_loadu_si128(reinterpret_cast(sxid1 + x * 4 + 8)); - __m128i sx1_id2 = _mm_loadu_si128(reinterpret_cast(sxid1 + x * 4 + 16)); - __m128i sx1_id3 = _mm_loadu_si128(reinterpret_cast(sxid1 + x * 4 + 24)); - - __m128i sx2_id0 = _mm_loadu_si128(reinterpret_cast(sxid2 + x * 4)); - __m128i sx2_id1 = _mm_loadu_si128(reinterpret_cast(sxid2 + x * 4 + 8)); - __m128i sx2_id2 = _mm_loadu_si128(reinterpret_cast(sxid2 + x * 4 + 16)); - __m128i sx2_id3 = _mm_loadu_si128(reinterpret_cast(sxid2 + x * 4 + 24)); - - __m128i sx3_id0 = _mm_loadu_si128(reinterpret_cast(sxid3 + x * 4)); - __m128i sx3_id1 = _mm_loadu_si128(reinterpret_cast(sxid3 + x * 4 + 8)); - __m128i sx3_id2 = _mm_loadu_si128(reinterpret_cast(sxid3 + x * 4 + 16)); - __m128i sx3_id3 = _mm_loadu_si128(reinterpret_cast(sxid3 + x * 4 + 24)); - - __m128i vert_sum0 = _mm_or_si128(_mm_or_si128(_mm_shuffle_epi8(chunk0, sx0_id0), - _mm_shuffle_epi8(chunk1, sx0_id1)), - _mm_or_si128(_mm_shuffle_epi8(chunk2, sx0_id2), - _mm_shuffle_epi8(chunk3, sx0_id3))); - __m128i vert_sum1 = _mm_or_si128(_mm_or_si128(_mm_shuffle_epi8(chunk0, sx1_id0), - _mm_shuffle_epi8(chunk1, sx1_id1)), - _mm_or_si128(_mm_shuffle_epi8(chunk2, sx1_id2), - _mm_shuffle_epi8(chunk3, sx1_id3))); - __m128i vert_sum2 = _mm_or_si128(_mm_or_si128(_mm_shuffle_epi8(chunk0, sx2_id0), - _mm_shuffle_epi8(chunk1, sx2_id1)), - _mm_or_si128(_mm_shuffle_epi8(chunk2, sx2_id2), - _mm_shuffle_epi8(chunk3, sx2_id3))); - __m128i vert_sum3 = _mm_or_si128(_mm_or_si128(_mm_shuffle_epi8(chunk0, sx3_id0), - _mm_shuffle_epi8(chunk1, sx3_id1)), - _mm_or_si128(_mm_shuffle_epi8(chunk2, sx3_id2), - _mm_shuffle_epi8(chunk3, sx3_id3))); - - res = _mm_add_epi16(res, _mm_mulhi_epu16(_mm_loadu_si128(reinterpret_cast(alpha0 + x)), vert_sum0)); - res = _mm_add_epi16(res, _mm_mulhi_epu16(_mm_loadu_si128(reinterpret_cast(alpha1 + x)), vert_sum1)); - res = _mm_add_epi16(res, _mm_mulhi_epu16(_mm_loadu_si128(reinterpret_cast(alpha2 + x)), vert_sum2)); - res = _mm_add_epi16(res, _mm_mulhi_epu16(_mm_loadu_si128(reinterpret_cast(alpha3 + x)), vert_sum3)); - - res = _mm_srli_epi16(res, 8); - res = _mm_packus_epi16(res, res); - _mm_storel_epi64(reinterpret_cast<__m128i*>(pdst_row + x), res); - } - - for (; x < dwidth; x++) { - uint16_t res = 1 << (8 - 1); - int id = xsi[x]; - res += mulq16(alpha0[x], vert_sum_[id + 0]); - res += mulq16(alpha1[x], vert_sum_[id + 1]); - res += mulq16(alpha2[x], vert_sum_[id + 2]); - res += mulq16(alpha3[x], vert_sum_[id + 3]); - pdst_row[x] = saturateU32toU8(res >> 8); - } - } else if (x_max_count <= 7) { - int x = 0; - for (; x <= dwidth - 8; x += 8) { - __m128i res = _mm_set1_epi16(1 << (16 - 8 - 1)); - for (int i = 0; i < x_max_count; i++) { - __m128i valpha = _mm_setr_epi16(xalpha[x * x_max_count + x_max_count * 0 + i], - xalpha[x * x_max_count + x_max_count * 1 + i], - xalpha[x * x_max_count + x_max_count * 2 + i], - xalpha[x * x_max_count + x_max_count * 3 + i], - xalpha[x * x_max_count + x_max_count * 4 + i], - xalpha[x * x_max_count + x_max_count * 5 + i], - xalpha[x * x_max_count + x_max_count * 6 + i], - xalpha[x * x_max_count + x_max_count * 7 + i]); - __m128i vvert_sum = _mm_setr_epi16(vert_sum_[xsi[x + 0] + i], - vert_sum_[xsi[x + 1] + i], - vert_sum_[xsi[x + 2] + i], - vert_sum_[xsi[x + 3] + i], - vert_sum_[xsi[x + 4] + i], - vert_sum_[xsi[x + 5] + i], - vert_sum_[xsi[x + 6] + i], - vert_sum_[xsi[x + 7] + i]); - - res = _mm_add_epi16(res, _mm_mulhi_epu16(valpha, vvert_sum)); - } - res = _mm_srli_epi16(res, 8); - res = _mm_packus_epi16(res, res); - _mm_storel_epi64(reinterpret_cast<__m128i*>(pdst_row + x), res); - } - - for (; x < dwidth; x++) { - uint16_t res = 1 << (8 - 1); - for (int i = 0; i < x_max_count; i++) { - uint16_t a = xalpha[x * x_max_count + i]; - int sx = xsi[x] + i; - - res += mulq16(a, vert_sum_[sx]); - } - pdst_row[x] = saturateU32toU8(res >> 8); - } - } else { - for (int x = 0; x < dwidth; x++) { - uint16_t res = 1 << (8 - 1); - __m128i vres = _mm_setzero_si128(); - int id = xsi[x]; - - int i = 0; - for (; i <= x_max_count - 8; i += 8) { - __m128i a = _mm_loadu_si128(reinterpret_cast(xalpha + x * x_max_count + i)); - __m128i s = _mm_loadu_si128(reinterpret_cast(vert_sum_ + id + i)); - - vres = _mm_add_epi16(vres, _mm_mulhi_epu16(a, s)); - } - vres = _mm_add_epi16(vres, _mm_slli_si128(vres, 2)); - vres = _mm_add_epi16(vres, _mm_slli_si128(vres, 4)); - vres = _mm_add_epi16(vres, _mm_slli_si128(vres, 8)); - res += static_cast(_mm_extract_epi16(vres, 7)); - - for (; i < x_max_count; i++) { - uint16_t a = xalpha[x * x_max_count + i]; - uint16_t s = vert_sum_[id + i]; - - res += mulq16(a, s); - } - - pdst_row[x] = saturateU32toU8(res >> 8); - } - } -} - -#endif // CVKL - -template void chanToPlaneRowImpl(sse42_tag, const uint8_t* in, const int chan, const int chs, uint8_t* out, const int length); -template void chanToPlaneRowImpl(sse42_tag, const float* in, const int chan, const int chs, float* out, const int length); - -template void nv12ToRgbRowImpl(sse42_tag, const uint8_t** y_rows, const uint8_t* uv_row, uint8_t** out_rows, const int buf_width); - -template void i420ToRgbRowImpl(sse42_tag, const uint8_t** y_rows, const uint8_t* u_row, - const uint8_t* v_row, uint8_t** out_rows, const int buf_width); - -template void splitRowImpl(sse42_tag, const uint8_t* in, std::array& outs, const int length); -template void splitRowImpl(sse42_tag, const float* in, std::array& outs, const int length); -template void splitRowImpl(sse42_tag, const uint8_t* in, std::array& outs, const int length); -template void splitRowImpl(sse42_tag, const float* in, std::array& outs, const int length); -template void splitRowImpl(sse42_tag, const uint8_t* in, std::array& outs, const int length); -template void splitRowImpl(sse42_tag, const float* in, std::array& outs, const int length); - -template void mergeRowImpl(sse42_tag, const std::array& ins, uint8_t* out, const int length); -template void mergeRowImpl(sse42_tag, const std::array& ins, float* out, const int length); -template void mergeRowImpl(sse42_tag, const std::array& ins, uint8_t* out, const int length); -template void mergeRowImpl(sse42_tag, const std::array& ins, float* out, const int length); -template void mergeRowImpl(sse42_tag, const std::array& ins, uint8_t* out, const int length); -template void mergeRowImpl(sse42_tag, const std::array& ins, float* out, const int length); - -template void calcRowLinear32FC1Impl(sse42_tag, float* dst[], const float* src0[], const float* src1[], - const float alpha[], const int mapsx[], - const float beta[], const Size& inSz, const Size& outSz, - const int lpi, const int l); - -template void calcRowAreaImpl(sse42_tag, uint8_t dst[], const uint8_t* src[], const Size& inSz, - const Size& outSz, Q0_16 yalpha, const MapperUnit8U &ymap, - int xmaxdf, const short xindex[], const Q0_16 xalpha[], - Q8_8 vbuf[]); - -template void calcRowAreaImpl(sse42_tag, float dst[], const float *src[], const Size& inSz, - const Size& outSz, float yalpha, const MapperUnit32F& ymap, - int xmaxdf, const int xindex[], const float xalpha[], - float vbuf[]); -} // namespace kernels -} // namespace gapi -} // namespace InferenceEngine diff --git a/src/common/preprocessing/src/cpu_x86_sse42/ie_preprocess_gapi_kernels_sse42.hpp b/src/common/preprocessing/src/cpu_x86_sse42/ie_preprocess_gapi_kernels_sse42.hpp deleted file mode 100644 index dcb51b52e62..00000000000 --- a/src/common/preprocessing/src/cpu_x86_sse42/ie_preprocess_gapi_kernels_sse42.hpp +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright (C) 2018-2023 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#pragma once - -#include "ie_preprocess_gapi_kernels.hpp" -#include "ie_preprocess_gapi_kernels_impl.hpp" -#include - -namespace InferenceEngine { -namespace gapi { -namespace kernels { - -using C3 = std::integral_constant; -using C4 = std::integral_constant; -//---------------------------------------------------------------------- - -void calcRowArea_8U(uchar dst[], const uchar *src[], const Size &inSz, const Size &outSz, - Q0_16 yalpha, const MapperUnit8U& ymap, int xmaxdf, const short xindex[], const Q0_16 xalpha[], - Q8_8 vbuf[]); - -void calcRowArea_32F(float dst[], const float *src[], const Size &inSz, const Size &outSz, - float yalpha, const MapperUnit32F& ymap, int xmaxdf, const int xindex[], const float xalpha[], - float vbuf[]); - -#if USE_CVKL -void calcRowArea_CVKL_U8_SSE42(const uchar * src[], - uchar dst[], - const Size & inSz, - const Size & outSz, - int y, - const uint16_t xsi[], - const uint16_t ysi[], - const uint16_t xalpha[], - const uint16_t yalpha[], - int x_max_count, - int y_max_count, - uint16_t vert_sum[]); -#endif - -//---------------------------------------------------------------------- -template -void chanToPlaneRowImpl(isa_tag_t, const T* in, const int chan, const int chs, - T* out, const int length); - -template -void nv12ToRgbRowImpl(isa_tag_t, const uint8_t** y_rows, const uint8_t* uv_row, uint8_t** out_rows, const int buf_width); - -template -void i420ToRgbRowImpl(isa_tag_t, const uint8_t** y_rows, const uint8_t* u_row, - const uint8_t* v_row, uint8_t** out_rows, const int buf_width); - -template -void splitRowImpl(isa_tag_t, const T* in, std::array& outs, const int length); - -template -void mergeRowImpl(isa_tag_t, const std::array& ins, T* out, const int length); - -template -bool calcRowLinear8UC1Impl(isa_tag_t, uint8_t* dst[], const uint8_t* src0[], const uint8_t* src1[], - const short alpha[], const short clone[], const short mapsx[], - const short beta[], uint8_t tmp[], const Size& inSz, - const Size& outSz, const int lpi, const int l); - -template -void calcRowLinear32FC1Impl(isa_tag_t, float* dst[], const float* src0[], const float* src1[], - const float alpha[], const int mapsx[], - const float beta[], const Size& inSz, const Size& outSz, - const int lpi, const int l); - -template -bool calcRowLinear8UC3C4Impl(isa_tag_t, std::array, chs>& dst, - const uint8_t* src0[], const uint8_t* src1[], - const short alpha[], const short clone[], const short mapsx[], - const short beta[], uint8_t tmp[], const Size& inSz, - const Size& outSz, const int lpi, const int l); - -template -void calcRowAreaImpl(isa_tag_t, T dst[], const T* src[], const Size& inSz, - const Size& outSz, A yalpha, const MapperUnit& ymap, - int xmaxdf, const I xindex[], const A xalpha[], W vbuf[]); -} // namespace kernels -} // namespace gapi -} // namespace InferenceEngine diff --git a/src/common/preprocessing/src/ie_preprocess_data.cpp b/src/common/preprocessing/src/ie_preprocess_data.cpp deleted file mode 100644 index 71129d68699..00000000000 --- a/src/common/preprocessing/src/ie_preprocess_data.cpp +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright (C) 2018-2023 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#include "ie_preprocess_gapi.hpp" -#include "ie_system_conf.h" -#include "ie_preprocess_data.hpp" -#include "ie_preprocess_itt.hpp" - -#include "debug.h" -#include - -#include - -namespace InferenceEngine { - -IE_SUPPRESS_DEPRECATED_START -/** - * @brief This class stores pre-process information for exact input - */ -class PreProcessData : public IPreProcessData { - /** - * @brief ROI blob. - */ - Blob::Ptr _userBlob = nullptr; - - /** - * @brief Pointer-to-implementation (PIMPL) hiding preprocessing implementation details. - * BEWARE! Will be shared among copies! - */ - std::shared_ptr _preproc; - -public: - void setRoiBlob(const Blob::Ptr &blob) override; - - Blob::Ptr getRoiBlob() const override; - - void execute(Blob::Ptr &preprocessedBlob, const PreProcessInfo &info, bool serial, int batchSize = -1) override; - - void isApplicable(const Blob::Ptr &src, const Blob::Ptr &dst) override; -}; - -void CreatePreProcessData(std::shared_ptr& data) { - data = std::make_shared(); -} - -void PreProcessData::setRoiBlob(const Blob::Ptr &blob) { - _userBlob = blob; -} - -Blob::Ptr PreProcessData::getRoiBlob() const { - return _userBlob; -} - -void PreProcessData::execute(Blob::Ptr &preprocessedBlob, const PreProcessInfo &info, bool serial, - int batchSize) { - OV_ITT_SCOPED_TASK(itt::domains::IEPreproc, "Preprocessing"); - - auto algorithm = info.getResizeAlgorithm(); - auto fmt = info.getColorFormat(); - - if (_userBlob == nullptr || preprocessedBlob == nullptr) { - IE_THROW() << "Input pre-processing is called with null " << (_userBlob == nullptr ? "_userBlob" : "preprocessedBlob"); - } - - batchSize = PreprocEngine::getCorrectBatchSize(batchSize, _userBlob); - - if (!_preproc) { - _preproc.reset(new PreprocEngine); - } - - _preproc->preprocessWithGAPI(_userBlob, preprocessedBlob, algorithm, fmt, serial, batchSize); -} - -void PreProcessData::isApplicable(const Blob::Ptr &src, const Blob::Ptr &dst) { - PreprocEngine::checkApplicabilityGAPI(src, dst); -} - -} // namespace InferenceEngine diff --git a/src/common/preprocessing/src/ie_preprocess_data.hpp b/src/common/preprocessing/src/ie_preprocess_data.hpp deleted file mode 100644 index 79d8e84c140..00000000000 --- a/src/common/preprocessing/src/ie_preprocess_data.hpp +++ /dev/null @@ -1,137 +0,0 @@ -// Copyright (C) 2018-2023 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#pragma once - -#include -#include -#include - -#include "openvino/runtime/common.hpp" -#include "openvino/util/file_util.hpp" -#include "openvino/util/shared_object.hpp" - -#include -#include -#include -#include "ie_version.hpp" - -namespace InferenceEngine { - -IE_SUPPRESS_DEPRECATED_START -/** - * @brief This class stores pre-process information for exact input - */ -class IPreProcessData : public std::enable_shared_from_this { -public: - /** - * @brief Sets ROI blob to be resized and placed to the default input blob during pre-processing. - * @param blob ROI blob. - */ - virtual void setRoiBlob(const Blob::Ptr &blob) = 0; - - /** - * @brief Gets pointer to the ROI blob used for a given input. - * @return Blob pointer. - */ - virtual Blob::Ptr getRoiBlob() const = 0; - - /** - * @brief Executes input pre-processing with a given pre-processing information. - * @param outBlob pre-processed output blob to be used for inference. - * @param info pre-processing info that specifies resize algorithm and color format. - * @param serial disable OpenMP threading if the value set to true. - * @param batchSize batch size for pre-processing. - */ - virtual void execute(Blob::Ptr &preprocessedBlob, const PreProcessInfo& info, bool serial, int batchSize = -1) = 0; - - virtual void isApplicable(const Blob::Ptr &src, const Blob::Ptr &dst) = 0; - -protected: - virtual ~IPreProcessData() = default; -}; - -OPENVINO_PLUGIN_API void CreatePreProcessData(std::shared_ptr& data); - -#define OV_PREPROC_PLUGIN_CALL_STATEMENT(...) \ - if (!_ptr) \ - IE_THROW() << "Wrapper used in the OV_PREPROC_PLUGIN_CALL_STATEMENT was not initialized."; \ - try { \ - __VA_ARGS__; \ - } catch (...) { \ - ::InferenceEngine::details::Rethrow(); \ - } - -class PreProcessDataPlugin { - std::shared_ptr _so = nullptr; - std::shared_ptr _ptr = nullptr; - -public: - PreProcessDataPlugin() { -#ifdef OPENVINO_STATIC_LIBRARY -# ifdef ENABLE_GAPI_PREPROCESSING - CreatePreProcessData(_ptr); - if (!_ptr) - IE_THROW() << "Failed to create IPreProcessData for G-API based preprocessing"; -# else - IE_THROW() << "OpenVINO Runtime is compiled without G-API preprocessing support.\n" - "Use 'cmake -DENABLE_GAPI_PREPROCESSING=ON ...'"; -# endif // ENABLE_GAPI_PREPROCESSING -#else - // preprocessing plugin can be found in the following locations - // 1. in openvino-X.Y.Z location relative to libopenvino.so - // 2. in the same folder as libopenvino.so - - ov::util::FilePath ovLibraryPath = getInferenceEngineLibraryPath(); - ov::util::FilePath libraryName = ov::util::to_file_path(std::string("openvino_gapi_preproc") + std::string(OV_BUILD_POSTFIX)); - libraryName = FileUtils::makePluginLibraryName({}, libraryName); - - std::ostringstream str; - str << "openvino-" << IE_VERSION_MAJOR << "." << IE_VERSION_MINOR << "." << IE_VERSION_PATCH; - ov::util::FilePath ovLibraryPathPlusOV = FileUtils::makePath(ovLibraryPath, ov::util::to_file_path(str.str())); - - const ov::util::FilePath preprocLibraryPathPlusOV = FileUtils::makePath(ovLibraryPathPlusOV, libraryName); - const ov::util::FilePath preprocLibraryPath = FileUtils::makePath(ovLibraryPath, libraryName); - const bool existsInOV = FileUtils::fileExist(preprocLibraryPathPlusOV); - const bool existsInLib = FileUtils::fileExist(preprocLibraryPath); - - if (!existsInOV && !existsInLib) { - IE_THROW() << "Please, make sure that pre-processing library " - << ov::util::from_file_path(libraryName) << " is in " - << ov::util::from_file_path(preprocLibraryPathPlusOV) << " or " << ov::util::from_file_path(preprocLibraryPath); - } - - using CreateF = void(std::shared_ptr& data); - _so = ov::util::load_shared_object(existsInOV ? preprocLibraryPathPlusOV.c_str() : preprocLibraryPath.c_str()); - reinterpret_cast(ov::util::get_symbol(_so, "CreatePreProcessData"))(_ptr); -#endif - } - - void setRoiBlob(const Blob::Ptr &blob) { - OV_PREPROC_PLUGIN_CALL_STATEMENT(_ptr->setRoiBlob(blob)); - } - - Blob::Ptr getRoiBlob() const { - OV_PREPROC_PLUGIN_CALL_STATEMENT(return _ptr->getRoiBlob()); - } - - void execute(Blob::Ptr &preprocessedBlob, const PreProcessInfo& info, bool serial, int batchSize = -1) { - OV_PREPROC_PLUGIN_CALL_STATEMENT(_ptr->execute(preprocessedBlob, info, serial, batchSize)); - } - - void isApplicable(const Blob::Ptr &src, const Blob::Ptr &dst) { - OV_PREPROC_PLUGIN_CALL_STATEMENT(_ptr->isApplicable(src, dst)); - } -}; - -#undef OV_PREPROC_PLUGIN_CALL_STATEMENT - -using PreProcessDataPtr = std::shared_ptr; - -inline PreProcessDataPtr CreatePreprocDataHelper() { - return std::make_shared(); -} - -IE_SUPPRESS_DEPRECATED_END -} // namespace InferenceEngine diff --git a/src/common/preprocessing/src/ie_preprocess_gapi.cpp b/src/common/preprocessing/src/ie_preprocess_gapi.cpp deleted file mode 100644 index 8bdf4b03e67..00000000000 --- a/src/common/preprocessing/src/ie_preprocess_gapi.cpp +++ /dev/null @@ -1,872 +0,0 @@ -// Copyright (C) 2018-2023 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#include -#include -#include -#include -#include -#include -#include -#include - -// Careful reader, don't worry -- it is not the whole OpenCV, -// it is just a single stand-alone component of it -#include -#include - -#include "ie_blob.h" -#include "ie_compound_blob.h" -#include "ie_input_info.hpp" -#include "ie_preprocess_gapi.hpp" -#include "ie_preprocess_gapi_kernels.hpp" -#include "ie_preprocess_itt.hpp" -#include "debug.h" - -#include "ie_parallel.hpp" - -#include // GFluidOutputRois - -IE_SUPPRESS_DEPRECATED_START - -namespace InferenceEngine { -namespace { -int get_cv_depth(const TensorDesc &ie_desc); - -namespace G { - struct Strides {int N; int C; int H; int W;}; - struct Dims {int N; int C; int H; int W;}; - struct Desc {Dims d; Strides s; int prec;}; - - void fix_strides_nhwc(const Dims &d, Strides &s) { - if (s.W > d.C) { - s.C = 1; - s.W = s.C*d.C; - s.H = s.W*d.W; - s.N = s.H*d.H; - } - } - - Desc decompose(const TensorDesc& ie_desc) { - const auto& ie_blk_desc = ie_desc.getBlockingDesc(); - const auto& ie_dims = ie_desc.getDims(); - const auto& ie_strides = ie_blk_desc.getStrides(); - const bool nhwc_layout = ie_desc.getLayout() == NHWC; - - Dims d = { - static_cast(ie_dims[0]), - static_cast(ie_dims[1]), - static_cast(ie_dims[2]), - static_cast(ie_dims[3]) - }; - - Strides s = { - static_cast(ie_strides[0]), - static_cast(nhwc_layout ? ie_strides[3] : ie_strides[1]), - static_cast(nhwc_layout ? ie_strides[1] : ie_strides[2]), - static_cast(nhwc_layout ? ie_strides[2] : ie_strides[3]), - }; - - if (nhwc_layout) fix_strides_nhwc(d, s); - - return Desc{d, s, get_cv_depth(ie_desc)}; - } - - Desc decompose(const Blob::Ptr& blob) { - return decompose(blob->getTensorDesc()); - } -} // namespace G - -inline int get_cv_depth(const TensorDesc &ie_desc) { - switch (ie_desc.getPrecision()) { - case Precision::U8: return CV_8U; - case Precision::FP32: return CV_32F; - case Precision::U16: return CV_16U; - case Precision::I16: return CV_16S; - case Precision::FP16: return CV_16F; - - default: IE_THROW() << "Unsupported data type"; - } -} - -std::vector> bind_to_blob(const Blob::Ptr& blob, - int batch_size) { - const auto& ie_desc = blob->getTensorDesc(); - const auto& ie_desc_blk = ie_desc.getBlockingDesc(); - const auto desc = G::decompose(blob); - const auto cv_depth = get_cv_depth(ie_desc); - const auto stride = desc.s.H*blob->element_size(); - const auto planeSize = cv::gapi::own::Size(desc.d.W, desc.d.H); - // Note: operating with strides (desc.s) rather than dimensions (desc.d) which is vital for ROI - // blobs (data buffer is shared but dimensions are different due to ROI != original image) - const auto batch_offset = desc.s.N * blob->element_size(); - - std::vector> result(batch_size); - - uint8_t* blob_ptr = static_cast(blob->buffer()); - if (blob_ptr == nullptr) { - IE_THROW() << "Blob buffer is nullptr"; - } - blob_ptr += blob->element_size()*ie_desc_blk.getOffsetPadding(); - - for (int i = 0; i < batch_size; ++i) { - uint8_t* curr_data_ptr = blob_ptr + i * batch_offset; - - std::vector planes; - if (ie_desc.getLayout() == Layout::NHWC) { - planes.emplace_back(planeSize.height, planeSize.width, CV_MAKETYPE(cv_depth, desc.d.C), - curr_data_ptr, stride); - } else { // NCHW - if (desc.d.C <= 0) { - IE_THROW() << "Invalid number of channels in blob tensor descriptor, " - "expected >0, actual: " << desc.d.C; - } - const auto planeType = CV_MAKETYPE(cv_depth, 1); - for (int ch = 0; ch < desc.d.C; ch++) { - cv::gapi::own::Mat plane(planeSize.height, planeSize.width, planeType, - curr_data_ptr + ch*desc.s.C*blob->element_size(), stride); - planes.emplace_back(plane); - } - } - - result[i] = std::move(planes); - } - return result; -} - -template -std::vector to_vec_impl(std::tuple &&gmats, cv::detail::Seq) { - return { std::get(gmats)... }; -} - -template -std::vector to_vec(std::tuple &&gmats) { - return to_vec_impl(std::move(gmats), typename cv::detail::MkSeq::type()); -} - -// convert input to planar format -std::vector split(const std::vector& inputs, - int channels) { - if (inputs.empty()) { - return inputs; - } - - std::vector planes; - - switch (channels) { - case 1: planes = { inputs[0] }; break; - case 2: planes = to_vec(gapi::Split2::on(inputs[0])); break; - case 3: planes = to_vec(gapi::Split3::on(inputs[0])); break; - case 4: planes = to_vec(gapi::Split4::on(inputs[0])); break; - default: - for (int chan = 0; chan < channels; chan++) - planes.emplace_back(gapi::ChanToPlane::on(inputs[0], chan)); - break; - } - - return planes; -} - -// convert input to interleaved format -std::vector merge(const std::vector& inputs, - int channels) { - if (inputs.empty()) { - return inputs; - } - - std::vector interleaved; - - switch (channels) { - case 1: interleaved.emplace_back(inputs[0]); break; - case 2: interleaved.emplace_back(gapi::Merge2::on(inputs[0], inputs[1])); break; - case 3: interleaved.emplace_back(gapi::Merge3::on(inputs[0], inputs[1], inputs[2])); break; - case 4: interleaved.emplace_back(gapi::Merge4::on(inputs[0], inputs[1], inputs[2], inputs[3])); break; - default: IE_THROW() << "output channels value " << channels - << " is not supported for HWC [by G-API]." - << " Expected range (inclusive): [1;4]."; - } - - return interleaved; -} - -// validate input/output ColorFormat-related parameters -void validateColorFormats(const G::Desc &in_desc, - const G::Desc &out_desc, - Layout in_layout, - Layout out_layout, - ColorFormat input_color_format, - ColorFormat output_color_format) { - const auto verify_desc = [] (const G::Desc& desc, ColorFormat fmt, const std::string& desc_prefix) { - const auto throw_invalid_number_of_channels = [&](){ - IE_THROW() << desc_prefix << " tensor descriptor " - << "has invalid number of channels " - << desc.d.C << " for " << fmt - << "color format"; - }; - switch (fmt) { - case ColorFormat::RGB: - case ColorFormat::BGR: { - if (desc.d.C != 3) throw_invalid_number_of_channels(); - break; - } - case ColorFormat::RGBX: - case ColorFormat::BGRX: { - if (desc.d.C != 4) throw_invalid_number_of_channels(); - break; - } - - default: break; - } - }; - - const auto verify_layout = [] (Layout layout, const std::string& layout_prefix) { - if (layout != NHWC && layout != NCHW) { - IE_THROW() << layout_prefix << " layout " << layout - << " is not supported by pre-processing [by G-API]"; - } - }; - - // verify inputs/outputs and throw on error - const bool color_conv_required = !((output_color_format == input_color_format) || (input_color_format == ColorFormat::RAW)); - if (color_conv_required && (output_color_format == ColorFormat::RAW)) { - IE_THROW() << "Network's expected color format is unspecified"; - } - - verify_layout(in_layout, "Input blob"); - verify_layout(out_layout, "Network's blob"); - - if (!color_conv_required) { - // verify input and output have the same number of channels - if (in_desc.d.C != out_desc.d.C) { - IE_THROW() << "Input and network expected blobs have different number of " - << "channels: expected " << out_desc.d.C << " channels but provided " - << in_desc.d.C << " channels"; - } - return; - } - - // planar 4-channel input is not supported, user can easily pass 3 channels instead of 4 - if (in_layout == NCHW - && (input_color_format == ColorFormat::RGBX || input_color_format == ColorFormat::BGRX)) { - IE_THROW() << "Input blob with NCHW layout and BGRX/RGBX color format is " - << "explicitly not supported, use NCHW + BGR/RGB color format " - << "instead (3 image planes instead of 4)"; - } - - // verify input and output against their corresponding color format - verify_desc(in_desc, input_color_format, "Input blob"); - verify_desc(out_desc, output_color_format, "Network's blob"); -} - -bool has_zeros(const SizeVector& vec) { - return std::any_of(vec.cbegin(), vec.cend(), [] (size_t e) { return e == 0; }); -} - -void validateTensorDesc(const TensorDesc& desc) { - auto supports_layout = [](Layout l) { return l == Layout::NCHW || l == Layout::NHWC; }; - const auto layout = desc.getLayout(); - const auto& dims = desc.getDims(); - if (!supports_layout(layout) - || dims.size() != 4 - || desc.getBlockingDesc().getStrides().size() != 4) { - IE_THROW() << "Preprocess support NCHW/NHWC only"; - } - if (has_zeros(dims)) { - IE_THROW() << "Invalid input data dimensions: " - << details::dumpVec(dims); - } -} - -void validateBlob(const MemoryBlob::Ptr &) {} - -const std::pair getTensorDescAndLayout(const MemoryBlob::Ptr &blob) { - const auto& desc = blob->getTensorDesc(); - return {desc, desc.getLayout()}; -} - -G::Desc getGDesc(G::Desc in_desc_y, const MemoryBlob::Ptr &) { - return in_desc_y; -} - -class PlanarColorConversions { - using GMats = std::vector; - using CvtFunction = std::function; - struct Hash { - inline size_t operator()(const std::pair& p) const { - return static_cast((p.first << 16) ^ p.second); - } - }; - std::unordered_map, CvtFunction, Hash> m_conversions; - - // convert RGB -> BGR and BGR -> RGB - static std::vector reverse3(const std::vector& inputs, - Layout in_layout, - Layout out_layout, - ResizeAlgorithm algorithm) { - auto planes = inputs; - if (in_layout == NHWC) { - planes = split(inputs, 3); - } - - // if there's no resize after color convert && output is planar, we have to copy input to - // output by doing actual G-API operation. otherwise, the graph will be empty (no - // operations) - if (algorithm == NO_RESIZE && in_layout == out_layout && out_layout == NCHW) { - std::vector reversed(3); - reversed[0] = gapi::ChanToPlane::on(planes[2], 0); - reversed[1] = gapi::ChanToPlane::on(planes[1], 0); - reversed[2] = gapi::ChanToPlane::on(planes[0], 0); - return reversed; - } - - std::reverse(planes.begin(), planes.end()); - return planes; - } - - // convert RGBX -> RGB and BGRX -> BGR - static std::vector dropLastChan(const std::vector& inputs, - Layout in_layout, - Layout out_layout, - ResizeAlgorithm /*algorithm*/) { - // Note: input is always interleaved, planar input is converted to RGB/BGR on the user side - auto planes = split(inputs, 4); - planes.pop_back(); - return planes; - } - - // convert RGBX -> BGR and BGRX -> RGB - static std::vector dropLastChanAndReverse(const std::vector& inputs, - Layout in_layout, - Layout out_layout, - ResizeAlgorithm algorithm) { - auto planes = dropLastChan(inputs, in_layout, out_layout, algorithm); - std::reverse(planes.begin(), planes.end()); - return planes; - } - - static std::vector NV12toRGB(const std::vector& inputs, - Layout, - Layout, - ResizeAlgorithm) { - // in_layout is always NCHW - auto interleaved_rgb = gapi::NV12toRGB::on(inputs[0], inputs[1]); - return split({interleaved_rgb}, 3); - } - - static std::vector NV12toBGR(const std::vector& inputs, - Layout in_layout, - Layout out_layout, - ResizeAlgorithm algorithm) { - auto planes = NV12toRGB(inputs, in_layout, out_layout, algorithm); - std::reverse(planes.begin(), planes.end()); - return planes; - } - - static std::vector I420toRGB(const std::vector& inputs, - Layout, - Layout, - ResizeAlgorithm) { - // in_layout is always NCHW - auto interleaved_rgb = gapi::I420toRGB::on(inputs[0], inputs[1], inputs[2]); - return split({interleaved_rgb}, 3); - } - - static std::vector I420toBGR(const std::vector& inputs, - Layout in_layout, - Layout out_layout, - ResizeAlgorithm algorithm) { - auto planes = I420toRGB(inputs, in_layout, out_layout, algorithm); - std::reverse(planes.begin(), planes.end()); - return planes; - } - -public: - PlanarColorConversions() { - IE_SUPPRESS_DEPRECATED_START - m_conversions = { - { {ColorFormat::RGB, ColorFormat::BGR}, reverse3 }, - { {ColorFormat::BGR, ColorFormat::RGB}, reverse3 }, - { {ColorFormat::RGBX, ColorFormat::RGB}, dropLastChan }, - { {ColorFormat::BGRX, ColorFormat::BGR}, dropLastChan }, - { {ColorFormat::RGBX, ColorFormat::BGR}, dropLastChanAndReverse }, - { {ColorFormat::BGRX, ColorFormat::RGB}, dropLastChanAndReverse } - }; - IE_SUPPRESS_DEPRECATED_END - } - - const CvtFunction& at(ColorFormat in_fmt, ColorFormat out_fmt) const { - auto cvtFunc = m_conversions.find(std::make_pair(in_fmt, out_fmt)); - if (cvtFunc == m_conversions.cend()) { - IE_THROW() << "Color conversion " << in_fmt << " -> " << out_fmt - << " is not supported"; - } - return cvtFunc->second; - } -}; - -// construct G-API graph pipeline to color convert input into output -// Note: always returns planar output -std::vector -convertColorPlanar(const std::vector& inputs, - const G::Desc& in_desc, - Layout in_layout, - Layout out_layout, - ColorFormat input_color_format, - ColorFormat output_color_format, - ResizeAlgorithm algorithm) { - // do not perform color conversions if not requested (ColorFormat::RAW) or if input already has - // network's expected color format - if (input_color_format == ColorFormat::RAW || input_color_format == output_color_format) { - // convert input into planar form - if (in_layout == NHWC) { - return split(inputs, in_desc.d.C); - } - return inputs; - } - - static PlanarColorConversions conversions; - - const auto& convert = conversions.at(input_color_format, output_color_format); - auto planes = convert(inputs, in_layout, out_layout, algorithm); - if (planes.empty()) { - IE_THROW() << "[G-API] internal error: failed to convert input data into planar " - << "format"; - } - - return planes; -} - -cv::GComputation buildGraph(const G::Desc &in_desc, - const G::Desc &out_desc, - Layout in_layout, - Layout out_layout, - ResizeAlgorithm algorithm, - ColorFormat input_color_format, - ColorFormat output_color_format) { - // perform basic validation to ensure our assumptions about input and output are correct - validateColorFormats(in_desc, out_desc, in_layout, out_layout, input_color_format, - output_color_format); - - std::vector inputs; // 1 element if NHWC, C elements if NCHW - if (in_layout == NHWC) { - inputs.resize(1); - } else if (in_layout == NCHW) { - inputs.resize(in_desc.d.C); - } - - // specific pre-processing case: - // 1. Requires interleaved image of type CV_8UC3/CV_8UC4 (except for NV12/I420 input) - // 2. Supports bilinear resize only - // 3. Supports NV12/I420 -> RGB/BGR color transformations - const auto io_color_formats = std::make_tuple(input_color_format, output_color_format); - const bool drop_channel = (io_color_formats == std::make_tuple(ColorFormat::RGBX, ColorFormat::RGB)) || - (io_color_formats == std::make_tuple(ColorFormat::BGRX, ColorFormat::BGR)); - const bool specific_case_of_preproc = ((in_layout == NHWC) - && (in_desc.d.C == 3 || drop_channel) - && ((in_desc.prec == CV_8U) && (in_desc.prec == out_desc.prec)) - && (algorithm == RESIZE_BILINEAR) - && (input_color_format == ColorFormat::RAW - || input_color_format == output_color_format - || drop_channel)); - if (specific_case_of_preproc) { - const auto input_sz = cv::gapi::own::Size(in_desc.d.W, in_desc.d.H); - const auto scale_sz = cv::gapi::own::Size(out_desc.d.W, out_desc.d.H); - - // convert color format to RGB in case of NV12 input - std::vector color_converted_input; - color_converted_input = inputs; - - auto planes = drop_channel ? - to_vec(gapi::ScalePlanes4:: on( - color_converted_input[0], in_desc.prec, input_sz, scale_sz, cv::INTER_LINEAR)) - : to_vec(gapi::ScalePlanes ::on( - color_converted_input[0], in_desc.prec, input_sz, scale_sz, cv::INTER_LINEAR)); - - if (drop_channel) { - planes.pop_back(); - } - - std::vector outputs; - if (out_layout == NHWC) { - outputs.emplace_back(gapi::Merge3::on(planes[0], planes[1], planes[2])); - } else { - outputs = planes; - } - return cv::GComputation(inputs, outputs); - } - - auto planes = convertColorPlanar(inputs, in_desc, in_layout, out_layout, - input_color_format, output_color_format, algorithm); - - const int number_of_planes = static_cast(planes.size()); - if (number_of_planes != out_desc.d.C) { - IE_THROW() << "[G-API] internal error: number of channels after color conversion " - << "!= network's expected number of channels: " - << number_of_planes << " != " << out_desc.d.C; - } - - const int tmp_prec = CV_32F; - - std::vector outputs; - const bool resize_needed = (algorithm != NO_RESIZE); - const bool need_tmp_prec_conv = resize_needed && (in_desc.prec != CV_8U) && (in_desc.prec != CV_32F); - - if (resize_needed) { - // resize every plane - std::vector out_planes; - out_planes.reserve(planes.size()); - const int interp_type = [](const ResizeAlgorithm &ar) { - switch (ar) { - case RESIZE_AREA: return cv::INTER_AREA; - case RESIZE_BILINEAR: return cv::INTER_LINEAR; - default: IE_THROW() << "Unsupported resize operation"; - } - } (algorithm); - - std::transform(planes.begin(), planes.end(), std::back_inserter(out_planes), [&](const cv::GMat& m) { - const auto input_sz = cv::gapi::own::Size(in_desc.d.W, in_desc.d.H); - const auto scale_sz = cv::gapi::own::Size(out_desc.d.W, out_desc.d.H); - - cv::GMat converted = m; - int prec = in_desc.prec; - - if (need_tmp_prec_conv) { - std::tie(converted, prec) = std::make_tuple(gapi::ConvertDepth::on(m, tmp_prec), tmp_prec); - } - - return gapi::ScalePlane::on(converted, prec, input_sz, scale_sz, interp_type); - }); - outputs = out_planes; - } else { - outputs = planes; - } - - if ((in_desc.prec != out_desc.prec) || need_tmp_prec_conv) { - auto convert_prec = [](const std::vector & src_gmats, int dst_precision) { - std::vector dst_gmats; - std::transform(src_gmats.begin(), src_gmats.end(), std::back_inserter(dst_gmats), [&](cv::GMat const& m){ - return gapi::ConvertDepth::on(m, dst_precision); - }); - return dst_gmats; - }; - - outputs = convert_prec(outputs, out_desc.prec); - } - // convert to interleaved if NHWC is required as output - if (out_layout == NHWC) { - outputs = merge(outputs, out_desc.d.C); - } - - return cv::GComputation(inputs, outputs); -} -} // anonymous namespace - -PreprocEngine::PreprocEngine() : _lastComp(parallel_get_max_threads()) {} - -PreprocEngine::Update PreprocEngine::needUpdate(const CallDesc &newCallOrig) const { - // Given our knowledge about Fluid, full graph rebuild is required - // if and only if: - // 0. This is the first call ever - // 1. precision has changed (affects kernel versions) - // 2. layout has changed (affects graph topology) - // 3. algorithm has changed (affects kernel version) - // 4. dimensions have changed from downscale to upscale or vice-versa if interpolation is AREA - // 5. color format has changed (affects graph topology) - if (!_lastCall) { - return Update::REBUILD; - } - - BlobDesc last_in; - BlobDesc last_out; - ResizeAlgorithm last_algo = ResizeAlgorithm::NO_RESIZE; - std::tie(last_in, last_out, last_algo) = *_lastCall; - - CallDesc newCall = newCallOrig; - BlobDesc new_in; - BlobDesc new_out; - ResizeAlgorithm new_algo = ResizeAlgorithm::NO_RESIZE; - std::tie(new_in, new_out, new_algo) = newCall; - - // Declare two empty vectors per each call - SizeVector last_in_size; - SizeVector last_out_size; - SizeVector new_in_size; - SizeVector new_out_size; - - // Now swap it with in/out descriptor vectors - // Now last_in/last_out would contain everything but sizes - last_in_size.swap(std::get<2>(last_in)); - last_out_size.swap(std::get<2>(last_out)); - new_in_size.swap(std::get<2>(new_in)); - new_out_size.swap(std::get<2>(new_out)); - - // If anything (except input sizes) changes, rebuild is required - if (last_in != new_in || last_out != new_out || last_algo != new_algo) { - return Update::REBUILD; - } - - // If output sizes change, graph should be regenerated (resize - // ratio is taken from parameters) - if (last_out_size != new_out_size) { - return Update::REBUILD; - } - - // If interpolation is AREA and sizes change upscale/downscale - // mode, rebuild is required - if (last_algo == RESIZE_AREA) { - // 0123 == NCHW - const auto is_upscale = [](const SizeVector &in, const SizeVector &out) -> bool { - return in[2] < out[2] || in[3] < out[3]; - }; - const bool old_upscale = is_upscale(last_in_size, last_out_size); - const bool new_upscale = is_upscale(new_in_size, new_out_size); - if (old_upscale != new_upscale) { - return Update::REBUILD; - } - } - - // If only sizes changes (considering the above exception), - // reshape is enough - if (last_in_size != new_in_size) { - return Update::RESHAPE; - } - - return Update::NOTHING; -} - -void PreprocEngine::checkApplicabilityGAPI(const Blob::Ptr &src, const Blob::Ptr &dst) { - // Note: src blob is the ROI blob, dst blob is the network's input blob - - if (!src->is()) { - IE_THROW() << "Unsupported input blob type: expected MemoryBlob"; - } - - // dst is always a memory blob - if (!dst->is()) { - IE_THROW() << "Unsupported network's input blob type: expected MemoryBlob"; - } - - const auto &src_dims = src->getTensorDesc().getDims(); - const auto &dst_dims = dst->getTensorDesc().getDims(); - - // dimensions sizes must be equal if both blobs are memory blobs - if (src_dims.size() != dst_dims.size()) { - IE_THROW() << "Preprocessing is not applicable. Source and destination blobs " - "have different number of dimensions."; - } - if (dst_dims.size() != 4) { - IE_THROW() << "Preprocessing is not applicable. Only 4D tensors are supported."; - } - - // dimensions must not have values that are equal to 0 - if (has_zeros(src_dims)) { - IE_THROW() << "Invalid input data dimensions: " << details::dumpVec(src_dims); - } - - if (has_zeros(dst_dims)) { - IE_THROW() << "Invalid network's input dimensions: " << details::dumpVec(dst_dims); - } -} - -int PreprocEngine::getCorrectBatchSize(int batch, const Blob::Ptr& blob) { - if (batch == 0) { - IE_THROW() << "Input pre-processing is called with invalid batch size " << batch; - } - - if (blob->is()) { - // batch size must always be 1 in compound blob case - if (batch > 1) { - IE_THROW() << "Provided input blob batch size " << batch - << " is not supported in compound blob pre-processing"; - } - batch = 1; - } else if (batch < 0) { - // if batch size is unspecified, process the whole input blob - batch = static_cast(blob->getTensorDesc().getDims()[0]); - } - - return batch; -} - -void PreprocEngine::executeGraph(Opt& lastComputation, - const std::vector>& batched_input_plane_mats, - std::vector>& batched_output_plane_mats, int batch_size, bool omp_serial, - Update update) { - - const int thread_num = -#if IE_THREAD == IE_THREAD_OMP - omp_serial ? 1 : // disable threading for OpenMP if was asked for -#endif - 0; // use all available threads - - // to suppress unused warnings - (void)(omp_serial); - - // Split the whole graph into `total_slices` slices, where - // `total_slices` is provided by the parallel runtime and assumed - // to be number of threads used. However it is not guaranteed - // that an actual number of threads will be as assumed, so it - // possible that all slices are processed by the same thread. - // - parallel_nt_static(thread_num, [&, this](int slice_n, const int total_slices) { - OV_ITT_SCOPED_TASK(itt::domains::IEPreproc, _perf_exec_tile); - - auto& compiled = _lastComp[slice_n]; - if (Update::REBUILD == update || Update::RESHAPE == update) { - // need to compile (or reshape) own object for a particular ROI - OV_ITT_SCOPED_TASK(itt::domains::IEPreproc, _perf_graph_compiling); - - using cv::gapi::own::Rect; - - // current design implies all images in batch are equal - const auto& input_plane_mats = batched_input_plane_mats[0]; - const auto& output_plane_mats = batched_output_plane_mats[0]; - - auto lines_per_thread = output_plane_mats[0].rows / total_slices; - const auto remainder = output_plane_mats[0].rows % total_slices; - - // remainder shows how many threads must calculate 1 additional row. now these additions - // must also be addressed in rect's Y coordinate: - int roi_y = 0; - if (slice_n < remainder) { - lines_per_thread++; // 1 additional row - roi_y = slice_n * lines_per_thread; // all previous rois have lines+1 rows - } else { - // remainder rois have lines+1 rows, the rest prior to slice_n have lines rows - roi_y = - remainder * (lines_per_thread + 1) + (slice_n - remainder) * lines_per_thread; - } - - if (lines_per_thread <= 0) return; // no job for current thread - - auto roi = Rect{0, roi_y, output_plane_mats[0].cols, lines_per_thread}; - std::vector rois(output_plane_mats.size(), roi); - - // TODO: make a ROI a runtime argument to avoid - // recompilations - auto args = cv::compile_args(gapi::preprocKernels(), cv::GFluidOutputRois{std::move(rois)}); - if (Update::REBUILD == update) { - auto& computation = lastComputation.value(); - compiled = computation.compile(descrs_of(input_plane_mats), std::move(args)); - } else { - IE_ASSERT(compiled); - compiled.reshape(descrs_of(input_plane_mats), std::move(args)); - } - } - - for (int i = 0; i < batch_size; ++i) { - const auto& input_plane_mats = batched_input_plane_mats[i]; - auto& output_plane_mats = batched_output_plane_mats[i]; - - cv::GRunArgs call_ins; - cv::GRunArgsP call_outs; - for (const auto & m : input_plane_mats) { call_ins.emplace_back(m);} - for (auto & m : output_plane_mats) { call_outs.emplace_back(&m);} - - OV_ITT_SCOPED_TASK(itt::domains::IEPreproc, _perf_exec_graph); - compiled(std::move(call_ins), std::move(call_outs)); - } - }); -} - -template -void PreprocEngine::preprocessBlob(const BlobTypePtr &inBlob, MemoryBlob::Ptr &outBlob, - ResizeAlgorithm algorithm, ColorFormat in_fmt, ColorFormat out_fmt, bool omp_serial, - int batch_size) { - - validateBlob(inBlob); - - auto desc_and_layout = getTensorDescAndLayout(inBlob); - - const auto& in_desc_ie = desc_and_layout.first; - const auto in_layout = desc_and_layout.second; - - const auto& out_desc_ie = outBlob->getTensorDesc(); - validateTensorDesc(in_desc_ie); - validateTensorDesc(out_desc_ie); - - - const auto out_layout = out_desc_ie.getLayout(); - - // For YUV420, check batch via Y plane descriptor - const G::Desc - in_desc = G::decompose(in_desc_ie), - out_desc = G::decompose(out_desc_ie); - - // according to the IE's current design, input blob batch size _must_ match networks's expected - // batch size, even if the actual processing batch size (set on infer request) is different. - if (in_desc.d.N != out_desc.d.N) { - IE_THROW() << "Input blob batch size is invalid: (input blob) " - << in_desc.d.N << " != " << out_desc.d.N << " (expected by network)"; - } - - // sanity check batch size - if (batch_size > out_desc.d.N) { - IE_THROW() << "Provided batch size is invalid: (provided)" - << batch_size << " > " << out_desc.d.N << " (expected by network)"; - } - - CallDesc thisCall = CallDesc{ BlobDesc{ in_desc_ie.getPrecision(), - in_layout, - in_desc_ie.getDims(), - in_fmt }, - BlobDesc{ out_desc_ie.getPrecision(), - out_layout, - out_desc_ie.getDims(), - out_fmt }, - algorithm }; - - if (algorithm == NO_RESIZE && std::get<0>(thisCall) == std::get<1>(thisCall)) { - //if requested output parameters match input blob no need to do anything - IE_THROW() << "No job to do in the PreProcessing ?"; - } - - const Update update = needUpdate(thisCall); - - Opt _lastComputation; - if (Update::REBUILD == update || Update::RESHAPE == update) { - _lastCall = cv::util::make_optional(std::move(thisCall)); - - if (Update::REBUILD == update) { - // rebuild the graph - OV_ITT_SCOPED_TASK(itt::domains::IEPreproc, _perf_graph_building); - // FIXME: what is a correct G::Desc to be passed for NV12/I420 case? - auto custom_desc = getGDesc(in_desc, inBlob); - _lastComputation = cv::util::make_optional( - buildGraph(custom_desc, - out_desc, - in_layout, - out_layout, - algorithm, - in_fmt, - out_fmt)); - } - } - - auto batched_input_plane_mats = bind_to_blob(inBlob, batch_size); - auto batched_output_plane_mats = bind_to_blob(outBlob, batch_size); - - executeGraph(_lastComputation, batched_input_plane_mats, batched_output_plane_mats, batch_size, - omp_serial, update); -} - -void PreprocEngine::preprocessWithGAPI(const Blob::Ptr &inBlob, Blob::Ptr &outBlob, - const ResizeAlgorithm& algorithm, ColorFormat in_fmt, bool omp_serial, int batch_size) { - const auto out_fmt = (in_fmt == ColorFormat::RAW) ? ColorFormat::RAW : ColorFormat::BGR; // FIXME: get expected color format from network - - // output is always a memory blob - auto outMemoryBlob = as(outBlob); - if (!outMemoryBlob) { - IE_THROW() << "Unsupported network's input blob type: expected MemoryBlob"; - } - - auto inMemoryBlob = as(inBlob); - if (!inMemoryBlob) { - IE_THROW() << "Unsupported input blob for color format " << in_fmt - << ": expected MemoryBlob"; - } - return preprocessBlob(inMemoryBlob, outMemoryBlob, algorithm, in_fmt, out_fmt, omp_serial, - batch_size); -} -} // namespace InferenceEngine diff --git a/src/common/preprocessing/src/ie_preprocess_gapi.hpp b/src/common/preprocessing/src/ie_preprocess_gapi.hpp deleted file mode 100644 index fd943e1532a..00000000000 --- a/src/common/preprocessing/src/ie_preprocess_gapi.hpp +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright (C) 2018-2023 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#pragma once - -#include "ie_blob.h" -#include "ie_compound_blob.h" -#include "ie_input_info.hpp" - -#include -#include -#include -#include -#include -#include - -// FIXME: Move this definition back to ie_preprocess_data, -// also free ie_preprocess_gapi of these details - -namespace InferenceEngine { - -IE_SUPPRESS_DEPRECATED_START -class PreprocEngine { - using BlobDesc = std::tuple; - using CallDesc = std::tuple; - template using Opt = cv::util::optional; - - Opt _lastCall; - std::vector _lastComp; - - openvino::itt::handle_t _perf_graph_building = openvino::itt::handle("Preproc Graph Building"); - openvino::itt::handle_t _perf_exec_tile = openvino::itt::handle("Preproc Calc Tile"); - openvino::itt::handle_t _perf_exec_graph = openvino::itt::handle("Preproc Exec Graph"); - openvino::itt::handle_t _perf_graph_compiling = openvino::itt::handle("Preproc Graph compiling"); - - enum class Update { REBUILD, RESHAPE, NOTHING }; - Update needUpdate(const CallDesc &newCall) const; - - void executeGraph(Opt& lastComputation, - const std::vector>& src, - std::vector>& dst, - int batch_size, - bool omp_serial, - Update update); - - template - void preprocessBlob(const BlobTypePtr &inBlob, MemoryBlob::Ptr &outBlob, - ResizeAlgorithm algorithm, ColorFormat in_fmt, ColorFormat out_fmt, bool omp_serial, - int batch_size); - -public: - PreprocEngine(); - static void checkApplicabilityGAPI(const Blob::Ptr &src, const Blob::Ptr &dst); - static int getCorrectBatchSize(int batch_size, const Blob::Ptr& roiBlob); - void preprocessWithGAPI(const Blob::Ptr &inBlob, Blob::Ptr &outBlob, const ResizeAlgorithm &algorithm, - ColorFormat in_fmt, bool omp_serial, int batch_size = -1); -}; -IE_SUPPRESS_DEPRECATED_END - -} // namespace InferenceEngine diff --git a/src/common/preprocessing/src/ie_preprocess_gapi_kernels.cpp b/src/common/preprocessing/src/ie_preprocess_gapi_kernels.cpp deleted file mode 100644 index 8999fd3f061..00000000000 --- a/src/common/preprocessing/src/ie_preprocess_gapi_kernels.cpp +++ /dev/null @@ -1,2079 +0,0 @@ -// Copyright (C) 2018-2023 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#include "ie_preprocess_gapi_kernels.hpp" -#include "ie_preprocess_gapi_kernels_impl.hpp" - -#if CPU_SIMD - #include "ie_system_conf.h" - -#ifdef HAVE_AVX512 - #include "cpu_x86_avx512/ie_preprocess_gapi_kernels_avx512.hpp" -#endif - -#ifdef HAVE_AVX2 - #include "cpu_x86_avx2/ie_preprocess_gapi_kernels_avx2.hpp" -#endif - -#ifdef HAVE_SSE - #include "cpu_x86_sse42/ie_preprocess_gapi_kernels_sse42.hpp" -#endif - -#endif - -#ifdef HAVE_NEON - #include "arm_neon/ie_preprocess_gapi_kernels_neon.hpp" -#endif - -#include -#include -#include - -#include -#include -#include -#include -#include - -#if defined(__GNUC__) && (__GNUC__ <= 5) -#include -#endif - -namespace InferenceEngine { -namespace gapi { -namespace kernels { - -using isas_set = typelist< -#ifdef HAVE_AVX512 - avx512_tag, -#endif -#ifdef HAVE_AVX2 - avx2_tag, -#endif -#ifdef HAVE_SSE - sse42_tag, -#endif -#ifdef HAVE_NEON - neon_tag, -#endif - //scalar "ISA" have to be the last one in the list, - //as the search for supported ISA is performed until first match - scalar_tag>; -#ifdef HAVE_AVX512 - inline bool is_present(avx512_tag) { return with_cpu_x86_avx512f(); } -#endif // HAVE_AVX512 - -#ifdef HAVE_AVX2 - inline bool is_present(avx2_tag) { return with_cpu_x86_avx2(); } -#endif // HAVE_AVX2 - -#ifdef HAVE_SSE - inline bool is_present(sse42_tag) { return with_cpu_x86_sse42(); } -#endif // HAVE_SSE - -#ifdef HAVE_NEON - inline bool is_present(neon_tag) { return true; } -#endif // HAVE_NEON - -//scalar version of kernels is always available -inline bool is_present(scalar_tag) { return true; } - -struct is_isa_present { - template< typename isa_tag_t> - bool operator()(type_to_type) { - return is_present(isa_tag_t{}); - } -}; - -namespace { - -using merge_supported_types = typelist; - -template -inline void mergeRowImpl(scalar_tag, const std::array& ins, T* out, const int length) { - for (int x = 0; x < length; ++x) { - for (int c = 0; c < chs; ++c) { - out[chs * x + c] = ins[c][x]; - } - } -} - -template -struct typed_merge_row { - using p_f = void (*)(const std::array& ins, uint8_t* out, const int length); - - template - inline typename std::enable_if::value || - (!std::is_same::value && - !std::is_same::value && - !std::is_same::value), p_f>::type - operator()(type_to_type ) { - return [](const std::array& ins, uint8_t* out, const int length) { - const auto inT = reinterpret_cast&>(ins); - auto outT = reinterpret_cast(out); - scalar_tag t; - mergeRowImpl(t, inT, outT, length); - }; - } - - template - inline typename std::enable_if::value, p_f>::type - operator()(type_to_type) { - return [](const std::array& ins, uint8_t* out, const int length) { - tag t; - mergeRowImpl(t, ins, out, length); - }; - } - - template - inline typename std::enable_if::value, p_f>::type - operator()(type_to_type) { - return [](const std::array& ins, uint8_t* out, const int length) { - const auto inT = reinterpret_cast&>(ins); - auto outT = reinterpret_cast(out); - tag t; - mergeRowImpl(t, inT, outT, length); - }; - } -}; - -} // namespace - -namespace { -using split_supported_types = typelist; - -template -inline void splitRowImpl(scalar_tag, const T* in, std::array& outs, const int length) { - for (int x = 0; x < length; ++x) { - for (int c = 0; c < chs; ++c) { - outs[c][x] = in[chs * x + c]; - } - } -} - -template -struct typed_split_row { - using p_f = void (*)(const uint8_t* in, std::array& outs, const int length); - - template - inline typename std::enable_if::value || - (!std::is_same::value && - !std::is_same::value && - !std::is_same::value), p_f>::type - operator()(type_to_type ) { - return [](const uint8_t* in, std::array& outs, const int length) { - const auto inT = reinterpret_cast(in); - auto outT = reinterpret_cast&>(outs); - scalar_tag t; - splitRowImpl(t, inT, outT, length); - }; - } - - template - inline typename std::enable_if::value, p_f>::type - operator()(type_to_type) { - return [](const uint8_t* in, std::array& outs, const int length) { - tag t; - splitRowImpl(t, in, outs, length); - }; - } - - template - inline typename std::enable_if::value, p_f>::type - operator()(type_to_type) { - return [](const uint8_t* in, std::array& outs, const int length) { - const auto inT = reinterpret_cast(in); - auto outT = reinterpret_cast&>(outs); - tag t; - splitRowImpl(t, inT, outT, length); - }; - } -}; -} // namespace - -// GAPI_OCV_KERNEL(OCVChanToPlane, ChanToPlane) { -// static void run(const cv::Mat &in, int chan, cv::Mat &out) { -// out.create(in.rows, in.cols, in.depth()); -// const auto rowFunc = (in.depth() == CV_8U) ? &chanToPlaneRow : &chanToPlaneRow; - -// for (int y = 0; y < out.rows; y++) -// { -// rowFunc(in.data + y*in.step, chan, in.channels(), out.data + y*out.step, in.cols); -// } -// } -// }; - -// GAPI_OCV_KERNEL(OCVScalePlane, ScalePlane) { -// static void run(const cv::Mat &in, int /*type*/, const Size &sz, int interp, cv::Mat &out) { -// cv::resize(in, out, sz, 0, 0, interp); -// } -// }; - -// GAPI_OCV_KERNEL(OCVMerge2, Merge2) { -// static void run(const cv::Mat &a, const cv::Mat &b, cv::Mat out) { -// out.create(a.rows, a.cols, CV_MAKETYPE(a.depth(), 2)); -// const auto rowFunc = (a.depth() == CV_8U) ? &mergeRow : &mergeRow; - -// for (int y = 0; y < out.rows; y++) -// { -// rowFunc({a.data + y*a.step, b.data + y*b.step}, out.data + out.step, a.cols); -// } -// } -// }; - -namespace { - -using chan_to_plane_supported_types = typelist; - -template -inline void chanToPlaneRowImpl(scalar_tag, const T* in, int chan, int chs, T* out, int length) { - for (int x = 0; x < length; x++) { - out[x] = in[x*chs + chan]; - } -} - -template -struct typed_chan_to_plane_row { - using p_f = void (*)(const uint8_t* in, int chan, int chs, uint8_t* out, int length); - - template - p_f operator()(type_to_type ) { - return [](const uint8_t* in, int chan, int chs, uint8_t* out, int length){ - const auto inT = reinterpret_cast(in); - auto outT = reinterpret_cast< type*>(out); - - chanToPlaneRowImpl(isa_tag_t{}, inT, chan, chs, outT, length); - }; - } -}; -} //namespace - -namespace { - -using nv12_to_rgb_supported_types = typelist; - -inline void nv12ToRgbRowImpl(scalar_tag, const uint8_t** y_rows, const uint8_t* uv_row, - uint8_t** out_rows, const int buf_width) { - for (int i = 0; i < buf_width; i += 2) { - uint8_t u = uv_row[i]; - uint8_t v = uv_row[i + 1]; - int ruv, guv, buv; - uvToRGBuv(u, v, ruv, guv, buv); - - for (int y = 0; y < 2; y++) { - for (int x = 0; x < 2; x++) { - uint8_t vy = y_rows[y][i + x]; - uint8_t r, g, b; - yRGBuvToRGB(vy, ruv, guv, buv, r, g, b); - - out_rows[y][3 * (i + x)] = r; - out_rows[y][3 * (i + x) + 1] = g; - out_rows[y][3 * (i + x) + 2] = b; - } - } - } -} - -template -struct typed_nv12_to_rgb_row { - using p_f = void (*)(const uint8_t** y_rows, const uint8_t* uv_row, - uint8_t** out_rows, const int buf_width); - - template - p_f operator()(type_to_type) { - return [](const uint8_t** y_rows, const uint8_t* uv_row, - uint8_t** out_rows, const int buf_width) { - const auto inT1 = reinterpret_cast(y_rows); - const auto inT2 = reinterpret_cast(uv_row); - auto outT = reinterpret_cast(out_rows); - - nv12ToRgbRowImpl(isa_tag_t{}, inT1, inT2, outT, buf_width); - }; - } -}; -} // namespace - -namespace { - -using i420_to_rgb_supported_types = typelist; - -inline void i420ToRgbRowImpl(scalar_tag, const uint8_t** y_rows, - const uint8_t* u_row, - const uint8_t* v_row, - uint8_t** out_rows, - const int buf_width) { - for (int i = 0; i < buf_width; i += 2) { - uchar u = u_row[i / 2]; - uchar v = v_row[i / 2]; - int ruv, guv, buv; - uvToRGBuv(u, v, ruv, guv, buv); - - for (int y = 0; y < 2; y++) { - for (int x = 0; x < 2; x++) { - uchar vy = y_rows[y][i + x]; - uchar r, g, b; - yRGBuvToRGB(vy, ruv, guv, buv, r, g, b); - - out_rows[y][3 * (i + x)] = r; - out_rows[y][3 * (i + x) + 1] = g; - out_rows[y][3 * (i + x) + 2] = b; - } - } - } -} - -template -struct typed_i420_to_rgb_row { - using p_f = void (*)(const uint8_t** y_rows, const uint8_t* u_row, const uint8_t* v_row, - uint8_t** out_rows, const int buf_width); - - template - p_f operator()(type_to_type) { - return [](const uint8_t** y_rows, const uint8_t* u_row, const uint8_t* v_row, - uint8_t** out_rows, const int buf_width) { - const auto inT1 = reinterpret_cast(y_rows); - const auto inT2 = reinterpret_cast(u_row); - const auto inT3 = reinterpret_cast(v_row); - auto outT = reinterpret_cast(out_rows); - - i420ToRgbRowImpl(isa_tag_t{}, inT1, inT2, inT3, outT, buf_width); - }; - } -}; -} // namespace - -namespace linear { -struct Mapper { - typedef short alpha_type; - typedef short index_type; - constexpr static const int unity = ONE; - - typedef MapperUnit Unit; - - static inline Unit map(double ratio, int start, int max, int outCoord) { - float f = static_cast((outCoord + 0.5) * ratio - 0.5); - int s = cvFloor(f); - f -= s; - - Unit u; - - u.index0 = std::max(s - start, 0); - u.index1 = ((f == 0.0) || s + 1 >= max) ? s - start : s - start + 1; - - u.alpha0 = saturate_cast(ONE * (1.0f - f)); - u.alpha1 = saturate_cast(ONE * f); - - return u; - } -}; -} // namespace linear - -namespace linear32f { -struct Mapper { - typedef float alpha_type; - typedef int index_type; - constexpr static const float unity = 1; - - typedef MapperUnit Unit; - - static inline Unit map(double ratio, int start, int max, int outCoord) { - float f = static_cast((outCoord + 0.5) * ratio - 0.5); - int s = cvFloor(f); - f -= s; - - Unit u; - - u.index0 = std::max(s - start, 0); - u.index1 = ((f == 0.0) || s + 1 >= max) ? s - start : s - start + 1; - - u.alpha0 = 1.f - f; - u.alpha1 = f; - - return u; - } -}; -} // namespace linear32f - -template -struct linearScratchDesc { - using alpha_t = typename Mapper::alpha_type; - using index_t = typename Mapper::index_type; - - alpha_t* alpha; - alpha_t* clone; - index_t* mapsx; - alpha_t* beta; - index_t* mapsy; - T* tmp; - - linearScratchDesc(int /*inW*/, int /*inH*/, int outW, int outH, void* data) { - alpha = reinterpret_cast(data); - clone = reinterpret_cast(alpha + outW); - mapsx = reinterpret_cast(clone + outW*4); - beta = reinterpret_cast(mapsx + outW); - mapsy = reinterpret_cast(beta + outH); - tmp = reinterpret_cast (mapsy + outH*2); - } - - static int bufSize(int inW, int /*inH*/, int outW, int outH, int lpi) { - auto size = outW * sizeof(alpha_t) + - outW * sizeof(alpha_t) * 4 + // alpha clones // previous alpha is redundant? - outW * sizeof(index_t) + - outH * sizeof(alpha_t) + - outH * sizeof(index_t) * 2 + - inW * sizeof(T) * lpi * chanNum; - - return static_cast(size); - } -}; - -static inline double invRatio(int inSz, int outSz) { - return static_cast(outSz) / inSz; -} - -static inline double ratio(int inSz, int outSz) { - return 1 / invRatio(inSz, outSz); -} - -template -static inline void initScratchLinear(const cv::GMatDesc& in, - const Size& outSz, - cv::gapi::fluid::Buffer& scratch, - int lpi) { - using alpha_type = typename Mapper::alpha_type; - static const auto unity = Mapper::unity; - - auto inSz = in.size; - auto sbufsize = linearScratchDesc::bufSize(inSz.width, inSz.height, outSz.width, outSz.height, lpi); - - Size scratch_size{sbufsize, 1}; - - cv::GMatDesc desc; - desc.chan = 1; - desc.depth = CV_8UC1; - desc.size = scratch_size; - - cv::gapi::fluid::Buffer buffer(desc); - scratch = std::move(buffer); - - double hRatio = ratio(in.size.width, outSz.width); - double vRatio = ratio(in.size.height, outSz.height); - - linearScratchDesc scr(inSz.width, inSz.height, outSz.width, outSz.height, scratch.OutLineB()); - - auto *alpha = scr.alpha; - auto *clone = scr.clone; - auto *index = scr.mapsx; - - for (int x = 0; x < outSz.width; x++) { - auto map = Mapper::map(hRatio, 0, in.size.width, x); - auto alpha0 = map.alpha0; - auto index0 = map.index0; - - // TRICK: - // Algorithm takes pair of input pixels, sx0'th and sx1'th, - // and compute result as alpha0*src[sx0] + alpha1*src[sx1]. - // By definition: sx1 == sx0 + 1 either sx1 == sx0, and - // alpha0 + alpha1 == unity (scaled appropriately). - // Here we modify formulas for alpha0 and sx1: by assuming - // that sx1 == sx0 + 1 always, and patching alpha0 so that - // result remains intact. - // Note that we need in.size.width >= 2, for both sx0 and - // sx0+1 were indexing pixels inside the input's width. - if (map.index1 != map.index0 + 1) { - GAPI_DbgAssert(map.index1 == map.index0); - GAPI_DbgAssert(in.size.width >= 2); - if (map.index0 < in.size.width-1) { - // sx1=sx0+1 fits inside row, - // make sure alpha0=unity and alpha1=0, - // so that result equals src[sx0]*unity - alpha0 = saturate_cast(unity); - } else { - // shift sx0 to left by 1 pixel, - // and make sure that alpha0=0 and alpha1==1, - // so that result equals to src[sx0+1]*unity - alpha0 = 0; - index0--; - } - } - - alpha[x] = alpha0; - index[x] = index0; - - for (int l = 0; l < 4; l++) { - clone[4*x + l] = alpha0; - } - } - - auto *beta = scr.beta; - auto *index_y = scr.mapsy; - - for (int y = 0; y < outSz.height; y++) { - auto mapY = Mapper::map(vRatio, 0, in.size.height, y); - beta[y] = mapY.alpha0; - index_y[y] = mapY.index0; - index_y[outSz.height + y] = mapY.index1; - } -} - -template -inline void calcRowLinearC1Impl(T* dst[], - const T* src0[], - const T* src1[], - const AT alpha[], - const IT mapsx[], - const AT beta[], - const Size& inSz, - const Size& outSz, - const int lpi, - const int length) { - using alpha_type = typename Mapper::alpha_type; - for (int l = 0; l < lpi; l++) { - constexpr static const auto unity = Mapper::unity; - - auto beta0 = beta[l]; - auto beta1 = saturate_cast(unity - beta[l]); - - for (int x = 0; x < length; x++) { - auto alpha0 = alpha[x]; - auto alpha1 = saturate_cast(unity - alpha[x]); - auto sx0 = mapsx[x]; - auto sx1 = sx0 + 1; - T tmp0 = calc(beta0, src0[l][sx0], beta1, src1[l][sx0]); - T tmp1 = calc(beta0, src0[l][sx1], beta1, src1[l][sx1]); - dst[l][x] = calc(alpha0, tmp0, alpha1, tmp1); - } - } -} - -namespace { - -using resizeLinearU8C1_suptypes = typelist; - -template -inline void calcRowLinear8UC1Impl(scalar_tag, uint8_t* dst[], - const uint8_t* src0[], - const uint8_t* src1[], - const short alpha[], - const short clone[], // 4 clones of alpha - const short mapsx[], - const short beta[], - uint8_t tmp[], - const Size& inSz, - const Size& outSz, - const int lpi, - const int length) { - calcRowLinearC1Impl(dst, src0, src1, alpha, mapsx, - beta, inSz, outSz, lpi, length); -} - -template -struct typed_resizeLinearU8C1 { - using p_f = void (*)(uint8_t* dst[], const uint8_t* src0[], const uint8_t* src1[], - const short alpha[], const short clone[], const short mapsx[], - const short beta[], uint8_t tmp[], const Size& inSz, - const Size& outSz, const int lpi, const int length); - - template - inline typename std::enable_if::value, p_f>::type - operator()(type_to_type) { - return [](uint8_t* dst[], const uint8_t* src0[], const uint8_t* src1[], - const short alpha[], const short clone[], const short mapsx[], - const short beta[], uint8_t tmp[], const Size& inSz, - const Size& outSz, const int lpi, const int length) { - if (!calcRowLinear8UC1Impl(isa_tag_t{}, dst, src0, - src1, alpha, clone, - mapsx, beta, tmp, - inSz, outSz, lpi, length)) - calcRowLinear8UC1Impl(scalar_tag{}, dst, src0, src1, alpha, clone, - mapsx, beta, tmp, inSz, outSz, lpi, length); - }; - } - - template - inline typename std::enable_if::value, p_f>::type - operator()(type_to_type) { - return [](uint8_t* dst[], const uint8_t* src0[], const uint8_t* src1[], - const short alpha[], const short clone[], const short mapsx[], - const short beta[], uint8_t tmp[], const Size& inSz, - const Size& outSz, const int lpi, const int length) { - calcRowLinear8UC1Impl(isa_tag_t{}, dst, src0, src1, alpha, clone, - mapsx, beta, tmp, inSz, outSz, lpi, length); - }; - } -}; -} // namespace - -namespace { - -using resizeLinearF32C1_suptypes = typelist; - -template -inline void calcRowLinear32FC1Impl(scalar_tag, float* dst[], - const float* src0[], - const float* src1[], - const float alpha[], - const int mapsx[], - const float beta[], - const Size& inSz, - const Size& outSz, - const int lpi, - const int length) { - calcRowLinearC1Impl(dst, src0, src1, alpha, mapsx, - beta, inSz, outSz, lpi, length); -} - -template -struct typed_resizeLinearF32C1 { - using p_f = void (*)(float* dst[], const float* src0[], const float* src1[], - const float alpha[], const int mapsx[], - const float beta[], const Size& inSz, - const Size& outSz, const int lpi, const int length); - - template - inline typename std::enable_if::value, p_f>::type - operator()(type_to_type) { - return [](float* dst[], const float* src0[], const float* src1[], - const float alpha[], const int mapsx[], - const float beta[], const Size& inSz, - const Size& outSz, const int lpi, const int length) { - calcRowLinear32FC1Impl(isa_tag_t{}, dst, src0, src1, alpha, - mapsx, beta, inSz, outSz, lpi, length); - }; - } - - template - inline typename std::enable_if::value, p_f>::type - operator()(type_to_type) { - return [](float* dst[], const float* src0[], const float* src1[], - const float alpha[], const int mapsx[], - const float beta[], const Size& inSz, - const Size& outSz, const int lpi, const int length) { - calcRowLinear32FC1Impl(isa_tag_t{}, dst, src0, src1, alpha, - mapsx, beta, inSz, outSz, lpi, length); - }; - } -}; -} // namespace - -namespace { - -using resizeLinearU8C3C4_suptypes = typelist; - -template -inline void calcRowLinear8UC3C4Impl(scalar_tag, - std::array, chs>& dst, - const uint8_t* src0[], - const uint8_t* src1[], - const short alpha[], - const short clone[], // 4 clones of alpha - const short mapsx[], - const short beta[], - uint8_t tmp[], - const Size& inSz, - const Size& outSz, - const int lpi, - const int length) { - using alpha_type = typename Mapper::alpha_type; - for (int l = 0; l < lpi; l++) { - constexpr static const auto unity = Mapper::unity; - - auto beta0 = beta[l]; - auto beta1 = saturate_cast(unity - beta[l]); - - for (int x = 0; x < length; x++) { - auto alpha0 = alpha[x]; - auto alpha1 = saturate_cast(unity - alpha[x]); - auto sx0 = mapsx[x]; - auto sx1 = sx0 + 1; - - for (int c = 0; c < chs; c++) { - auto idx0 = chs * sx0 + c; - auto idx1 = chs * sx1 + c; - uint8_t tmp0 = calc(beta0, src0[l][idx0], beta1, src1[l][idx0]); - uint8_t tmp1 = calc(beta0, src0[l][idx1], beta1, src1[l][idx1]); - dst[c][l][x] = calc(alpha0, tmp0, alpha1, tmp1); - } - } - } -} - -template -struct typed_resizeLinearU8C3C4 { - using p_f = void (*)(std::array, chs>& dst, const uint8_t* src0[], const uint8_t* src1[], - const short alpha[], const short clone[], const short mapsx[], - const short beta[], uint8_t tmp[], const Size& inSz, - const Size& outSz, const int lpi, const int length); - - template - inline typename std::enable_if::value, p_f>::type - operator()(type_to_type) { - return [](std::array, chs>& dst, const uint8_t* src0[], const uint8_t* src1[], - const short alpha[], const short clone[], const short mapsx[], - const short beta[], uint8_t tmp[], const Size& inSz, - const Size& outSz, const int lpi, const int length) { - if (!calcRowLinear8UC3C4Impl(isa_tag_t{}, dst, src0, - src1, alpha, clone, - mapsx, beta, tmp, - inSz, outSz, lpi, length)) - calcRowLinear8UC3C4Impl(scalar_tag{}, dst, src0, src1, alpha, clone, - mapsx, beta, tmp, inSz, outSz, lpi, length); - }; - } - - template - inline typename std::enable_if::value, p_f>::type - operator()(type_to_type) { - return [](std::array, chs>& dst, const uint8_t* src0[], - const uint8_t* src1[], const short alpha[], const short clone[], - const short mapsx[], const short beta[], uint8_t tmp[], const Size& inSz, - const Size& outSz, const int lpi, const int length) { - calcRowLinear8UC3C4Impl(isa_tag_t{}, dst, src0, src1, alpha, clone, - mapsx, beta, tmp, inSz, outSz, lpi, length); - }; - } -}; -} // namespace - -template -struct AreaDownMapper { - typedef A alpha_type; - typedef I index_type; - typedef W work_type; - - typedef MapperUnit Unit; - - inline Unit map(int outCoord) { - double inCoord0 = outCoord * ratio; - double inCoord1 = (outCoord + 1) * ratio; - - double index0 = std::floor(inCoord0 + 0.001); - double index1 = std::ceil(inCoord1 - 0.001); - - double alpha0 = (index0 + 1 - inCoord0) * inv_ratio; - double alpha1 = - (index1 - 1 - inCoord1) * inv_ratio; - - GAPI_Assert((0 <= outCoord) && (outCoord <= outSz-1)); - GAPI_Assert((0 <= index0) && (index0 < index1) && (index1 <= inSz)); - - Unit unit; - - unit.index0 = checked_cast(index0); - unit.index1 = checked_cast(index1); - - unit.alpha0 = convert_cast(alpha0); - unit.alpha1 = convert_cast(alpha1); - - return unit; - } - - int inSz, outSz; - double ratio, inv_ratio; - - alpha_type alpha; // == inv_ratio, rounded - - AreaDownMapper(int _inSz, int _outSz) { - inSz = _inSz; - outSz = _outSz; - - inv_ratio = invRatio(inSz, outSz); - ratio = 1.0 / inv_ratio; - - alpha = convert_cast(inv_ratio); - } -}; - -namespace areaDownscale32f { -struct Mapper: public AreaDownMapper { - Mapper(int _inSz, int _outSz):AreaDownMapper(_inSz, _outSz) {} -}; -} - -namespace areaDownscale8u { -struct Mapper: public AreaDownMapper { - Mapper(int _inSz, int _outSz):AreaDownMapper(_inSz, _outSz) {} -}; -} - -namespace areaUpscale { -struct Mapper { - typedef short alpha_type; - typedef short index_type; - constexpr static const int unity = ONE; - - typedef MapperUnit Unit; - - static inline Unit map(double ratio, int start, int max, int outCoord) { - const int s = cvFloor(outCoord*ratio); - float res = static_cast((outCoord + 1) - (s + 1)/ratio); - res = res <= 0 ? 0.f : res - cvFloor(res); - - Unit u; - - u.index0 = std::max(s - start, 0); - u.index1 = ((res == 0.0) || (s + 1 >= max)) ? s - start : s - start + 1; - - u.alpha0 = saturate_cast(ONE * (1.0f - res)); - u.alpha1 = saturate_cast(ONE * res); - - return u; - } -}; -} // namespace areaUpscale - -namespace areaUpscale32f { -struct Mapper { - typedef float alpha_type; - typedef int index_type; - constexpr static const float unity = 1; - - typedef MapperUnit Unit; - - static inline Unit map(double ratio, int start, int max, int outCoord) { - int s = cvFloor(outCoord*ratio); - float f = static_cast((outCoord+1) - (s+1)/ratio); - f = f <= 0 ? 0.f : f - cvFloor(f); - - Unit u; - - u.index0 = std::max(s - start, 0); - u.index1 = ((f == 0.0) || s + 1 >= max) ? s - start : s - start + 1; - - u.alpha0 = 1.0f - f; - u.alpha1 = f; - - return u; - } -}; -} // namespace areaUpscale32f - -template -static void initScratchArea(const cv::GMatDesc& in, const Size& outSz, - cv::gapi::fluid::Buffer &scratch) { - using Unit = typename Mapper::Unit; - using alpha_type = typename Mapper::alpha_type; - using index_type = typename Mapper::index_type; - - // compute the chunk of input pixels for each output pixel, - // along with the coefficients for taking the weigthed sum - - Size inSz = in.size; - Mapper mapper(inSz.width, outSz.width); - - std::vector xmaps(outSz.width); - int maxdif = 0; - - for (int w = 0; w < outSz.width; w++) { - Unit map = mapper.map(w); - xmaps[w] = map; - - maxdif = std::max(maxdif, map.index1 - map.index0); - } - - // This assertion is critical for our trick with chunk sizes: - // we would expand a chunk it is smaller than maximal size - GAPI_Assert(inSz.width >= maxdif); - - // pack the input chunks positions and coefficients into scratch-buffer, - // along with the maximal size of chunk (note that chunk size may vary) - - size_t scratch_bytes = sizeof(int) - + outSz.width * sizeof(index_type) - + outSz.width * sizeof(alpha_type) * maxdif - + inSz.width * sizeof(alpha_type); - Size scratch_size{static_cast(scratch_bytes), 1}; - - cv::GMatDesc desc(CV_8UC1, 1, scratch_size); - - cv::gapi::fluid::Buffer buffer(desc); - scratch = std::move(buffer); - - auto *maxdf = scratch.OutLine(); - auto *index = reinterpret_cast(maxdf + 1); - auto *alpha = reinterpret_cast(index + outSz.width); - - for (int w = 0; w < outSz.width; w++) { - // adjust input indices so that: - // - data chunk is exactly maxdif pixels - // - data chunk fits inside input width - int index0 = xmaps[w].index0; - int index1 = xmaps[w].index1; - int i0 = index0; - int i1 = index1; - i1 = std::min(i0 + maxdif, in.size.width); - i0 = i1 - maxdif; - GAPI_DbgAssert(i0 >= 0); - - // fulfill coefficients for the data chunk, - // extending with zeros if any extra pixels - alpha_type *alphaw = &alpha[w * maxdif]; - for (int i = 0; i < maxdif; i++) { - if (i + i0 == index0) { - alphaw[i] = xmaps[w].alpha0; - - } else if (i + i0 == index1 - 1) { - alphaw[i] = xmaps[w].alpha1; - - } else if (i + i0 > index0 && i + i0 < index1 - 1) { - alphaw[i] = mapper.alpha; - - } else { - alphaw[i] = 0; - } - } - - // start input chunk with adjusted position - index[w] = i0; - } - - *maxdf = maxdif; -} - -#if USE_CVKL -static int getResizeAreaTabSize(int dst_go, int ssize, int dsize, float scale) { - static const float threshold = 1e-3f; - int max_count = 0; - - for (int col = dst_go; col < dst_go + dsize; col++) { - int count = 0; - - float fsx1 = col * scale; - float fsx2 = fsx1 + scale; - - int sx1 = static_cast(ceil(fsx1)); - int sx2 = static_cast(floor(fsx2)); - - sx2 = std::min(sx2, ssize - 1); - sx1 = std::min(sx1, sx2); - - if ((sx1 - fsx1) > threshold) { - count++; - } - - for (int sx = sx1; sx < sx2; sx++) { - count++; - } - - if ((fsx2 - sx2) > threshold) { - count++; - } - max_count = std::max(max_count, count); - } - - return max_count; -} - -// taken from: ie_preprocess_data.cpp -static void computeResizeAreaTab(int src_go, int dst_go, int ssize, int dsize, float scale, - uint16_t* si, uint16_t* alpha, int max_count) { - static const float threshold = 1e-3f; - int k = 0; - - for (int col = dst_go; col < dst_go + dsize; col++) { - int count = 0; - - float fsx1 = col * scale; - float fsx2 = fsx1 + scale; - float cellWidth = (std::min)(scale, ssize - fsx1); - - int sx1 = static_cast(ceil(fsx1)); - int sx2 = static_cast(floor(fsx2)); - - sx2 = (std::min)(sx2, ssize - 1); - sx1 = (std::min)(sx1, sx2); - - si[col - dst_go] = (uint16_t)(sx1 - src_go); - - if (sx1 - fsx1 > threshold) { - si[col - dst_go] = (uint16_t)(sx1 - src_go - 1); - alpha[k++] = (uint16_t)((1 << 16) * ((sx1 - fsx1) / cellWidth)); - count++; - } - - for (int sx = sx1; sx < sx2; sx++) { - alpha[k++] = (uint16_t)((1 << 16) * (1.0f / cellWidth)); - count++; - } - - if (fsx2 - sx2 > threshold) { - alpha[k++] = (uint16_t)((1 << 16) * ((std::min)((std::min)(fsx2 - sx2, 1.f), cellWidth) / cellWidth)); - count++; - } - - if (count != max_count) { - alpha[k++] = 0; - } - } -} - -// teken from: ie_preprocess_data.cpp -static void generate_alpha_and_id_arrays(int x_max_count, int dcols, const uint16_t* xalpha, uint16_t* xsi, - uint16_t** alpha, uint16_t** sxid) { - if (x_max_count <= 4) { - for (int col = 0; col < dcols; col++) { - for (int x = 0; x < x_max_count; x++) { - alpha[x][col] = xalpha[col*x_max_count + x]; - } - } - } - if (x_max_count <= 4) { - for (int col = 0; col <= dcols - 8; col += 8) { - for (int chunk_num_h = 0; chunk_num_h < x_max_count; chunk_num_h++) { - for (int i = 0; i < 128 / 16; i++) { - int id_diff = xsi[col + i] - xsi[col]; - - for (int chunk_num_v = 0; chunk_num_v < x_max_count; chunk_num_v++) { - uint16_t* sxidp = sxid[chunk_num_v] + col * x_max_count + chunk_num_h * 8; - - int id0 = (id_diff + chunk_num_v) * 2 + 0; - int id1 = (id_diff + chunk_num_v) * 2 + 1; - - (reinterpret_cast(sxidp + i))[0] = static_cast(id0 >= (chunk_num_h * 16) && id0 < (chunk_num_h + 1) * 16 ? id0 : -1); - (reinterpret_cast(sxidp + i))[1] = static_cast(id1 >= (chunk_num_h * 16) && id1 < (chunk_num_h + 1) * 16 ? id1 : -1); - } - } - } - } - } -} - -// taken from: ie_preprocess_data.cpp -// (and simplified for specifically downscale area 8u) -static size_t resize_get_buffer_size(const Size& inSz, const Size& outSz) { - int dst_full_width = outSz.width; - int dst_full_height = outSz.height; - int src_full_width = inSz.width; - int src_full_height = inSz.height; - - auto resize_area_u8_downscale_sse_buffer_size = [&]() { - const int dwidth = outSz.width; - const int dheight = outSz.height; - const int swidth = inSz.width; - - const int dst_go_x = 0; - const int dst_go_y = 0; - - int x_max_count = getResizeAreaTabSize(dst_go_x, src_full_width, dwidth, static_cast(src_full_width) / dst_full_width) + 1; - int y_max_count = getResizeAreaTabSize(dst_go_y, src_full_height, dheight, static_cast(src_full_height) / dst_full_height) + 1; - - size_t si_buf_size = sizeof(uint16_t) * dwidth + sizeof(uint16_t) * dheight; - size_t alpha_buf_size = - sizeof(uint16_t) * (dwidth * x_max_count + 8 * 16) + sizeof(uint16_t) * dheight * y_max_count; - size_t vert_sum_buf_size = sizeof(uint16_t) * (swidth * 2); - size_t alpha_array_buf_size = sizeof(uint16_t) * 4 * dwidth; - size_t sxid_array_buf_size = sizeof(uint16_t) * 4 * 4 * dwidth; - - size_t buffer_size = si_buf_size + - alpha_buf_size + - vert_sum_buf_size + - alpha_array_buf_size + - sxid_array_buf_size; - - return buffer_size; - }; - - return resize_area_u8_downscale_sse_buffer_size(); -} - -// buffer-fulfill is taken from: ie_preprocess_data_sse42.cpp -static void initScratchArea_CVKL_U8(const cv::GMatDesc & in, - const Size & outSz, - cv::gapi::fluid::Buffer & scratch) { - const Size& inSz = in.size; - - // estimate buffer size - size_t scratch_bytes = resize_get_buffer_size(inSz, outSz); - - // allocate buffer - - Size scratch_size{static_cast(scratch_bytes), 1}; - - cv::GMatDesc desc; - desc.chan = 1; - desc.depth = CV_8UC1; - desc.size = scratch_size; - - cv::gapi::fluid::Buffer buffer(desc); - scratch = std::move(buffer); - - // fulfil buffer - { - // this code is taken from: ie_preprocess_data_sse42.cpp - // (and simplified for 1-channel cv::Mat instead of blob) - - auto dwidth = outSz.width; - auto dheight = outSz.height; - auto swidth = inSz.width; - auto sheight = inSz.height; - - const int src_go_x = 0; - const int src_go_y = 0; - const int dst_go_x = 0; - const int dst_go_y = 0; - - auto src_full_width = swidth; - auto src_full_height = sheight; - auto dst_full_width = dwidth; - auto dst_full_height = dheight; - - float scale_x = static_cast(src_full_width) / dst_full_width; - float scale_y = static_cast(src_full_height) / dst_full_height; - - int x_max_count = getResizeAreaTabSize(dst_go_x, src_full_width, dwidth, scale_x); - int y_max_count = getResizeAreaTabSize(dst_go_y, src_full_height, dheight, scale_y); - - auto* maxdif = scratch.OutLine(); - auto* xsi = reinterpret_cast(maxdif + 2); - auto* ysi = xsi + dwidth; - auto* xalpha = ysi + dheight; - auto* yalpha = xalpha + dwidth*x_max_count + 8*16; - - maxdif[0] = x_max_count; - maxdif[1] = y_max_count; - - computeResizeAreaTab(src_go_x, dst_go_x, src_full_width, dwidth, scale_x, xsi, xalpha, x_max_count); - computeResizeAreaTab(src_go_y, dst_go_y, src_full_height, dheight, scale_y, ysi, yalpha, y_max_count); - - int vest_sum_size = 2*swidth; - uint16_t* vert_sum = yalpha + dheight*y_max_count; - uint16_t* alpha0 = vert_sum + vest_sum_size; - uint16_t* alpha1 = alpha0 + dwidth; - uint16_t* alpha2 = alpha1 + dwidth; - uint16_t* alpha3 = alpha2 + dwidth; - uint16_t* sxid0 = alpha3 + dwidth; - uint16_t* sxid1 = sxid0 + 4*dwidth; - uint16_t* sxid2 = sxid1 + 4*dwidth; - uint16_t* sxid3 = sxid2 + 4*dwidth; - - uint16_t* alpha[] = {alpha0, alpha1, alpha2, alpha3}; - uint16_t* sxid[] = {sxid0, sxid1, sxid2, sxid3}; - generate_alpha_and_id_arrays(x_max_count, dwidth, xalpha, xsi, alpha, sxid); - } -} - -static void calcAreaRow_CVKL_U8(const cv::gapi::fluid::View & in, - cv::gapi::fluid::Buffer & out, - cv::gapi::fluid::Buffer & scratch) { - Size inSz = in.meta().size; - Size outSz = out.meta().size; - - // this method is valid only for down-scale - GAPI_DbgAssert(inSz.width >= outSz.width); - GAPI_DbgAssert(inSz.height >= outSz.height); - - int dwidth = outSz.width; - int dheight = outSz.height; - - auto* maxdif = scratch.OutLine(); - int x_max_count = maxdif[0]; - int y_max_count = maxdif[1]; - - auto* xsi = reinterpret_cast(maxdif + 2); - auto* ysi = xsi + dwidth; - auto* xalpha = ysi + dheight; - auto* yalpha = xalpha + dwidth*x_max_count + 8*16; - auto* vert_sum = yalpha + dheight*y_max_count; - - int iny = in.y(); - int y = out.y(); - - int lpi = out.lpi(); - GAPI_DbgAssert(y + lpi <= outSz.height); - - for (int l = 0; l < lpi; l++) { - int yin0 = ysi[y + l]; - int yin1 = yin0 + y_max_count; - - GAPI_Assert(yin1 - yin0 <= 32); - const uint8_t *src[32] = {}; - - for (int yin = yin0; yin < yin1 && yin < inSz.height; yin++) { - if (yalpha[(y+l)*y_max_count + yin - yin0] == 0) { - src[yin - yin0] = in.InLine(yin - iny - 1); - } else { - src[yin - yin0] = in.InLine(yin - iny); - } - } - - uint8_t *dst = out.OutLine(l); - - calcRowArea_CVKL_U8_SSE42(src, dst, inSz, outSz, y + l, xsi, ysi, - xalpha, yalpha, x_max_count, y_max_count, vert_sum); - } -} - -#endif // USE_CVKL - -namespace { - -using resizeArea_suptypes = typelist; - -template -inline void calcRowAreaImpl(scalar_tag, - T dst[], const T* src[], const Size& inSz, - const Size& outSz, A yalpha, - const MapperUnit& ymap, int xmaxdf, - const I xindex[], const A xalpha[], - W vbuf[]) { - // vertical pass - int y_1st = ymap.index0; - int ylast = ymap.index1 - 1; - if (y_1st < ylast) { - for (int w = 0; w < inSz.width; w++) { - vbuf[w] = mulas(ymap.alpha0, src[0][w]) // Q8_8 = Q0_16 * U8 - + mulas(ymap.alpha1, src[ylast - y_1st][w]); - } - - for (int i = 1; i < ylast - y_1st; i++) { - for (int w = 0; w < inSz.width; w++) { - vbuf[w] += mulas(yalpha, src[i][w]); - } - } - } else { - for (int w = 0; w < inSz.width; w++) { - vbuf[w] = convert_cast(src[0][w]); // Q8_8 = U8 - } - } - - // horizontal pass - for (int x = 0; x < outSz.width; x++) { - W sum = 0; - - auto index = xindex[x]; - const auto *alpha = &xalpha[x * xmaxdf]; - - for (int i = 0; i < xmaxdf; i++) { - sum += mulaw(alpha[i], vbuf[index + i]); // Q8_8 = Q0_16 * Q8_8 - } - - dst[x] = convert_cast(sum); // U8 = Q8_8 - } -} - -template -struct typed_resizeArea { - using p_f = void (*)(T dst[], const T* src[], const Size& inSz, const Size& outSz, - A yalpha, const MapperUnit& ymap, int xmaxdf, - const I xindex[], const A xalpha[], W vbuf[]); - -template -inline p_f operator()(type_to_type) { - return [](T dst[], const T* src[], const Size& inSz, const Size& outSz, - A yalpha, const MapperUnit& ymap, int xmaxdf, - const I xindex[], const A xalpha[], W vbuf[]) { - calcRowAreaImpl(isa_tag_t{}, dst, src, inSz, outSz, yalpha, - ymap, xmaxdf, xindex, xalpha, vbuf); - }; -} -}; -} // namespace - -template -struct choose_impl { -GAPI_FLUID_KERNEL(FChanToPlane, ChanToPlane, false) { - static const int Window = 1; - static void run(const cv::gapi::fluid::View& in, int chan, - cv::gapi::fluid::Buffer& out) { - GAPI_DbgAssert(is_cv_type_in_list(out.meta().depth)); - - const auto rowFunc = type_dispatch(out.meta().depth, cv_type_id{}, typed_chan_to_plane_row{}, nullptr); - - GAPI_DbgAssert(rowFunc); - - rowFunc(in.InLineB(0), chan, in.meta().chan, out.OutLineB(), in.length()); - } -}; - -GAPI_FLUID_KERNEL(FNV12toRGB, NV12toRGB, false) { - static const int Window = 1; - static const int LPI = 2; - static const auto Kind = cv::GFluidKernel::Kind::YUV420toRGB; - - static void run(const cv::gapi::fluid::View & in_y, - const cv::gapi::fluid::View & in_uv, - cv::gapi::fluid::Buffer & out) { - GAPI_DbgAssert(is_cv_type_in_list(out.meta().depth)); - - const uchar* uv_row = in_uv.InLineB(0); - const uchar* y_rows[2] = { in_y.InLineB(0), in_y.InLineB(1) }; - uchar* out_rows[2] = { out.OutLineB(0), out.OutLineB(1) }; - - int buf_width = out.length(); - - const auto rowFunc = type_dispatch(out.meta().depth, cv_type_id{}, typed_nv12_to_rgb_row{}, nullptr); - - GAPI_DbgAssert(rowFunc); - - rowFunc(y_rows, uv_row, out_rows, buf_width); - } -}; - -GAPI_FLUID_KERNEL(FI420toRGB, I420toRGB, false) { - static const int Window = 1; - static const int LPI = 2; - static const auto Kind = cv::GFluidKernel::Kind::YUV420toRGB; - - static void run(const cv::gapi::fluid::View & in_y, - const cv::gapi::fluid::View & in_u, - const cv::gapi::fluid::View & in_v, - cv::gapi::fluid::Buffer & out) { - GAPI_DbgAssert(is_cv_type_in_list(out.meta().depth)); - - const uchar* u_row = in_u.InLineB(0); - const uchar* v_row = in_v.InLineB(0); - const uchar* y_rows[2] = { in_y.InLineB(0), in_y.InLineB(1) }; - uchar* out_rows[2] = { out.OutLineB(0), out.OutLineB(1) }; - - int buf_width = out.length(); - GAPI_DbgAssert(in_u.length() == in_v.length()); - - const auto rowFunc = type_dispatch(out.meta().depth, cv_type_id{}, - typed_i420_to_rgb_row{}, - nullptr); - - GAPI_DbgAssert(rowFunc); - - rowFunc(y_rows, u_row, v_row, out_rows, buf_width); - } -}; - -GAPI_FLUID_KERNEL(FSplit2, Split2, false) { - static const int LPI = 4; - static const int Window = 1; - static void run(const cv::gapi::fluid::View & in, - cv::gapi::fluid::Buffer & out1, - cv::gapi::fluid::Buffer & out2) { - GAPI_DbgAssert(2 == in.meta().chan); - GAPI_DbgAssert(1 == out1.meta().chan); - GAPI_DbgAssert(1 == out2.meta().chan); - GAPI_DbgAssert(in.meta().depth == out1.meta().depth); - GAPI_DbgAssert(in.meta().depth == out2.meta().depth); - GAPI_DbgAssert(is_cv_type_in_list(in.meta().depth)); - - const auto rowFunc = type_dispatch(in.meta().depth, cv_type_id{}, - typed_split_row{}, nullptr); - for (int i = 0, lpi = out1.lpi(); i < lpi; i++) { - std::array outs = { out1.OutLineB(i), out2.OutLineB(i) }; - rowFunc(in.InLineB(i), outs, in.length()); - } - } -}; - -GAPI_FLUID_KERNEL(FSplit3, Split3, false) { - static const int LPI = 4; - static const int Window = 1; - static void run(const cv::gapi::fluid::View & in, - cv::gapi::fluid::Buffer & out1, - cv::gapi::fluid::Buffer & out2, - cv::gapi::fluid::Buffer & out3) { - GAPI_DbgAssert(3 == in.meta().chan); - GAPI_DbgAssert(1 == out1.meta().chan); - GAPI_DbgAssert(1 == out2.meta().chan); - GAPI_DbgAssert(1 == out3.meta().chan); - GAPI_DbgAssert(in.meta().depth == out1.meta().depth); - GAPI_DbgAssert(in.meta().depth == out2.meta().depth); - GAPI_DbgAssert(in.meta().depth == out3.meta().depth); - - GAPI_DbgAssert(is_cv_type_in_list(in.meta().depth)); - - const auto rowFunc = type_dispatch(in.meta().depth, cv_type_id{}, - typed_split_row{}, nullptr); - for (int i = 0, lpi = out1.lpi(); i < lpi; i++) { - std::array outs = { out1.OutLineB(i), out2.OutLineB(i), - out3.OutLineB(i) }; - rowFunc(in.InLineB(i), outs, in.length()); - } - } -}; - -GAPI_FLUID_KERNEL(FSplit4, Split4, false) { - static const int LPI = 4; - static const int Window = 1; - static void run(const cv::gapi::fluid::View & in, - cv::gapi::fluid::Buffer & out1, - cv::gapi::fluid::Buffer & out2, - cv::gapi::fluid::Buffer & out3, - cv::gapi::fluid::Buffer & out4) { - GAPI_DbgAssert(4 == in.meta().chan); - GAPI_DbgAssert(1 == out1.meta().chan); - GAPI_DbgAssert(1 == out2.meta().chan); - GAPI_DbgAssert(1 == out3.meta().chan); - GAPI_DbgAssert(1 == out4.meta().chan); - GAPI_DbgAssert(in.meta().depth == out1.meta().depth); - GAPI_DbgAssert(in.meta().depth == out2.meta().depth); - GAPI_DbgAssert(in.meta().depth == out3.meta().depth); - GAPI_DbgAssert(in.meta().depth == out4.meta().depth); - GAPI_DbgAssert(is_cv_type_in_list(in.meta().depth)); - - const auto rowFunc = type_dispatch(in.meta().depth, cv_type_id{}, - typed_split_row{}, nullptr); - for (int i = 0, lpi = out1.lpi(); i < lpi; i++) { - std::array outs = { out1.OutLineB(i), out2.OutLineB(i), - out3.OutLineB(i), out4.OutLineB(i) }; - rowFunc(in.InLineB(i), outs, in.length()); - } - } -}; - -GAPI_FLUID_KERNEL(FMerge2, Merge2, false) { - static const int LPI = 4; - static const int Window = 1; - static void run(const cv::gapi::fluid::View & a, - const cv::gapi::fluid::View & b, - cv::gapi::fluid::Buffer & out) { - GAPI_DbgAssert(is_cv_type_in_list(out.meta().depth)); - - const auto rowFunc = type_dispatch(out.meta().depth, cv_type_id{}, - typed_merge_row{}, nullptr); - for (int l = 0; l < out.lpi(); l++) { - rowFunc({ a.InLineB(l), b.InLineB(l) }, out.OutLineB(l), a.length()); - } - } -}; - -GAPI_FLUID_KERNEL(FMerge3, Merge3, false) { - static const int LPI = 4; - static const int Window = 1; - static void run(const cv::gapi::fluid::View & a, - const cv::gapi::fluid::View & b, - const cv::gapi::fluid::View & c, - cv::gapi::fluid::Buffer & out) { - GAPI_DbgAssert(is_cv_type_in_list(out.meta().depth)); - - const auto rowFunc = type_dispatch(out.meta().depth, cv_type_id{}, - typed_merge_row{}, nullptr); - for (int l = 0; l < out.lpi(); l++) { - rowFunc({ a.InLineB(l), b.InLineB(l), c.InLineB(l) }, out.OutLineB(l), a.length()); - } - } -}; - -GAPI_FLUID_KERNEL(FMerge4, Merge4, false) { - static const int LPI = 4; - static const int Window = 1; - static void run(const cv::gapi::fluid::View & a, - const cv::gapi::fluid::View & b, - const cv::gapi::fluid::View & c, - const cv::gapi::fluid::View & d, - cv::gapi::fluid::Buffer & out) { - GAPI_DbgAssert(is_cv_type_in_list(out.meta().depth)); - - const auto rowFunc = type_dispatch(out.meta().depth, cv_type_id{}, - typed_merge_row{}, nullptr); - for (int l = 0; l < out.lpi(); l++) { - rowFunc({ a.InLineB(l), b.InLineB(l), c.InLineB(l), d.InLineB(l) }, out.OutLineB(l), a.length()); - } - } -}; - -template -static inline void callRowFunc(uint8_t* dst[], const uint8_t* src0[], - const uint8_t* src1[], const short alpha[], - const short clone[], const short mapsx[], - const short beta[], uint8_t tmp[], const Size& inSz, - const Size& outSz, const int lpi, const int length, const int depth) { - const auto rowFunc = type_dispatch(depth, cv_type_id{}, KT{}, nullptr); - - GAPI_DbgAssert(rowFunc); - - rowFunc(dst, src0, src1, alpha, clone, mapsx, beta, tmp, inSz, outSz, lpi, length); -} - -template -static inline void callRowFunc(float* dst[], const float* src0[], const float* src1[], - const float alpha[], const float clone[], const int mapsx[], - const float beta[], float tmp[], const Size& inSz, - const Size& outSz, const int lpi, const int length, const int depth) { - const auto rowFunc = type_dispatch(depth, cv_type_id{}, KT{}, nullptr); - - GAPI_DbgAssert(rowFunc); - - rowFunc(dst, src0, src1, alpha, mapsx, beta, inSz, outSz, lpi, length); -} - -template -static inline void calcRowLinear(const cv::gapi::fluid::View& in, - cv::gapi::fluid::Buffer& out, - cv::gapi::fluid::Buffer& scratch) { - GAPI_DbgAssert(is_cv_type_in_list(out.meta().depth)); - - auto inSz = in.meta().size; - auto outSz = out.meta().size; - - auto inY = in.y(); - int length = out.length(); - int outY = out.y(); - int lpi = out.lpi(); - GAPI_DbgAssert(outY + lpi <= outSz.height); - - GAPI_DbgAssert(lpi <= 4); - - linearScratchDesc scr(inSz.width, inSz.height, outSz.width, - outSz.height, scratch.OutLineB()); - - const auto* alpha = scr.alpha; - const auto* clone = scr.clone; - const auto* mapsx = scr.mapsx; - const auto* beta0 = scr.beta; - const auto* mapsy = scr.mapsy; - auto* tmp = scr.tmp; - - const auto* beta = beta0 + outY; - const T* src0[4]; - const T* src1[4]; - T* dst[4]; - - for (int l = 0; l < lpi; l++) { - auto index0 = mapsy[outY + l] - inY; - auto index1 = mapsy[outSz.height + outY + l] - inY; - src0[l] = in.InLine(index0); - src1[l] = in.InLine(index1); - dst[l] = out.OutLine(l); - } - - callRowFunc(dst, src0, src1, alpha, clone, mapsx, beta, tmp, inSz, outSz, - lpi, length, out.meta().depth); -} - -GAPI_FLUID_KERNEL(FScalePlane8u, ScalePlane8u, true) { - static const int Window = 1; - static const int LPI = 4; - static const auto Kind = cv::GFluidKernel::Kind::Resize; - - static void initScratch(const cv::GMatDesc & in, - Size outSz, int /*interp*/, - cv::gapi::fluid::Buffer & scratch) { - initScratchLinear(in, outSz, scratch, LPI); - } - - static void resetScratch(cv::gapi::fluid::Buffer& /*scratch*/) { - } - - static void run(const cv::gapi::fluid::View & in, Size /*sz*/, int /*interp*/, - cv::gapi::fluid::Buffer & out, cv::gapi::fluid::Buffer & scratch) { - calcRowLinear, - resizeLinearU8C1_suptypes>(in, out, scratch); - } -}; - -GAPI_FLUID_KERNEL(FScalePlane32f, ScalePlane32f, true) { - static const int Window = 1; - static const int LPI = 4; - static const auto Kind = cv::GFluidKernel::Kind::Resize; - - static void initScratch(const cv::GMatDesc & in, - Size outSz, int /*interp*/, - cv::gapi::fluid::Buffer & scratch) { - GAPI_DbgAssert(in.depth == CV_32F && in.chan == 1); - - initScratchLinear(in, outSz, scratch, 0); - } - - static void resetScratch(cv::gapi::fluid::Buffer& /*scratch*/) { - } - - static void run(const cv::gapi::fluid::View & in, Size /*sz*/, int /*interp*/, - cv::gapi::fluid::Buffer & out, cv::gapi::fluid::Buffer & scratch) { - calcRowLinear, - resizeLinearF32C1_suptypes>(in, out, scratch); - } -}; - -GAPI_FLUID_KERNEL(FUpscalePlaneArea8u, UpscalePlaneArea8u, true) { - static const int Window = 1; - static const int LPI = 4; - static const auto Kind = cv::GFluidKernel::Kind::Resize; - - static void initScratch(const cv::GMatDesc & in, - Size outSz, int /*interp*/, - cv::gapi::fluid::Buffer & scratch) { - initScratchLinear(in, outSz, scratch, LPI); - } - - static void resetScratch(cv::gapi::fluid::Buffer& /*scratch*/) { - } - - static void run(const cv::gapi::fluid::View & in, Size /*sz*/, int /*interp*/, - cv::gapi::fluid::Buffer & out, cv::gapi::fluid::Buffer & scratch) { - calcRowLinear, - resizeLinearU8C1_suptypes>(in, out, scratch); - } -}; - -GAPI_FLUID_KERNEL(FUpscalePlaneArea32f, UpscalePlaneArea32f, true) { - static const int Window = 1; - static const int LPI = 4; - static const auto Kind = cv::GFluidKernel::Kind::Resize; - - static void initScratch(const cv::GMatDesc & in, - Size outSz, int /*interp*/, - cv::gapi::fluid::Buffer & scratch) { - initScratchLinear(in, outSz, scratch, 0); - } - - static void resetScratch(cv::gapi::fluid::Buffer& /*scratch*/) { - } - - static void run(const cv::gapi::fluid::View & in, Size /*sz*/, int /*interp*/, - cv::gapi::fluid::Buffer & out, cv::gapi::fluid::Buffer & scratch) { - calcRowLinear, - resizeLinearF32C1_suptypes>(in, out, scratch); - } -}; - -template -static inline void calcRowLinearC(const cv::gapi::fluid::View& in, - std::array, chs>& out, - cv::gapi::fluid::Buffer& scratch) { - GAPI_DbgAssert(is_cv_type_in_list(in.meta().depth)); - - auto inSz = in.meta().size; - auto outSz = out[0].get().meta().size; - - auto inY = in.y(); - auto outY = out[0].get().y(); - auto lpi = out[0].get().lpi(); - - GAPI_DbgAssert(outY + lpi <= outSz.height); - GAPI_DbgAssert(lpi <= 4); - - linearScratchDesc scr(inSz.width, inSz.height, outSz.width, outSz.height, scratch.OutLineB()); - - const auto *alpha = scr.alpha; - const auto *clone = scr.clone; - const auto *mapsx = scr.mapsx; - const auto *beta0 = scr.beta; - const auto *mapsy = scr.mapsy; - auto *tmp = scr.tmp; - - const auto *beta = beta0 + outY; - const T *src0[4]; - const T *src1[4]; - std::array, chs> dst; - - for (int l = 0; l < lpi; l++) { - auto index0 = mapsy[outY + l] - inY; - auto index1 = mapsy[outSz.height + outY + l] - inY; - src0[l] = in.InLine(index0); - src1[l] = in.InLine(index1); - for (int c=0; c < chs; c++) { - dst[c][l] = out[c].get().template OutLine(l); - } - } - auto length = out[0].get().length(); - - const auto rowFunc = type_dispatch(in.meta().depth, - cv_type_id{}, - typed_resizeLinearU8C3C4{}, - nullptr); - GAPI_DbgAssert(rowFunc); - - rowFunc(dst, src0, src1, alpha, clone, mapsx, beta, tmp, inSz, outSz, lpi, length); -} - -GAPI_FLUID_KERNEL(FScalePlanes, ScalePlanes, true) { - static const int Window = 1; - static const int LPI = 4; - static const auto Kind = cv::GFluidKernel::Kind::Resize; - - static void initScratch(const cv::GMatDesc& in, int, Size, - Size outSz, int /*interp*/, - cv::gapi::fluid::Buffer &scratch) { - initScratchLinear(in, outSz, scratch, LPI); - } - - static void resetScratch(cv::gapi::fluid::Buffer& /*scratch*/) { - } - - static void run(const cv::gapi::fluid::View& in, int, Size, Size/*sz*/, int /*interp*/, - cv::gapi::fluid::Buffer& out1, - cv::gapi::fluid::Buffer& out2, - cv::gapi::fluid::Buffer& out3, - cv::gapi::fluid::Buffer& scratch) { - constexpr int numChan = 3; - std::array, numChan> out = {out1, out2, out3}; - calcRowLinearC(in, out, scratch); - } -}; - -GAPI_FLUID_KERNEL(FScalePlanes4, ScalePlanes4, true) { - static const int Window = 1; - static const int LPI = 4; - static const auto Kind = cv::GFluidKernel::Kind::Resize; - - static void initScratch(const cv::GMatDesc& in, int, Size, - Size outSz, int /*interp*/, - cv::gapi::fluid::Buffer &scratch) { - initScratchLinear(in, outSz, scratch, LPI); - } - - static void resetScratch(cv::gapi::fluid::Buffer& /*scratch*/) { - } - - static void run(const cv::gapi::fluid::View& in, int, Size, Size/*sz*/, int /*interp*/, - cv::gapi::fluid::Buffer& out1, - cv::gapi::fluid::Buffer& out2, - cv::gapi::fluid::Buffer& out3, - cv::gapi::fluid::Buffer& out4, - cv::gapi::fluid::Buffer& scratch) { - constexpr int numChan = 4; - std::array, numChan> out = {out1, out2, out3, out4}; - calcRowLinearC(in, out, scratch); - } -}; - -#if defined __GNUC__ -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wstrict-aliasing" -#endif - -template -static inline void calcAreaRow(const cv::gapi::fluid::View& in, cv::gapi::fluid::Buffer& out, - cv::gapi::fluid::Buffer& scratch) { - using Unit = typename Mapper::Unit; - using alpha_type = typename Mapper::alpha_type; - using index_type = typename Mapper::index_type; - using work_type = typename Mapper::work_type; - - Size inSz = in.meta().size; - Size outSz = out.meta().size; - - // this method is valid only for down-scale - GAPI_DbgAssert(inSz.width >= outSz.width); - GAPI_DbgAssert(inSz.height >= outSz.height); - - Mapper ymapper(inSz.height, outSz.height); - - auto *xmaxdf = scratch.OutLine(); - auto maxdif = xmaxdf[0]; - - auto *xindex = reinterpret_cast(xmaxdf + 1); - auto *xalpha = reinterpret_cast(xindex + outSz.width); - auto *vbuf_c = reinterpret_cast(xalpha + outSz.width * maxdif); - - auto *vbuf = const_cast(vbuf_c); - - int iny = in.y(); - int y = out.y(); - - int lpi = out.lpi(); - GAPI_DbgAssert(y + lpi <= outSz.height); - - const auto rowFunc = type_dispatch(in.meta().depth, - cv_type_id{}, - typed_resizeArea{}, - nullptr); - GAPI_DbgAssert(rowFunc); - constexpr int max_num = 32; - for (int l = 0; l < lpi; l++) { - Unit ymap = ymapper.map(y + l); - - GAPI_Assert(ymap.index1 - ymap.index0 <= max_num); - GAPI_Assert(ymap.index1 - ymap.index0 > 0); - const T *src[max_num] = {}; - - for (int yin = ymap.index0; yin < ymap.index1; yin++) { - src[yin - ymap.index0] = in.InLine(yin - iny); - } - - auto dst = out.OutLine(l); - - rowFunc(dst, src, inSz, outSz, ymapper.alpha, ymap, xmaxdf[0], xindex, xalpha, vbuf); - } -} - -#if defined __GNUC__ -# pragma GCC diagnostic pop -#endif - -GAPI_FLUID_KERNEL(FScalePlaneArea32f, ScalePlaneArea32f, true) { - static const int Window = 1; - static const int LPI = 4; - static const auto Kind = cv::GFluidKernel::Kind::Resize; - - static void initScratch(const cv::GMatDesc& in, - Size outSz, int /*interp*/, - cv::gapi::fluid::Buffer& scratch) { - initScratchArea(in, outSz, scratch); - } - - static void resetScratch(cv::gapi::fluid::Buffer& /*scratch*/) { - } - - static void run(const cv::gapi::fluid::View& in, Size /*sz*/, int /*interp*/, - cv::gapi::fluid::Buffer& out, cv::gapi::fluid::Buffer& scratch) { - calcAreaRow(in, out, scratch); - } -}; - -GAPI_FLUID_KERNEL(FScalePlaneArea8u, ScalePlaneArea8u, true) { - static const int Window = 1; - static const int LPI = 4; - static const auto Kind = cv::GFluidKernel::Kind::Resize; - - static void initScratch(const cv::GMatDesc& in, - Size outSz, int /*interp*/, - cv::gapi::fluid::Buffer& scratch) { -#if USE_CVKL - if (with_cpu_x86_sse42()) { - const Size& inSz = in.size; - if (inSz.width > outSz.width && inSz.height > outSz.height) { - // CVKL code we use supports only downscale - initScratchArea_CVKL_U8(in, outSz, scratch); - return; - } - } -#endif - - initScratchArea(in, outSz, scratch); - } - - static void resetScratch(cv::gapi::fluid::Buffer& /*scratch*/) { - } - - static void run(const cv::gapi::fluid::View& in, Size /*sz*/, int /*interp*/, - cv::gapi::fluid::Buffer& out, cv::gapi::fluid::Buffer& scratch) { -#if USE_CVKL - if (with_cpu_x86_sse42()) { - auto inSz = in.meta().size; - auto outSz = out.meta().size; - if (inSz.width > outSz.width && inSz.height > outSz.height) { - // CVKL's code supports only downscale - calcAreaRow_CVKL_U8(in, out, scratch); - return; - } - } -#endif - - calcAreaRow(in, out, scratch); - } -}; -}; - -namespace { -struct CC_and_MergeISA { - cv::gapi::GKernelPackage& pckg; - - CC_and_MergeISA(cv::gapi::GKernelPackage& _pckg) : pckg(_pckg) {} - - template - inline bool operator()(type_to_type) { - pckg.include::FI420toRGB>(); - pckg.include::FNV12toRGB>(); - pckg.include::FChanToPlane>(); - pckg.include::FMerge2>(); - pckg.include::FMerge3>(); - pckg.include::FMerge4>(); - //at the moment type_dispatch requires something to be returned by the lambda - return true; - } -}; - -struct Split_ResizeISA { - cv::gapi::GKernelPackage& pckg; - - Split_ResizeISA(cv::gapi::GKernelPackage& _pckg) : pckg(_pckg) {} - - template - inline bool operator()(type_to_type) { - pckg.include::FSplit2>(); - pckg.include::FSplit3>(); - pckg.include::FSplit4>(); - pckg.include::FScalePlane8u>(); - pckg.include::FScalePlane32f>(); - pckg.include::FScalePlanes>(); - pckg.include::FScalePlanes4>(); - pckg.include::FScalePlaneArea8u>(); - pckg.include::FScalePlaneArea32f>(); - pckg.include::FUpscalePlaneArea8u>(); - pckg.include::FUpscalePlaneArea32f>(); - //at the moment type_dispatch requires something to be returned by the lambda - return true; - } -}; -} //namespace - -inline cv::gapi::GKernelPackage FKernelsChooseISA() { - // At the moment AVX512 implementation of wide universal intrinsics is slower than AVX2. - // So, disable it for now. - using isas = remove_t; - - cv::gapi::GKernelPackage pckg1, pckg2; - CC_and_MergeISA ccISA{ pckg1 }; - Split_ResizeISA sISA{ pckg2 }; - - type_dispatch(is_isa_present{}, ccISA, false); - type_dispatch(is_isa_present{}, sISA, false); - - return combine(pckg1, pckg2); -} - -//---------------------------------------------------------------------- - -GAPI_COMPOUND_KERNEL(FScalePlane, ScalePlane) { - static cv::GMat expand(cv::GMat in, int type, const Size& szIn, const Size& szOut, int interp) { - GAPI_DbgAssert(CV_8UC1 == type || CV_32FC1 == type); - GAPI_DbgAssert(cv::INTER_AREA == interp || cv::INTER_LINEAR == interp); - - if (cv::INTER_AREA == interp) { - bool upscale = szIn.width < szOut.width || szIn.height < szOut.height; - if (CV_8UC1 == type) { - if (upscale) - return UpscalePlaneArea8u::on(in, szOut, interp); - else - return ScalePlaneArea8u::on(in, szOut, interp); - } - if (CV_32FC1 == type) { - if (upscale) - return UpscalePlaneArea32f::on(in, szOut, interp); - else - return ScalePlaneArea32f::on(in, szOut, interp); - } - } - - if (cv::INTER_LINEAR == interp) { - if (CV_8UC1 == type) { - return ScalePlane8u::on(in, szOut, interp); - } - if (CV_32FC1 == type) { - return ScalePlane32f::on(in, szOut, interp); - } - } - - GAPI_Assert(!"unsupported parameters"); - return {}; - } -}; - -//------------------------------------------------------------------------------ - -namespace { - -template -void convert_precision(const uint8_t* src, uint8_t* dst, const int width) { - const auto *in = reinterpret_cast(src); - auto *out = reinterpret_cast(dst); - - for (int i = 0; i < width; i++) { - out[i] = saturate_cast(in[i]); - } -} - -} // namespace - -GAPI_FLUID_KERNEL(FConvertDepth, ConvertDepth, false) { - static const int Window = 1; - - static void run(const cv::gapi::fluid::View& src, int depth, cv::gapi::fluid::Buffer& dst) { - GAPI_Assert(src.meta().depth == CV_8U || src.meta().depth == CV_32F || src.meta().depth == CV_16U); - GAPI_Assert(dst.meta().depth == CV_8U || dst.meta().depth == CV_32F || dst.meta().depth == CV_16U); - GAPI_Assert(src.meta().chan == 1); - GAPI_Assert(dst.meta().chan == 1); - GAPI_Assert(src.length() == dst.length()); - - constexpr unsigned supported_types_n = 3; - using p_f = void (*)( const uint8_t* src, uint8_t* dst, const int width); - using table_string_t = std::array; - - constexpr std::array func_table = { - table_string_t{convert_precision, convert_precision, convert_precision}, - table_string_t{convert_precision, convert_precision, convert_precision}, - table_string_t{convert_precision, convert_precision, convert_precision} - }; - - auto depth_to_index = [](int depth){ - switch (depth) { - case CV_16U: return 0; - case CV_32F: return 1; - case CV_8U: return 2; - default: GAPI_Assert(!"not supported depth"); return -1; - } - }; - const auto *in = src.InLineB(0); - auto *out = dst.OutLineB(); - - auto const width = dst.length(); - auto const src_index = depth_to_index(src.meta().depth); - auto const dst_index = depth_to_index(dst.meta().depth); - - (func_table[src_index][dst_index])(in, out, width); - } -}; - -namespace { - template - void sub(const uint8_t* src, uint8_t* dst, const int width, double c) { - const auto *in = reinterpret_cast(src); - auto *out = reinterpret_cast(dst); - - for (int i = 0; i < width; i++) { - out[i] = saturate_cast(in[i] - c); - } - } - - template - void div(const uint8_t* src, uint8_t* dst, const int width, double c) { - const auto *in = reinterpret_cast(src); - auto *out = reinterpret_cast(dst); - - for (int i = 0; i < width; i++) { - out[i] = saturate_cast(in[i] / c); - } - } -} // namespace - -GAPI_FLUID_KERNEL(FSubC, GSubC, false) { - static const int Window = 1; - - static void run(const cv::gapi::fluid::View& src, const cv::Scalar &scalar, int depth, cv::gapi::fluid::Buffer& dst) { - GAPI_Assert(src.meta().depth == CV_32F && src.meta().chan == 1); - - const auto *in = src.InLineB(0); - auto *out = dst.OutLineB(); - - auto const width = dst.length(); - - sub(in, out, width, scalar[0]); - } -}; - -GAPI_FLUID_KERNEL(FDivC, GDivC, false) { - static const int Window = 1; - - static void run(const cv::gapi::fluid::View &src, const cv::Scalar &scalar, double _scale, int /*dtype*/, - cv::gapi::fluid::Buffer &dst) { - GAPI_Assert(src.meta().depth == CV_32F && src.meta().chan == 1); - - const auto *in = src.InLineB(0); - auto *out = dst.OutLineB(); - - auto const width = dst.length(); - - div(in, out, width, scalar[0]); - } -}; -} // namespace kernels - -//---------------------------------------------------------------------- - -using namespace kernels; - -cv::gapi::GKernelPackage preprocKernels() { - return combine( - FKernelsChooseISA(), - cv::gapi::kernels - < FScalePlane - , FConvertDepth - , FSubC - , FDivC - >()); -} - -} // namespace gapi -} // namespace InferenceEngine diff --git a/src/common/preprocessing/src/ie_preprocess_gapi_kernels.hpp b/src/common/preprocessing/src/ie_preprocess_gapi_kernels.hpp deleted file mode 100644 index 0887cc60e34..00000000000 --- a/src/common/preprocessing/src/ie_preprocess_gapi_kernels.hpp +++ /dev/null @@ -1,364 +0,0 @@ -// Copyright (C) 2018-2023 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#pragma once - -# ifndef GAPI_STANDALONE -# error non standalone GAPI -# endif - -#include -#include - -#include -#include -#include -#include - -namespace InferenceEngine { -namespace gapi { - using Size = cv::gapi::own::Size; - - using GMat2 = std::tuple; - using GMat3 = std::tuple; - using GMat4 = std::tuple; - - G_TYPED_KERNEL(ChanToPlane, , "com.intel.ie.chan_to_plane") { - static cv::GMatDesc outMeta(const cv::GMatDesc &in, int chan) { - GAPI_Assert(chan < in.chan); - return in.withType(in.depth, 1); - } - }; - - G_TYPED_KERNEL(ScalePlane, , "com.intel.ie.scale_plane") { - static cv::GMatDesc outMeta(const cv::GMatDesc &in, int type, const Size &szIn, const Size &szOut, int) { - GAPI_Assert(type == in.depth); - return in.withSize(szOut); - } - }; - - G_TYPED_KERNEL_M(ScalePlanes, , "com.intel.ie.scale_planes") { - static std::tuple outMeta(const cv::GMatDesc &in, int /*type*/, const Size &szIn, - const Size &szOut, int interp) { - // This kernel supports only RGB 8U inputs - GAPI_Assert(in.depth == CV_8U); - GAPI_Assert(in.chan == 3); - // cv::INTER_LINEAR is the only supported interpolation - GAPI_Assert(interp == cv::INTER_LINEAR); - // ad-hoc withChan - cv::GMatDesc out_desc = in.withType(in.depth, 1).withSize(szOut); - return std::make_tuple(out_desc, out_desc, out_desc); - } - }; - - G_TYPED_KERNEL_M(ScalePlanes4, , "com.intel.ie.scale_planes4") { - static std::tuple outMeta(const cv::GMatDesc &in, int /*type*/, const Size &szIn, - const Size &szOut, int interp) { - // This kernel supports only RGB 8U inputs - GAPI_Assert(in.depth == CV_8U); - GAPI_Assert(in.chan == 4); - // cv::INTER_LINEAR is the only supported interpolation - GAPI_Assert(interp == cv::INTER_LINEAR); - // ad-hoc withChan - cv::GMatDesc out_desc = in.withType(in.depth, 1).withSize(szOut); - return std::make_tuple(out_desc, out_desc, out_desc, out_desc); - } - }; - - G_TYPED_KERNEL(ScalePlane8u, , "com.intel.ie.scale_plane_8u") { - static cv::GMatDesc outMeta(const cv::GMatDesc & in, const Size & sz, int) { - GAPI_DbgAssert(in.depth == CV_8U && in.chan == 1); - return in.withSize(sz); - } - }; - - G_TYPED_KERNEL(ScalePlane32f, , "com.intel.ie.scale_plane_32f") { - static cv::GMatDesc outMeta(const cv::GMatDesc & in, const Size & sz, int) { - GAPI_DbgAssert(in.depth == CV_32F && in.chan == 1); - return in.withSize(sz); - } - }; - - G_TYPED_KERNEL(UpscalePlaneArea8u, , "com.intel.ie.upscale_plane_area_8u") { - static cv::GMatDesc outMeta(const cv::GMatDesc & in, const Size & sz, int) { - GAPI_DbgAssert(in.depth == CV_8U && in.chan == 1); - GAPI_DbgAssert(in.size.width < sz.width || in.size.height < sz.height); - return in.withSize(sz); - } - }; - - G_TYPED_KERNEL(UpscalePlaneArea32f, , "com.intel.ie.upscale_plane_area_32f") { - static cv::GMatDesc outMeta(const cv::GMatDesc & in, const Size & sz, int) { - GAPI_DbgAssert(in.depth == CV_32F && in.chan == 1); - GAPI_DbgAssert(in.size.width < sz.width || in.size.height < sz.height); - return in.withSize(sz); - } - }; - - G_TYPED_KERNEL(ScalePlaneArea8u, , "com.intel.ie.scale_plane_area_8u") { - static cv::GMatDesc outMeta(const cv::GMatDesc & in, const Size & sz, int) { - GAPI_DbgAssert(in.depth == CV_8U && in.chan == 1); - GAPI_DbgAssert(in.size.width >= sz.width && in.size.height >= sz.height); - return in.withSize(sz); - } - }; - - G_TYPED_KERNEL(ScalePlaneArea32f, , "com.intel.ie.scale_plane_area_32f") { - static cv::GMatDesc outMeta(const cv::GMatDesc & in, const Size & sz, int) { - GAPI_DbgAssert(in.depth == CV_32F && in.chan == 1); - GAPI_DbgAssert(in.size.width >= sz.width && in.size.height >= sz.height); - return in.withSize(sz); - } - }; - - G_TYPED_KERNEL(Merge2, , "com.intel.ie.merge2") { - static cv::GMatDesc outMeta(const cv::GMatDesc &in, const cv::GMatDesc &) { - // FIXME: check a/b are equal! - return in.withType(in.depth, 2); - } - }; - - G_TYPED_KERNEL(Merge3, , "com.intel.ie.merge3") { - static cv::GMatDesc outMeta(const cv::GMatDesc &in, const cv::GMatDesc &, const cv::GMatDesc &) { - // FIXME: check a/b are equal! - return in.withType(in.depth, 3); - } - }; - - G_TYPED_KERNEL(Merge4, , "com.intel.ie.merge4") { - static cv::GMatDesc outMeta(const cv::GMatDesc& in, - const cv::GMatDesc&, const cv::GMatDesc&, const cv::GMatDesc&) { - // FIXME: check a/b are equal! - return in.withType(in.depth, 4); - } - }; - - G_TYPED_KERNEL_M(Split2, , "com.intel.ie.split2") { - static std::tuple outMeta(const cv::GMatDesc& in) { - const auto out_depth = in.depth; - const auto out_desc = in.withType(out_depth, 1); - return std::make_tuple(out_desc, out_desc); - } - }; - - G_TYPED_KERNEL_M(Split3, , "com.intel.ie.split3") { - static std::tuple outMeta(const cv::GMatDesc& in) { - const auto out_depth = in.depth; - const auto out_desc = in.withType(out_depth, 1); - return std::make_tuple(out_desc, out_desc, out_desc); - } - }; - - G_TYPED_KERNEL_M(Split4, , "com.intel.ie.split4") { - static std::tuple outMeta(const cv::GMatDesc& in) { - const auto out_depth = in.depth; - const auto out_desc = in.withType(out_depth, 1); - return std::make_tuple(out_desc, out_desc, out_desc, out_desc); - } - }; - - G_TYPED_KERNEL(NV12toRGB, , "com.intel.ie.nv12torgb") { - static cv::GMatDesc outMeta(cv::GMatDesc in_y, cv::GMatDesc in_uv) { - GAPI_Assert(in_y.chan == 1); - GAPI_Assert(in_uv.chan == 2); - GAPI_Assert(in_y.depth == CV_8U); - GAPI_Assert(in_uv.depth == CV_8U); - // UV size should be aligned with Y - GAPI_Assert(in_y.size.width == 2 * in_uv.size.width); - GAPI_Assert(in_y.size.height == 2 * in_uv.size.height); - return in_y.withType(CV_8U, 3); - } - }; - - G_TYPED_KERNEL(I420toRGB, , "com.intel.ie.i420torgb") { - static cv::GMatDesc outMeta(cv::GMatDesc in_y, cv::GMatDesc in_u, cv::GMatDesc in_v) { - GAPI_Assert(in_y.chan == 1); - GAPI_Assert(in_u.chan == 1); - GAPI_Assert(in_v.chan == 1); - GAPI_Assert(in_y.depth == CV_8U); - GAPI_Assert(in_u.depth == CV_8U); - GAPI_Assert(in_v.depth == CV_8U); - // U and V size should be aligned with Y - GAPI_Assert(in_y.size.width == 2 * in_u.size.width); - GAPI_Assert(in_y.size.height == 2 * in_u.size.height); - - GAPI_Assert(in_y.size.width == 2 * in_v.size.width); - GAPI_Assert(in_y.size.height == 2 * in_v.size.height); - - return in_y.withType(CV_8U, 3); - } - }; - - G_TYPED_KERNEL(ConvertDepth, , "com.intel.ie.ConvertDepth") { - static cv::GMatDesc outMeta(const cv::GMatDesc& in, int depth) { - GAPI_Assert(in.depth == CV_8U || in.depth == CV_16U || in.depth == CV_32F); - GAPI_Assert(depth == CV_8U || depth == CV_32F || depth == CV_16U); - - return in.withDepth(depth); - } - }; - - G_TYPED_KERNEL(GSubC, , "com.intel.ie.math.subC") { - static cv::GMatDesc outMeta(cv::GMatDesc a, cv::GScalarDesc, int ddepth) { - return a.withDepth(ddepth); - } - }; - - G_TYPED_KERNEL(GDivC, , "com.intel.ie.math.divC") { - static cv::GMatDesc outMeta(cv::GMatDesc a, cv::GScalarDesc, double, int ddepth) { - return a.withDepth(ddepth); - } - }; - cv::gapi::GKernelPackage preprocKernels(); - - -namespace kernels { - -struct fp_16_t { - int16_t v; -}; - -template -struct cv_type_to_depth; - -template<> struct cv_type_to_depth { enum { depth = CV_8U }; }; -template<> struct cv_type_to_depth { enum { depth = CV_8S }; }; -template<> struct cv_type_to_depth { enum { depth = CV_16U }; }; -template<> struct cv_type_to_depth { enum { depth = CV_16S }; }; -template<> struct cv_type_to_depth { enum { depth = CV_32S }; }; -template<> struct cv_type_to_depth { enum { depth = CV_32F }; }; -template<> struct cv_type_to_depth { enum { depth = CV_16F }; }; - -template -struct typelist {}; - -template -struct head; - -template class list, typename head_t, typename ... types> -struct head> { using type = head_t; }; - -template -using head_t = typename head::type; - -template -struct type_to_type {}; - -template -struct type_dispatch_impl; - -//FIXME: add test for type_dispatch -template class typelist, typename... type> -struct type_dispatch_impl> { - template - static result_t dispatch(type_id_t type_id, type_to_id_t&& type_to_id, type_to_value_t&& type_to_value, default_t default_value) { - result_t res = default_value; - - bool matched = false; - std::initializer_list({ - !matched && (type_id == type_to_id(type_to_type{})) ? - (matched = true, res = type_to_value(type_to_type{})), 0 - : 0 - ... - }); - return res; - } - - template - static result_t dispatch(pred_t&& pred, type_to_value_t&& type_to_value, default_t default_value) { - result_t res = default_value; - - bool matched = false; - std::initializer_list({ - !matched && pred(type_to_type{}) ? - (matched = true, res = type_to_value(type_to_type{})), 0 - : 0 - ... - }); - return res; - } -}; - -template -struct concat; - -template -using concat_t = typename concat::type; - -template class left_list, typename ... left_types, template class right_list, typename ... right_types> -struct concat, right_list> { - using type = left_list; -}; - -template< class T, class U > -using is_same_t = typename std::is_same::type; - -template struct if_c_impl; - -template struct if_c_impl { - using type = T; -}; - -template struct if_c_impl { - using type = E; -}; - -template -using if_c = typename if_c_impl::type; - -template -using if_ = typename if_c_impl::type; - -template -struct remove; - -template -using remove_t = typename remove::type; - - -template class list, typename head_t, typename ... types, typename t> -struct remove, t> { - using type = concat_t< - if_, list<>, list>, - remove_t, t> - >; -}; - -template class list, typename t> -struct remove, t> { - using type = list<>; -}; - -template ()(type_to_type> {})) > - inline result_t type_dispatch(type_id_t type_id, type_to_id_t&& type_to_id, type_to_value_t&& type_to_value, default_t default_value = {}) { - return type_dispatch_impl::template dispatch(std::forward(type_id), - std::forward(type_to_id), - std::forward(type_to_value), - std::forward(default_value)); -} - -template ()(type_to_type> {})) > - inline result_t type_dispatch(pred_t&& pred, type_to_value_t&& type_to_value, default_t default_value = {}) { - return type_dispatch_impl::template dispatch(std::forward(pred), - std::forward(type_to_value), - std::forward(default_value)); -} - -namespace { -struct cv_type_id { - template - int operator()(type_to_type) { return cv_type_to_depth::depth; } -}; - -} // namespace - -template -bool is_cv_type_in_list(const int type_id) { - return type_dispatch(type_id, cv_type_id{}, [](...) { return true; }, false); -} -} // namespace kernels -} // namespace gapi -} // namespace InferenceEngine diff --git a/src/common/preprocessing/src/ie_preprocess_gapi_kernels_impl.hpp b/src/common/preprocessing/src/ie_preprocess_gapi_kernels_impl.hpp deleted file mode 100644 index 944c5fe7e36..00000000000 --- a/src/common/preprocessing/src/ie_preprocess_gapi_kernels_impl.hpp +++ /dev/null @@ -1,154 +0,0 @@ -// Copyright (C) 2018-2023 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#pragma once - -#if defined(HAVE_SSE) || defined(HAVE_AVX2) || defined(HAVE_AVX512) || defined(HAVE_NEON) - #define MANUAL_SIMD 1 // 1=call manually vectored code, 0=don't -#else - #define MANUAL_SIMD 0 -#endif - -#ifdef HAVE_SSE - #define USE_CVKL 1 // 1=reuse CVKL code for Resize, 0=don't -#else - #define USE_CVKL 0 -#endif - -#if defined(HAVE_SSE) || defined(HAVE_AVX2) || defined(HAVE_AVX512) - #define CPU_SIMD 1 -#else - #define CPU_SIMD 0 -#endif - -#include -#include -#include - -#if defined(__GNUC__) && (__GNUC__ <= 5) -#include -#endif - -namespace InferenceEngine { -namespace gapi { -namespace kernels { - -// dummy field is needed to workaround ABI change issue between CXX ABI 11 (g++-7) and 12 (g++-8) -// see https://gcc.gnu.org/onlinedocs/gcc/C_002b_002b-Dialect-Options.html -struct avx512_tag { int _ = 0; }; -struct avx2_tag { int _ = 0; }; -struct sse42_tag { int _ = 0; }; -struct neon_tag { int _ = 0; }; -struct scalar_tag { int _ = 0; }; - -template static inline DST saturate_cast(SRC x); -template<> inline short saturate_cast(int x) { return (std::min)(SHRT_MAX, (std::max)(SHRT_MIN, x)); } -template<> inline short saturate_cast(float x) { return saturate_cast(static_cast(std::rint(x))); } -template<> inline float saturate_cast(float x) { return x; } -template<> inline short saturate_cast(short x) { return x; } - -template<> inline uint16_t saturate_cast(uint16_t x) { return x; } -template<> inline float saturate_cast(uint16_t x) { return x; } - -template<> inline uint16_t saturate_cast(int x) { return (std::min)(USHRT_MAX, (std::max)(0, x)); } -template<> inline uint16_t saturate_cast(float x) { return saturate_cast(static_cast(std::rint(x))); } -template<> inline uchar saturate_cast(int v) { return (uchar)((unsigned)v <= UCHAR_MAX ? v : v > 0 ? UCHAR_MAX : 0); } - -template<> inline uint16_t saturate_cast(uint8_t x) { return x; } -template<> inline float saturate_cast(uint8_t x) { return x; } -template<> inline uint8_t saturate_cast(uint8_t x) { return x; } - -template<> inline uint8_t saturate_cast(uint16_t x) { - using lim = std::numeric_limits; - return (uint8_t)std::min(static_cast(lim::max()), - std::max(static_cast(lim::min()), x)); -} -template<> inline uint8_t saturate_cast(float x) { return saturate_cast(static_cast(std::rint(x))); } - -template<> inline float saturate_cast(double x) { return static_cast(x); } -//------------------------------------------------------------------------------ - -constexpr static const int ONE = 1 << 15; - -inline static uint8_t calc(short alpha0, uint8_t src0, short alpha1, uint8_t src1) { - constexpr static const int half = 1 << 14; - return (src0 * alpha0 + src1 * alpha1 + half) >> 15; -} - -inline static float calc(float alpha0, float src0, float alpha1, float src1) { - return src0 * alpha0 + src1 * alpha1; -} - -//------------------------------------------------------------------------------ - -// Variants: -// - F=float, I=int -// - F=short, I=short (e.g. F is Q1.7.8 encoded with short) -template -struct MapperUnit { - F alpha0, alpha1; - I index0, index1; -}; - -//------------------------------------------------------------------------------ - -typedef uint16_t Q0_16; // value [0..1) with 16 fractional bits -typedef uint16_t Q8_8; // value [0..255) with 8 fractional bits -typedef uint8_t U8; // value [0..255) - -typedef MapperUnit MapperUnit32F; -typedef MapperUnit MapperUnit8U; - -template static inline DST convert_cast(SRC x); -template<> inline uint8_t convert_cast(uint8_t x) { return x; } -template<> inline uint8_t convert_cast(float x) { return static_cast(x); } -template<> inline float convert_cast(float x) { return x; } -template<> inline float convert_cast(double x) { return static_cast(x); } -template<> inline Q0_16 convert_cast(double x) { - int ix = static_cast(std::rint(x * (1 << 16))); - return saturate_cast(ix); -} -template<> inline Q8_8 convert_cast(uchar x) { return x << 8; } -template<> inline uchar convert_cast(Q8_8 x) { return x >> 8; } - -template static inline DST checked_cast(SRC x) { - short dx = static_cast(x); - GAPI_Assert(x == dx); // check - return dx; -} - -static inline Q8_8 mulas(Q0_16 a, U8 s) { return static_cast((a * s) >> 8); } -static inline Q8_8 mulaw(Q0_16 a, Q8_8 w) { return static_cast((static_cast(a) * w) >> 16); } - -static inline float mulas(float a, float s) { return a * s; } -static inline float mulaw(float a, float w) { return a * w; } - -static const int ITUR_BT_601_CY = 1220542; -static const int ITUR_BT_601_CUB = 2116026; -static const int ITUR_BT_601_CUG = -409993; -static const int ITUR_BT_601_CVG = -852492; -static const int ITUR_BT_601_CVR = 1673527; -static const int ITUR_BT_601_SHIFT = 20; - -static inline void uvToRGBuv(const uchar u, const uchar v, int& ruv, int& guv, int& buv) { - int uu, vv; - uu = static_cast(u) - 128; - vv = static_cast(v) - 128; - - ruv = (1 << (ITUR_BT_601_SHIFT - 1)) + ITUR_BT_601_CVR * vv; - guv = (1 << (ITUR_BT_601_SHIFT - 1)) + ITUR_BT_601_CVG * vv + ITUR_BT_601_CUG * uu; - buv = (1 << (ITUR_BT_601_SHIFT - 1)) + ITUR_BT_601_CUB * uu; -} - -static inline void yRGBuvToRGB(const uchar vy, const int ruv, const int guv, const int buv, - uchar& r, uchar& g, uchar& b) { - int yy = static_cast(vy); - int y = std::max(0, yy - 16) * ITUR_BT_601_CY; - r = saturate_cast((y + ruv) >> ITUR_BT_601_SHIFT); - g = saturate_cast((y + guv) >> ITUR_BT_601_SHIFT); - b = saturate_cast((y + buv) >> ITUR_BT_601_SHIFT); -} -} // namespace kernels -} // namespace gapi -} // namespace InferenceEngine diff --git a/src/common/preprocessing/src/ie_preprocess_gapi_kernels_simd_impl.hpp b/src/common/preprocessing/src/ie_preprocess_gapi_kernels_simd_impl.hpp deleted file mode 100644 index 99f7d968771..00000000000 --- a/src/common/preprocessing/src/ie_preprocess_gapi_kernels_simd_impl.hpp +++ /dev/null @@ -1,765 +0,0 @@ -// Copyright (C) 2018-2023 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#ifndef IE_PREPROCESS_GAPI_KERNELS_SIMD_IMPL_H -#define IE_PREPROCESS_GAPI_KERNELS_SIMD_IMPL_H - -#include -#include - -#include "ie_preprocess_gapi_kernels_impl.hpp" - -using namespace cv; - -namespace InferenceEngine { - -namespace gapi { - -namespace kernels { - -template -CV_ALWAYS_INLINE void mergeRowC2_Impl(const T in0[], const T in1[], - T out[], const int length) { - int x = 0; - -#if MANUAL_SIMD - constexpr int nlanes = VecT::nlanes; - GAPI_DbgAssert(length >= nlanes); - - VecT r0, r1; - for (; length >= nlanes;) { - for (; x <= length - nlanes; x += nlanes) { - r0 = vx_load(&in0[x]); - r1 = vx_load(&in1[x]); - v_store_interleave(&out[2*x], r0, r1); - } - - if (x < length) { - x = length - nlanes; - continue; - } - break; - } -#endif - - for (; x < length; ++x) { - out[2*x + 0] = in0[x]; - out[2*x + 1] = in1[x]; - } -} - -template -CV_ALWAYS_INLINE void mergeRowC3_Impl(const T in0[], const T in1[], - const T in2[], T out[], const int length) { - int x = 0; - -#if MANUAL_SIMD - constexpr int nlanes = VecT::nlanes; - GAPI_DbgAssert(length >= nlanes); - - VecT r0, r1, r2; - for (; length >= nlanes;) { - for (; x <= length - nlanes; x += nlanes) { - r0 = vx_load(&in0[x]); - r1 = vx_load(&in1[x]); - r2 = vx_load(&in2[x]); - v_store_interleave(&out[3*x], r0, r1, r2); - } - - if (x < length) { - x = length - nlanes; - continue; - } - break; - } -#endif - - for (; x < length; ++x) { - out[3*x + 0] = in0[x]; - out[3*x + 1] = in1[x]; - out[3*x + 2] = in2[x]; - } -} - -template -CV_ALWAYS_INLINE void mergeRowC4_Impl(const T in0[], const T in1[], - const T in2[], const T in3[], - T out[], const int length) { - int x = 0; - -#if MANUAL_SIMD - constexpr int nlanes = VecT::nlanes; - GAPI_DbgAssert(length >= nlanes); - - VecT r0, r1, r2, r3; - for (; length >= nlanes;) { - for (; x <= length - nlanes; x += nlanes) { - r0 = vx_load(&in0[x]); - r1 = vx_load(&in1[x]); - r2 = vx_load(&in2[x]); - r3 = vx_load(&in3[x]); - v_store_interleave(&out[4* x], r0, r1, r2, r3); - } - - if (x < length) { - x = length - nlanes; - continue; - } - break; - } -#endif - - for (; x < length; ++x) { - out[4*x + 0] = in0[x]; - out[4*x + 1] = in1[x]; - out[4*x + 2] = in2[x]; - out[4*x + 3] = in3[x]; - } -} -//------------------------------------------------------------------------------ -template -CV_ALWAYS_INLINE void splitRowC2_Impl(const T in[], T out0[], - T out1[], const int length) { - int x = 0; - -#if MANUAL_SIMD - constexpr int nlanes = VecT::nlanes; - GAPI_DbgAssert(length >= nlanes); - - VecT r0, r1; - for (; length >= nlanes;) { - for (; x <= length - nlanes; x += nlanes) { - v_load_deinterleave(&in[2*x], r0, r1); - vx_store(&out0[x], r0); - vx_store(&out1[x], r1); - } - - if (x < length) { - x = length - nlanes; - continue; - } - break; - } -#endif - - for (; x < length; ++x) { - out0[x] = in[2*x + 0]; - out1[x] = in[2*x + 1]; - } -} - -template -CV_ALWAYS_INLINE void splitRowC3_Impl(const T in[], T out0[], - T out1[], T out2[], const int length) { - int x = 0; - -#if MANUAL_SIMD - constexpr int nlanes = VecT::nlanes; - - VecT r0, r1, r2; - for (; length >= nlanes;) { - for (; x <= length - nlanes; x += nlanes) { - v_load_deinterleave(&in[3*x], r0, r1, r2); - vx_store(&out0[x], r0); - vx_store(&out1[x], r1); - vx_store(&out2[x], r2); - } - - if (x < length) { - x = length - nlanes; - continue; - } - break; - } -#endif - - for (; x < length; ++x) { - out0[x] = in[3*x + 0]; - out1[x] = in[3*x + 1]; - out2[x] = in[3*x + 2]; - } -} - -template -CV_ALWAYS_INLINE void splitRowC4_Impl(const T in[], T out0[], T out1[], - T out2[], T out3[], const int length) { - int x = 0; - -#if MANUAL_SIMD - constexpr int nlanes = VecT::nlanes; - GAPI_DbgAssert(length >= nlanes); - - VecT r0, r1, r2, r3; - for (; length >= nlanes;) { - for (; x <= length - nlanes; x += nlanes) { - v_load_deinterleave(&in[4*x], r0, r1, r2, r3); - vx_store(&out0[x], r0); - vx_store(&out1[x], r1); - vx_store(&out2[x], r2); - vx_store(&out3[x], r3); - } - - if (x < length) { - x = length - nlanes; - continue; - } - break; - } -#endif - - for (; x < length; ++x) { - out0[x] = in[4*x + 0]; - out1[x] = in[4*x + 1]; - out2[x] = in[4*x + 2]; - out3[x] = in[4*x + 3]; - } -} -//------------------------------------------------------------------------------ - -CV_ALWAYS_INLINE void uvToRGBuv(const v_uint8& u, const v_uint8& v, - v_int32 (&ruv)[4], v_int32 (&guv)[4], - v_int32 (&buv)[4]) { - v_uint8 v128 = vx_setall_u8(128); - v_int8 su = v_reinterpret_as_s8(v_sub_wrap(u, v128)); - v_int8 sv = v_reinterpret_as_s8(v_sub_wrap(v, v128)); - - v_int16 uu0, uu1, vv0, vv1; - v_expand(su, uu0, uu1); - v_expand(sv, vv0, vv1); - v_int32 uu[4], vv[4]; - v_expand(uu0, uu[0], uu[1]); v_expand(uu1, uu[2], uu[3]); - v_expand(vv0, vv[0], vv[1]); v_expand(vv1, vv[2], vv[3]); - - v_int32 vshift = vx_setall_s32(1 << (ITUR_BT_601_SHIFT - 1)); - v_int32 vr = vx_setall_s32(ITUR_BT_601_CVR); - v_int32 vg = vx_setall_s32(ITUR_BT_601_CVG); - v_int32 ug = vx_setall_s32(ITUR_BT_601_CUG); - v_int32 ub = vx_setall_s32(ITUR_BT_601_CUB); - - for (int k = 0; k < 4; k++) { - ruv[k] = vshift + vr * vv[k]; - guv[k] = vshift + vg * vv[k] + ug * uu[k]; - buv[k] = vshift + ub * uu[k]; - } -} - -CV_ALWAYS_INLINE void yRGBuvToRGB(const v_uint8& vy, - const v_int32 (&ruv)[4], - const v_int32 (&guv)[4], - const v_int32 (&buv)[4], - v_uint8& rr, v_uint8& gg, v_uint8& bb) { - v_uint8 v16 = vx_setall_u8(16); - v_uint8 posY = vy - v16; - v_uint16 yy0, yy1; - v_expand(posY, yy0, yy1); - v_int32 yy[4]; - v_int32 yy00, yy01, yy10, yy11; - v_expand(v_reinterpret_as_s16(yy0), yy[0], yy[1]); - v_expand(v_reinterpret_as_s16(yy1), yy[2], yy[3]); - - v_int32 vcy = vx_setall_s32(ITUR_BT_601_CY); - - v_int32 y[4], r[4], g[4], b[4]; - for (int k = 0; k < 4; k++) { - y[k] = yy[k]*vcy; - r[k] = (y[k] + ruv[k]) >> ITUR_BT_601_SHIFT; - g[k] = (y[k] + guv[k]) >> ITUR_BT_601_SHIFT; - b[k] = (y[k] + buv[k]) >> ITUR_BT_601_SHIFT; - } - - v_int16 r0, r1, g0, g1, b0, b1; - r0 = v_pack(r[0], r[1]); - r1 = v_pack(r[2], r[3]); - g0 = v_pack(g[0], g[1]); - g1 = v_pack(g[2], g[3]); - b0 = v_pack(b[0], b[1]); - b1 = v_pack(b[2], b[3]); - - rr = v_pack_u(r0, r1); - gg = v_pack_u(g0, g1); - bb = v_pack_u(b0, b1); -} - -template -CV_ALWAYS_INLINE void nv12ToRgbRowImpl(isa_tag_t, const uchar** srcY, const uchar* srcUV, - uchar** dstRGBx, const int width) { - int i = 0; - -#if MANUAL_SIMD - constexpr int nlanes = v_uint8::nlanes; - - for (; i <= width - 2 * nlanes; i += 2 * nlanes) { - v_uint8 u, v; - v_load_deinterleave(srcUV + i, u, v); - - v_uint8 vy[4]; - v_load_deinterleave(srcY[0] + i, vy[0], vy[1]); - v_load_deinterleave(srcY[1] + i, vy[2], vy[3]); - - v_int32 ruv[4], guv[4], buv[4]; - uvToRGBuv(u, v, ruv, guv, buv); - - v_uint8 r[4], g[4], b[4]; - - for (int k = 0; k < 4; k++) { - yRGBuvToRGB(vy[k], ruv, guv, buv, r[k], g[k], b[k]); - } - - for (int k = 0; k < 4; k++) - std::swap(r[k], b[k]); - - // [r0...], [r1...] => [r0, r1, r0, r1...], [r0, r1, r0, r1...] - v_uint8 r0_0, r0_1, r1_0, r1_1; - v_zip(r[0], r[1], r0_0, r0_1); - v_zip(r[2], r[3], r1_0, r1_1); - v_uint8 g0_0, g0_1, g1_0, g1_1; - v_zip(g[0], g[1], g0_0, g0_1); - v_zip(g[2], g[3], g1_0, g1_1); - v_uint8 b0_0, b0_1, b1_0, b1_1; - v_zip(b[0], b[1], b0_0, b0_1); - v_zip(b[2], b[3], b1_0, b1_1); - - v_store_interleave(dstRGBx[0] + i * 3, b0_0, g0_0, r0_0); - v_store_interleave(dstRGBx[0] + i * 3 + 3 * nlanes, b0_1, g0_1, r0_1); - - v_store_interleave(dstRGBx[1] + i * 3, b1_0, g1_0, r1_0); - v_store_interleave(dstRGBx[1] + i * 3 + 3 * nlanes, b1_1, g1_1, r1_1); - } - //vx_cleanup(); -#endif - - for (; i < width; i += 2) { - uchar u = srcUV[i]; - uchar v = srcUV[i + 1]; - int ruv, guv, buv; - uvToRGBuv(u, v, ruv, guv, buv); - - for (int y = 0; y < 2; y++) { - for (int x = 0; x < 2; x++) { - uchar vy = srcY[y][i + x]; - uchar r, g, b; - yRGBuvToRGB(vy, ruv, guv, buv, r, g, b); - - dstRGBx[y][3 * (i + x)] = r; - dstRGBx[y][3 * (i + x) + 1] = g; - dstRGBx[y][3 * (i + x) + 2] = b; - } - } - } -} - -template -CV_ALWAYS_INLINE void i420ToRgbRowImpl(isa_tag_t, const uint8_t** srcY, const uint8_t* srcU, - const uint8_t* srcV, uint8_t** dstRGBx, const int width) { - int i = 0; - -#if MANUAL_SIMD - constexpr int nlanes = v_uint8::nlanes; - - for (; i <= width - 2 * nlanes; i += 2 * nlanes) { - v_uint8 u = vx_load(srcU + i / 2); - v_uint8 v = vx_load(srcV + i / 2); - - v_uint8 vy[4]; - v_load_deinterleave(srcY[0] + i, vy[0], vy[1]); - v_load_deinterleave(srcY[1] + i, vy[2], vy[3]); - - v_int32 ruv[4], guv[4], buv[4]; - uvToRGBuv(u, v, ruv, guv, buv); - - v_uint8 r[4], g[4], b[4]; - - for (int k = 0; k < 4; k++) { - yRGBuvToRGB(vy[k], ruv, guv, buv, r[k], g[k], b[k]); - } - - for (int k = 0; k < 4; k++) - std::swap(r[k], b[k]); - - // [r0...], [r1...] => [r0, r1, r0, r1...], [r0, r1, r0, r1...] - v_uint8 r0_0, r0_1, r1_0, r1_1; - v_zip(r[0], r[1], r0_0, r0_1); - v_zip(r[2], r[3], r1_0, r1_1); - v_uint8 g0_0, g0_1, g1_0, g1_1; - v_zip(g[0], g[1], g0_0, g0_1); - v_zip(g[2], g[3], g1_0, g1_1); - v_uint8 b0_0, b0_1, b1_0, b1_1; - v_zip(b[0], b[1], b0_0, b0_1); - v_zip(b[2], b[3], b1_0, b1_1); - - v_store_interleave(dstRGBx[0] + i * 3, b0_0, g0_0, r0_0); - v_store_interleave(dstRGBx[0] + i * 3 + 3 * nlanes, b0_1, g0_1, r0_1); - - v_store_interleave(dstRGBx[1] + i * 3, b1_0, g1_0, r1_0); - v_store_interleave(dstRGBx[1] + i * 3 + 3 * nlanes, b1_1, g1_1, r1_1); - } - //vx_cleanup(); -#endif - for (; i < width; i += 2) { - uchar u = srcU[i / 2]; - uchar v = srcV[i / 2]; - int ruv, guv, buv; - uvToRGBuv(u, v, ruv, guv, buv); - - for (int y = 0; y < 2; y++) { - for (int x = 0; x < 2; x++) { - uchar vy = srcY[y][i + x]; - uchar r, g, b; - yRGBuvToRGB(vy, ruv, guv, buv, r, g, b); - - dstRGBx[y][3 * (i + x)] = r; - dstRGBx[y][3 * (i + x) + 1] = g; - dstRGBx[y][3 * (i + x) + 2] = b; - } - } - } -} - -//------------------------------------------------------------------------------ - -// vertical pass -template -CV_ALWAYS_INLINE void downy(const T *src[], int inWidth, const MapperUnit& ymap, - A yalpha, W vbuf[]) { - int y_1st = ymap.index0; - int ylast = ymap.index1 - 1; - - // yratio > 1, so at least 2 rows - GAPI_DbgAssert(y_1st < ylast); - -#if MANUAL_SIMD - constexpr int nlanes = v_uint16::nlanes; -#endif - - // 1st and last rows - { - int w = 0; - -#if MANUAL_SIMD - if (std::is_same::value) { - for (; w <= inWidth - nlanes; w += nlanes) { - v_uint16 vsrc0 = vx_load_expand(reinterpret_cast(& src[0][w])); - v_uint16 vsrc1 = vx_load_expand(reinterpret_cast(& src[ylast - y_1st][w])); - v_uint16 vres = v_mulhi(vsrc0 << 8, static_cast(ymap.alpha0)) + - v_mulhi(vsrc1 << 8, static_cast(ymap.alpha1)); - vx_store(reinterpret_cast(& vbuf[w]), vres); - } - } -#endif - - for (; w < inWidth; w++) { - vbuf[w] = mulas(ymap.alpha0, src[0][w]) - + mulas(ymap.alpha1, src[ylast - y_1st][w]); - } - } - - // inner rows (if any) - for (int i = 1; i < ylast - y_1st; i++) { - int w = 0; - -#if MANUAL_SIMD - if (std::is_same::value) { - for (; w <= inWidth - nlanes; w += nlanes) { - v_uint16 vsrc = vx_load_expand(reinterpret_cast(& src[i][w])); - v_uint16 vres = vx_load(reinterpret_cast(& vbuf[w])); - vres = vres + v_mulhi(vsrc << 8, static_cast(yalpha)); - vx_store(reinterpret_cast(& vbuf[w]), vres); - } - } -#endif - - for (; w < inWidth; w++) { - vbuf[w] += mulas(yalpha, src[i][w]); - } - } -} - -// horizontal pass -template -CV_ALWAYS_INLINE void downx(T dst[], int outWidth, int xmaxdf, const I xindex[], - const A xalpha[], const W vbuf[]) { -// TO DO: try lambda here -#define HSUM(xmaxdf) \ - for (int x = 0; x < outWidth; x++) { \ - int index = xindex[x]; \ - const A *alpha = &xalpha[x * xmaxdf]; \ - \ - W sum = 0; \ - for (int i = 0; i < xmaxdf; i++) { \ - sum += mulaw(alpha[i], vbuf[index + i]); \ - } \ - \ - dst[x] = convert_cast(sum); \ - } - - if (2 == xmaxdf) { - HSUM(2); - } else if (3 == xmaxdf) { - HSUM(3); - } else if (4 == xmaxdf) { - HSUM(4); - } else if (5 == xmaxdf) { - HSUM(5); - } else if (6 == xmaxdf) { - HSUM(6); - } else if (7 == xmaxdf) { - HSUM(7); - } else if (8 == xmaxdf) { - HSUM(8); - } else { - HSUM(xmaxdf); - } -#undef HSUM -} - -template -CV_ALWAYS_INLINE void calcRowAreaImpl(isa_tag_t, T dst[], const T *src[], - const Size& inSz, const Size& outSz, A yalpha, - const MapperUnit& ymap, int xmaxdf, - const I xindex[], const A xalpha[], W vbuf[]) { - bool xRatioEq1 = inSz.width == outSz.width; - bool yRatioEq1 = inSz.height == outSz.height; - - if (!yRatioEq1 && !xRatioEq1) { - downy(src, inSz.width, ymap, yalpha, vbuf); - downx(dst, outSz.width, xmaxdf, xindex, xalpha, vbuf); - - } else if (!yRatioEq1) { - GAPI_DbgAssert(xRatioEq1); - downy(src, inSz.width, ymap, yalpha, vbuf); - for (int x = 0; x < outSz.width; x++) { - dst[x] = convert_cast(vbuf[x]); - } - - } else if (!xRatioEq1) { - GAPI_DbgAssert(yRatioEq1); - for (int w = 0; w < inSz.width; w++) { - vbuf[w] = convert_cast(src[0][w]); - } - downx(dst, outSz.width, xmaxdf, xindex, xalpha, vbuf); - - } else { - GAPI_DbgAssert(xRatioEq1 && yRatioEq1); - memcpy(dst, src[0], outSz.width * sizeof(T)); - } -} - -//------------------------------------------------------------------------------ - -template -CV_ALWAYS_INLINE void copyRow_Impl(const T in[], T out[], int length) { - int l = 0; - -#if MANUAL_SIMD - const int nlanes = VecT::nlanes; - - auto copy_row = [](const T in[], T out[], int l) { - VecT r = vx_load(&in[l]); - vx_store(&out[l], r); - }; - - for (; l <= length - nlanes; l += nlanes) { - copy_row(in, out, l); - } - - if (l < length && length >= nlanes) { - copy_row(in, out, length - nlanes); - l = length; - } -#endif - - for (; l < length; l++) { - out[l] = in[l]; - } -} - -// Resize (bi-linear, 32FC1) -template -CV_ALWAYS_INLINE void calcRowLinear32FC1Impl(isa_tag_t, - float *dst[], - const float *src0[], - const float *src1[], - const float alpha[], - const int mapsx[], - const float beta[], - const Size& inSz, - const Size& outSz, - const int lpi, - const int) { - bool xRatioEq1 = inSz.width == outSz.width; - bool yRatioEq1 = inSz.height == outSz.height; - -#if MANUAL_SIMD - constexpr int nlanes = v_float32::nlanes; -#endif - - if (!xRatioEq1 && !yRatioEq1) { - for (int line = 0; line < lpi; ++line) { - float beta0 = beta[line]; - float beta1 = 1 - beta0; - - int x = 0; - -#if MANUAL_SIMD - v_float32 low1, high1, s00, s01; - v_float32 low2, high2, s10, s11; - for (; x <= outSz.width - nlanes; x += nlanes) { - v_float32 alpha0 = vx_load(&alpha[x]); - // v_float32 alpha1 = 1.f - alpha0; - - v_gather_pairs(src0[line], mapsx, x, low1, high1); - v_deinterleave(low1, high1, s00, s01); - - // v_float32 res0 = s00*alpha0 + s01*alpha1; - v_float32 res0 = v_fma(s00 - s01, alpha0, s01); - - v_gather_pairs(src1[line], mapsx, x, low2, high2); - v_deinterleave(low2, high2, s10, s11); - - // v_float32 res1 = s10*alpha0 + s11*alpha1; - v_float32 res1 = v_fma(s10 - s11, alpha0, s11); - - // v_float32 d = res0*beta0 + res1*beta1; - v_float32 d = v_fma(res0 - res1, beta0, res1); - - vx_store(&dst[line][x], d); - } -#endif - - for (; x < outSz.width; ++x) { - float alpha0 = alpha[x]; - float alpha1 = 1 - alpha0; - int sx0 = mapsx[x]; - int sx1 = sx0 + 1; - float res0 = src0[line][sx0] * alpha0 + src0[line][sx1] * alpha1; - float res1 = src1[line][sx0] * alpha0 + src1[line][sx1] * alpha1; - dst[line][x] = beta0 * res0 + beta1 * res1; - } - } - - } else if (!xRatioEq1) { - GAPI_DbgAssert(yRatioEq1); - - for (int line = 0; line < lpi; ++line) { - int x = 0; - -#if MANUAL_SIMD - v_float32 low, high, s00, s01; - for (; x <= outSz.width - nlanes; x += nlanes) { - v_float32 alpha0 = vx_load(&alpha[x]); - // v_float32 alpha1 = 1.f - alpha0; - - v_gather_pairs(src0[line], mapsx, x, low, high); - v_deinterleave(low, high, s00, s01); - - // v_float32 d = s00*alpha0 + s01*alpha1; - v_float32 d = v_fma(s00 - s01, alpha0, s01); - - vx_store(&dst[line][x], d); - } -#endif - - for (; x < outSz.width; ++x) { - float alpha0 = alpha[x]; - float alpha1 = 1 - alpha0; - int sx0 = mapsx[x]; - int sx1 = sx0 + 1; - dst[line][x] = src0[line][sx0] * alpha0 + src0[line][sx1] * alpha1; - } - } - - } else if (!yRatioEq1) { - GAPI_DbgAssert(xRatioEq1); - int length = inSz.width; // == outSz.width - - for (int line = 0; line < lpi; ++line) { - float beta0 = beta[line]; - float beta1 = 1 - beta0; - - int x = 0; - -#if MANUAL_SIMD - for (; x <= length - nlanes; x += nlanes) { - v_float32 s0 = vx_load(&src0[line][x]); - v_float32 s1 = vx_load(&src1[line][x]); - - // v_float32 d = s0*beta0 + s1*beta1; - v_float32 d = v_fma(s0 - s1, beta0, s1); - - vx_store(&dst[line][x], d); - } -#endif - - for (; x < length; ++x) { - dst[line][x] = beta0 * src0[line][x] + beta1 * src1[line][x]; - } - } - - } else { - GAPI_DbgAssert(xRatioEq1 && yRatioEq1); - int length = inSz.width; // == outSz.width - for (int line = 0; line < lpi; ++line) { - memcpy(dst[line], src0[line], length * sizeof(float)); - } - } -} - -template -struct vector_type_of; - -template -using vector_type_of_t = typename vector_type_of::type; - -template struct vector_type_of { using type = v_uint8; }; -template struct vector_type_of { using type = v_float32;}; - -template -CV_ALWAYS_INLINE void chanToPlaneRowImpl(isa_tag_t, const T* in, const int chan, - const int chs, T* out, const int length) { - if (chs == 1) { - copyRow_Impl, T>(in, out, length); - return; - } - - for (int x = 0; x < length; x++) { - out[x] = in[x*chs + chan]; - } -} - -template -CV_ALWAYS_INLINE void splitRowImpl(isa_tag_t, const T* in, std::array& outs, const int length) { - static_assert(chs > 1 && chs < 5, "This number of channels isn't supported."); - - if (chs == 2) { - splitRowC2_Impl, T>(in, outs[0], outs[1], length); - return; - } else if (chs == 3) { - splitRowC3_Impl, T>(in, outs[0], outs[1], outs[2], length); - return; - } else { - splitRowC4_Impl, T>(in, outs[0], outs[1], outs[2], outs[3], length); - return; - } -} - -template -CV_ALWAYS_INLINE void mergeRowImpl(isa_tag_t, const std::array& ins, T* out, const int length) { - static_assert(chs > 1 && chs < 5, "This number of channels isn't supported."); - - if (chs == 2) { - mergeRowC2_Impl, T>(ins[0], ins[1], out, length); - return; - } else if (chs == 3) { - mergeRowC3_Impl, T>(ins[0], ins[1], ins[2], out, length); - return; - } else { - mergeRowC4_Impl, T>(ins[0], ins[1], ins[2], ins[3], out, length); - return; - } -} -} // namespace kernels -} // namespace gapi -} // namespace InferenceEngine - -#endif // IE_PREPROCESS_GAPI_KERNELS_SIMD_IMPL_H diff --git a/src/common/preprocessing/src/ie_preprocess_itt.hpp b/src/common/preprocessing/src/ie_preprocess_itt.hpp deleted file mode 100644 index 037bc45457b..00000000000 --- a/src/common/preprocessing/src/ie_preprocess_itt.hpp +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright (C) 2018-2023 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -/** - * @brief Defines openvino domains for tracing - * @file ie_preprocess_itt.hpp - */ - -#pragma once - -#include - -namespace InferenceEngine { -namespace itt { -namespace domains { - OV_ITT_DOMAIN(IEPreproc); -} -} -} diff --git a/src/common/preprocessing/tests/CMakeLists.txt b/src/common/preprocessing/tests/CMakeLists.txt deleted file mode 100644 index f3a754e7673..00000000000 --- a/src/common/preprocessing/tests/CMakeLists.txt +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright (C) 2018-2023 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 -# - -set(TARGET fluid_preproc_tests) - -ov_disable_deprecated_warnings() - -find_package(OpenCV QUIET COMPONENTS gapi core imgproc) -if(NOT OpenCV_FOUND OR NOT OpenCV_VERSION VERSION_GREATER_EQUAL 4) - message(WARNING "OpenCV ver. 4.0+ is not found, ${TARGET} skipped") - return() -endif() - -add_subdirectory(fluid_test_computations) - -file(GLOB SOURCES *.cpp common/*.cpp cpu/*.cpp) -file(GLOB HEADERS *.hpp common/*.hpp cpu/*.hpp) - -add_executable(${TARGET} ${SOURCES} ${HEADERS}) - -target_include_directories(${TARGET} PRIVATE - "${CMAKE_CURRENT_SOURCE_DIR}/common" - "${CMAKE_CURRENT_SOURCE_DIR}/cpu" - $ - $) - -target_compile_definitions(${TARGET} PRIVATE - $) - -target_link_libraries(${TARGET} PRIVATE opencv_core opencv_imgproc openvino::util - inference_engine fluid_test_computations gtest gtest_main) - -if(GAPI_TEST_PERF) - target_compile_definitions(${TARGET} PRIVATE -DPERF_TEST=1) -else() - target_compile_definitions(${TARGET} PRIVATE -DPERF_TEST=0) -endif() - -add_test(NAME ${TARGET} COMMAND ${TARGET}) -set_property(TEST ${TARGET} PROPERTY LABELS IE PREPROC) - -install(TARGETS ${TARGET} - RUNTIME DESTINATION tests - COMPONENT tests - EXCLUDE_FROM_ALL) diff --git a/src/common/preprocessing/tests/common/fluid_tests.cpp b/src/common/preprocessing/tests/common/fluid_tests.cpp deleted file mode 100644 index 5644c7ee594..00000000000 --- a/src/common/preprocessing/tests/common/fluid_tests.cpp +++ /dev/null @@ -1,1138 +0,0 @@ -// Copyright (C) 2018-2023 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#include "fluid_tests.hpp" - -#include "blob_factory.hpp" -#include "blob_transform.hpp" -#include "ie_preprocess.hpp" -#include "ie_preprocess_data.hpp" -#include "ie_compound_blob.h" - -#include -#include -#include -#include - -#include -#include -#include - -#include - -#include - -#include - -#include - -// Can be set externally (via CMake) if built with -DGAPI_TEST_PERF=ON -#ifndef PERF_TEST -#define PERF_TEST 0 // 1=test performance, 0=don't -#endif - -namespace { -#if PERF_TEST -// performance test: iterate function, measure and print milliseconds per call -template void test_ms(F func, int iter, const char format[], ...) -{ - using std::chrono::high_resolution_clock; - - std::vector samples(iter); samples.clear(); - if (0 == iter) - return; - - for (int i=0; i < iter; i++) - { - auto start = high_resolution_clock::now(); - func(); // iterate calls - samples.push_back(high_resolution_clock::now() - start); - } - - std::sort(samples.begin(), samples.end()); - - auto median = samples[samples.size() / 2]; - - double median_ms = std::chrono::duration_cast(median).count() * 0.001; // convert to milliseconds - - printf("Performance(ms): %lg ", median_ms); - - va_list args; - va_start(args, format); - vprintf(format, args); - va_end(args); - - printf("\n"); -} - -cv::String interpToString(int interp) -{ - switch(interp) - { - case cv::INTER_AREA : return "INTER_AREA"; - case cv::INTER_LINEAR : return "INTER_LINEAR"; - case cv::INTER_NEAREST: return "INTER_NEAREST"; - } - CV_Assert(!"ERROR: unsupported interpolation!"); - return nullptr; -} - -cv::String depthToString(int depth) -{ - switch(depth) - { - case CV_8U : return "CV_8U"; - case CV_32F : return "CV_32F"; - case CV_16U : return "CV_16U"; - } - CV_Assert(!"ERROR: unsupported depth!"); - return nullptr; -} - -cv::String typeToString(int type) -{ - switch(type) - { - case CV_8UC1 : return "CV_8UC1"; - case CV_8UC2 : return "CV_8UC2"; - case CV_8UC3 : return "CV_8UC3"; - case CV_8UC4 : return "CV_8UC4"; - case CV_32FC1 : return "CV_32FC1"; - case CV_32FC2 : return "CV_32FC2"; - case CV_32FC3 : return "CV_32FC3"; - case CV_32FC4 : return "CV_32FC4"; - case CV_8SC1 : return "CV_8SC1"; - case CV_8SC2 : return "CV_8SC2"; - case CV_8SC3 : return "CV_8SC3"; - case CV_8SC4 : return "CV_8SC4"; - case CV_16SC1 : return "CV_16SC1"; - case CV_16SC2 : return "CV_16SC2"; - case CV_16SC3 : return "CV_16SC3"; - case CV_16SC4 : return "CV_16SC4"; - case CV_16UC1 : return "CV_16UC1"; - case CV_16UC2 : return "CV_16UC2"; - case CV_16UC3 : return "CV_16UC3"; - case CV_16UC4 : return "CV_16UC4"; - case CV_32SC1 : return "CV_32SC1"; - case CV_32SC2 : return "CV_32SC2"; - case CV_32SC3 : return "CV_32SC3"; - case CV_32SC4 : return "CV_32SC4"; - } - CV_Assert(!"ERROR: unsupported type!"); - return nullptr; -} - -cv::String colorFormatToString(InferenceEngine::ColorFormat f) { - using namespace InferenceEngine; - switch (f) - { - case ColorFormat::RAW: return "RAW"; - case ColorFormat::RGB: return "RGB"; - case ColorFormat::BGR: return "BGR"; - case ColorFormat::RGBX: return "RGBX"; - case ColorFormat::BGRX: return "BGRX"; - default: IE_THROW() << "Unrecognized color format"; - } -} - -cv::String layoutToString(InferenceEngine::Layout l) { - using namespace InferenceEngine; - switch (l) { - case Layout::NCHW: return "NCHW"; - case Layout::NHWC: return "NHWC"; - default: return "?"; - } -} -#endif // PERF_TEST - -test::Mat to_test(cv::Mat& mat) { return {mat.rows, mat.cols, mat.type(), mat.data, mat.step}; } -std::vector to_test(std::vector& mats) -{ - std::vector test_mats(mats.size()); - for (size_t i = 0; i < mats.size(); i++) { - test_mats[i] = to_test(mats[i]); - } - return test_mats; -} - -test::Rect to_test(cv::Rect& rect) { return {rect.x, rect.y, rect.width, rect.height}; } -test::Scalar to_test(cv::Scalar const& sc) { return {sc[0], sc[1], sc[2], sc[3]}; } - -cv::ColorConversionCodes toCvtColorCode(InferenceEngine::ColorFormat in, - InferenceEngine::ColorFormat out) { - using namespace InferenceEngine; - IE_SUPPRESS_DEPRECATED_START - static const std::map, cv::ColorConversionCodes> types = { - {{ColorFormat::RGBX, ColorFormat::BGRX}, cv::COLOR_RGBA2BGRA}, - {{ColorFormat::RGBX, ColorFormat::BGR}, cv::COLOR_RGBA2BGR}, - {{ColorFormat::RGBX, ColorFormat::RGB}, cv::COLOR_RGBA2RGB}, - {{ColorFormat::BGRX, ColorFormat::RGBX}, cv::COLOR_BGRA2RGBA}, - {{ColorFormat::BGRX, ColorFormat::BGR}, cv::COLOR_BGRA2BGR}, - {{ColorFormat::BGRX, ColorFormat::RGB}, cv::COLOR_BGRA2RGB}, - {{ColorFormat::RGB, ColorFormat::RGBX}, cv::COLOR_RGB2RGBA}, - {{ColorFormat::RGB, ColorFormat::BGRX}, cv::COLOR_RGB2BGRA}, - {{ColorFormat::RGB, ColorFormat::BGR}, cv::COLOR_RGB2BGR}, - {{ColorFormat::BGR, ColorFormat::RGBX}, cv::COLOR_BGR2RGBA}, - {{ColorFormat::BGR, ColorFormat::BGRX}, cv::COLOR_BGR2BGRA}, - {{ColorFormat::BGR, ColorFormat::RGB}, cv::COLOR_BGR2RGB}, - }; - IE_SUPPRESS_DEPRECATED_END - return types.at(std::make_pair(in, out)); -} - -cv::ColorConversionCodes toCvtColorCode(InferenceEngine::ColorFormat fmt) { - using namespace InferenceEngine; - // Note: OpenCV matrices are always in BGR format by default - return toCvtColorCode(ColorFormat::BGR, fmt); -} - -int numChannels(InferenceEngine::ColorFormat fmt) { - using namespace InferenceEngine; - switch (fmt) { - // case ColorFormat::RAW: return 0; // any number of channels apply - case ColorFormat::RGB: return 3; - case ColorFormat::BGR: return 3; - case ColorFormat::RGBX: return 4; - case ColorFormat::BGRX: return 4; - default: IE_THROW() << "Unrecognized color format"; - } -} - -// FIXME: Copy-paste from cropRoi tests -template -InferenceEngine::Blob::Ptr img2Blob(cv::Mat &img, InferenceEngine::Layout layout) { - using namespace InferenceEngine; - using data_t = typename PrecisionTrait::value_type; - - const size_t channels = img.channels(); - const size_t height = img.size().height; - const size_t width = img.size().width; - - CV_Assert(cv::DataType::depth == img.depth() || (PRC == Precision::FP16 && img.depth() == CV_16F)); - - SizeVector dims = {1, channels, height, width}; - Blob::Ptr resultBlob = make_shared_blob(TensorDesc(PRC, dims, layout));; - resultBlob->allocate(); - - data_t* blobData = resultBlob->buffer().as(); - - switch (layout) { - case Layout::NCHW: { - for (size_t c = 0; c < channels; c++) { - for (size_t h = 0; h < height; h++) { - for (size_t w = 0; w < width; w++) { - blobData[c * width * height + h * width + w] = img.ptr(h,w)[c]; - } - } - } - } - break; - case Layout::NHWC: { - for (size_t h = 0; h < height; h++) { - for (size_t w = 0; w < width; w++) { - for (size_t c = 0; c < channels; c++) { - blobData[h * width * channels + w * channels + c] = img.ptr(h,w)[c]; - } - } - } - } - break; - default: - IE_THROW() << "Inconsistent input layout for image processing: " << layout; - } - return resultBlob; -} - -template -void Blob2Img(const InferenceEngine::Blob::Ptr& blobP, cv::Mat& img, InferenceEngine::Layout layout) { - using namespace InferenceEngine; - using data_t = typename PrecisionTrait::value_type; - - const size_t channels = img.channels(); - const size_t height = img.size().height; - const size_t width = img.size().width; - - //IE and OpenCV use different data types for FP16 representation, so need to check for it explicitly - CV_Assert(cv::DataType::depth == img.depth() || ((img.depth() == CV_16F) && (PRC == Precision::FP16))); - - data_t* blobData = blobP->buffer().as(); - - switch (layout) { - case Layout::NCHW: { - for (size_t c = 0; c < channels; c++) { - for (size_t h = 0; h < height; h++) { - for (size_t w = 0; w < width; w++) { - img.ptr(h,w)[c] = blobData[c * width * height + h * width + w]; - } - } - } - } - break; - case Layout::NHWC: { - for (size_t h = 0; h < height; h++) { - for (size_t w = 0; w < width; w++) { - for (size_t c = 0; c < channels; c++) { - img.ptr(h,w)[c] = blobData[h * width * channels + w * channels + c]; - } - } - } - } - break; - default: - IE_THROW() << "Inconsistent input layout for image processing: " << layout; - } -} -} // anonymous namespace - -TEST_P(ResizeTestGAPI, AccuracyTest) -{ - int type = 0, interp = 0; - cv::Size sz_in, sz_out; - double tolerance = 0.0; - std::pair sizes; - std::tie(type, interp, sizes, tolerance) = GetParam(); - std::tie(sz_in, sz_out) = sizes; - - cv::Mat in_mat1 (sz_in, type ); - cv::Scalar mean = cv::Scalar::all(127); - cv::Scalar stddev = cv::Scalar::all(40.f); - - cv::randn(in_mat1, mean, stddev); - - cv::Mat out_mat(sz_out, type); - cv::Mat out_mat_ocv(sz_out, type); - - // G-API code ////////////////////////////////////////////////////////////// - FluidResizeComputation rc(to_test(in_mat1), to_test(out_mat), interp); - rc.warmUp(); - -#if PERF_TEST - // iterate testing, and print performance - test_ms([&](){ rc.apply(); }, - 100, "Resize GAPI %s %s %dx%d -> %dx%d", - interpToString(interp).c_str(), typeToString(type).c_str(), - sz_in.width, sz_in.height, sz_out.width, sz_out.height); -#endif - - // OpenCV code ///////////////////////////////////////////////////////////// - { - cv::resize(in_mat1, out_mat_ocv, sz_out, 0, 0, interp); - } - // Comparison ////////////////////////////////////////////////////////////// - { - EXPECT_LE(cv::norm(out_mat, out_mat_ocv, cv::NORM_INF), tolerance); - } -} - -TEST_P(ResizeRGB8UTestGAPI, AccuracyTest) -{ - int type = 0, interp = 0; - cv::Size sz_in, sz_out; - double tolerance = 0.0; - std::pair sizes; - std::tie(type, interp, sizes, tolerance) = GetParam(); - std::tie(sz_in, sz_out) = sizes; - - cv::Mat in_mat1 (sz_in, type ); - cv::Scalar mean = cv::Scalar::all(127); - cv::Scalar stddev = cv::Scalar::all(40.f); - - cv::randn(in_mat1, mean, stddev); - - cv::Mat out_mat(sz_out, type); - cv::Mat out_mat_ocv(sz_out, type); - - // G-API code ////////////////////////////////////////////////////////////// - FluidResizeRGB8UComputation rc(to_test(in_mat1), to_test(out_mat), interp); - rc.warmUp(); - -#if PERF_TEST - // iterate testing, and print performance - test_ms([&](){ rc.apply(); }, - 100, "Resize GAPI %s %s %dx%d -> %dx%d", - interpToString(interp).c_str(), typeToString(type).c_str(), - sz_in.width, sz_in.height, sz_out.width, sz_out.height); -#endif - - // OpenCV code ///////////////////////////////////////////////////////////// - { - cv::resize(in_mat1, out_mat_ocv, sz_out, 0, 0, interp); - } - // Comparison ////////////////////////////////////////////////////////////// - { - EXPECT_LE(cv::norm(out_mat, out_mat_ocv, cv::NORM_INF), tolerance); - } -} - -TEST_P(ResizeRoiTestGAPI, AccuracyTest) -{ - int type = 0, interp = 0; - cv::Size sz_in, sz_out; - cv::Rect roi; - double tolerance = 0.0; - std::pair sizes; - std::tie(type, interp, sizes, roi, tolerance) = GetParam(); - std::tie(sz_in, sz_out) = sizes; - - cv::Mat in_mat1 (sz_in, type); - cv::Scalar mean = cv::Scalar::all(127); - cv::Scalar stddev = cv::Scalar::all(40.f); - - cv::randn(in_mat1, mean, stddev); - - cv::Mat out_mat(sz_out, type); - cv::Mat out_mat_ocv(sz_out, type); - - // G-API code ////////////////////////////////////////////////////////////// - FluidResizeComputation rc(to_test(in_mat1), to_test(out_mat), interp); - rc.warmUp(to_test(roi)); - -#if PERF_TEST - // iterate testing, and print performance - test_ms([&](){ rc.apply(); }, - 100, "Resize GAPI %s %s %dx%d -> %dx%d", - interpToString(interp).c_str(), typeToString(type).c_str(), - sz_in.width, sz_in.height, sz_out.width, sz_out.height); -#endif - - // OpenCV code ///////////////////////////////////////////////////////////// - { - cv::resize(in_mat1, out_mat_ocv, sz_out, 0, 0, interp); - } - // Comparison ////////////////////////////////////////////////////////////// - { - EXPECT_LE(cv::norm(out_mat(roi), out_mat_ocv(roi), cv::NORM_INF), tolerance); - } -} - -TEST_P(ResizeRGB8URoiTestGAPI, AccuracyTest) -{ - int type = 0, interp = 0; - cv::Size sz_in, sz_out; - cv::Rect roi; - double tolerance = 0.0; - std::pair sizes; - std::tie(type, interp, sizes, roi, tolerance) = GetParam(); - std::tie(sz_in, sz_out) = sizes; - - cv::Mat in_mat1 (sz_in, type); - cv::Scalar mean = cv::Scalar::all(127); - cv::Scalar stddev = cv::Scalar::all(40.f); - - cv::randn(in_mat1, mean, stddev); - - cv::Mat out_mat(sz_out, type); - cv::Mat out_mat_ocv(sz_out, type); - - // G-API code ////////////////////////////////////////////////////////////// - FluidResizeRGB8UComputation rc(to_test(in_mat1), to_test(out_mat), interp); - rc.warmUp(to_test(roi)); - -#if PERF_TEST - // iterate testing, and print performance - test_ms([&](){ rc.apply(); }, - 100, "Resize GAPI %s %s %dx%d -> %dx%d", - interpToString(interp).c_str(), typeToString(type).c_str(), - sz_in.width, sz_in.height, sz_out.width, sz_out.height); -#endif - - // OpenCV code ///////////////////////////////////////////////////////////// - { - cv::resize(in_mat1, out_mat_ocv, sz_out, 0, 0, interp); - } - // Comparison ////////////////////////////////////////////////////////////// - { - EXPECT_LE(cv::norm(out_mat(roi), out_mat_ocv(roi), cv::NORM_INF), tolerance); - } -} - -TEST_P(SplitTestGAPI, AccuracyTest) -{ - const auto params = GetParam(); - int planes = std::get<0>(params); - int depth = std::get<1>(params); - cv::Size sz = std::get<2>(params); - double tolerance = std::get<3>(params); - - auto make_src_type = [planes](int d){ - return CV_MAKE_TYPE(d, planes); - }; - int srcType = make_src_type(depth); - int dstType = CV_MAKE_TYPE(depth, 1); - - cv::Mat in_mat(sz, srcType); - bool const is_fp16 = (depth == CV_16F); - cv::Mat rnd_mat = is_fp16 ? cv::Mat(sz, make_src_type(CV_32F)) : in_mat; - cv::randn(rnd_mat, cv::Scalar::all(127), cv::Scalar::all(40.f)); - - if (is_fp16) { - rnd_mat.convertTo(in_mat, depth); - } - - std::vector out_mats_gapi(planes, cv::Mat::zeros(sz, dstType)); - std::vector out_mats_ocv (planes, cv::Mat::zeros(sz, dstType)); - - // G-API code ////////////////////////////////////////////////////////////// - FluidSplitComputation sc(to_test(in_mat), to_test(out_mats_gapi)); - sc.warmUp(); - -#if PERF_TEST - // iterate testing, and print performance - test_ms([&](){ sc.apply(); }, - 400, "Split GAPI %s %dx%d", typeToString(srcType).c_str(), sz.width, sz.height); -#endif - - // OpenCV code ///////////////////////////////////////////////////////////// - { - cv::split(in_mat, out_mats_ocv); - } - // Comparison ////////////////////////////////////////////////////////////// - { - for (int p = 0; p < planes; p++) { - EXPECT_LE(cv::norm(out_mats_ocv[p], out_mats_gapi[p], cv::NORM_INF), tolerance); - } - } -} - -TEST_P(ChanToPlaneTestGAPI, AccuracyTest) -{ - const auto params = GetParam(); - int planes = std::get<0>(params); - int depth = std::get<1>(params); - cv::Size sz = std::get<2>(params); - double tolerance = std::get<3>(params); - - int inType = CV_MAKE_TYPE(depth, planes); - int outType = CV_MAKE_TYPE(depth, 1); - - cv::Mat in_mat(sz, inType); - cv::randn(in_mat, cv::Scalar::all(127), cv::Scalar::all(40.f)); - - cv::Mat out_mat_gapi(sz, outType); - std::vector out_mats_ocv; - - // OpenCV code ///////////////////////////////////////////////////////////// - { - cv::split(in_mat, out_mats_ocv); - } - - for(int i = 0; i < planes; ++i){ - // G-API code ////////////////////////////////////////////////////////////// - FluidChanToPlaneComputation sc(to_test(in_mat), to_test(out_mat_gapi), i); - sc.warmUp(); - - #if PERF_TEST - // run just for a single plane - if(i == 0){ - // iterate testing, and print performance - test_ms([&](){ sc.apply(); }, - 400, "ChanToPlane GAPI %s %dx%d", typeToString(inType).c_str(), sz.width, sz.height); - } - #endif - - // Comparison ////////////////////////////////////////////////////////////// - { - EXPECT_LE(cv::norm(out_mats_ocv[i], out_mat_gapi, cv::NORM_INF), tolerance); - } - } -} - -TEST_P(MergeTestGAPI, AccuracyTest) -{ - const auto params = GetParam(); - int planes = std::get<0>(params); - int depth = std::get<1>(params); - cv::Size sz = std::get<2>(params); - double tolerance = std::get<3>(params); - - auto make_src_type = [](int d){ - return CV_MAKE_TYPE(d, 1); - }; - int srcType = make_src_type(depth); - int dstType = CV_MAKE_TYPE(depth, planes); - - std::vector in_mats(planes, cv::Mat(sz, srcType)); - for (int p = 0; p < planes; p++) { - bool const is_fp16 = (depth == CV_16F); - cv::Mat rnd_mat = is_fp16 ? cv::Mat(sz, make_src_type(CV_32F)) : in_mats[p]; - cv::randn(rnd_mat, cv::Scalar::all(127), cv::Scalar::all(40.f)); - - if (is_fp16) { - rnd_mat.convertTo(in_mats[p], depth); - } - } - - cv::Mat out_mat_ocv = cv::Mat::zeros(sz, dstType); - cv::Mat out_mat_gapi = cv::Mat::zeros(sz, dstType); - - // G-API code ////////////////////////////////////////////////////////////// - FluidMergeComputation mc(to_test(in_mats), to_test(out_mat_gapi)); - mc.warmUp(); - -#if PERF_TEST - // iterate testing, and print performance - test_ms([&](){ mc.apply(); }, - 400, "Merge GAPI %s %dx%d", typeToString(dstType).c_str(), sz.width, sz.height); -#endif - - // OpenCV code ///////////////////////////////////////////////////////////// - { - cv::merge(in_mats, out_mat_ocv); - } - // Comparison ////////////////////////////////////////////////////////////// - { - EXPECT_LE(cv::norm(out_mat_ocv, out_mat_gapi, cv::NORM_INF), tolerance); - } -} - -TEST_P(NV12toRGBTestGAPI, AccuracyTest) -{ - const auto params = GetParam(); - cv::Size sz = std::get<0>(params); - double tolerance = std::get<1>(params); - - cv::Mat in_mat_y(sz, CV_8UC1); - cv::Mat in_mat_uv(cv::Size(sz.width / 2, sz.height / 2), CV_8UC2); - cv::randn(in_mat_y, cv::Scalar::all(127), cv::Scalar::all(40.f)); - cv::randn(in_mat_uv, cv::Scalar::all(127), cv::Scalar::all(40.f)); - - cv::Mat out_mat_gapi(cv::Mat::zeros(sz, CV_8UC3)); - cv::Mat out_mat_ocv (cv::Mat::zeros(sz, CV_8UC3)); - - // G-API code ////////////////////////////////////////////////////////////// - FluidNV12toRGBComputation cc(to_test(in_mat_y), to_test(in_mat_uv), to_test(out_mat_gapi)); - cc.warmUp(); - -#if PERF_TEST - // iterate testing, and print performance - test_ms([&](){ cc.apply(); }, - 400, "NV12toRGB GAPI %s %dx%d", typeToString(CV_8UC3).c_str(), sz.width, sz.height); -#endif - - // OpenCV code ///////////////////////////////////////////////////////////// - { - cv::cvtColorTwoPlane(in_mat_y,in_mat_uv,out_mat_ocv,cv::COLOR_YUV2RGB_NV12); - } - // Comparison ////////////////////////////////////////////////////////////// - { - EXPECT_LE(cv::norm(out_mat_ocv, out_mat_gapi, cv::NORM_INF), tolerance); - EXPECT_EQ(sz, out_mat_gapi.size()); - } -} - - -TEST_P(I420toRGBTestGAPI, AccuracyTest) -{ - const auto params = GetParam(); - cv::Size sz = std::get<0>(params); - double tolerance = std::get<1>(params); - - cv::Mat in_mat_y(sz, CV_8UC1); - cv::Mat in_mat_u(cv::Size(sz.width / 2, sz.height / 2), CV_8UC1); - cv::Mat in_mat_v(cv::Size(sz.width / 2, sz.height / 2), CV_8UC1); - cv::randn(in_mat_y, cv::Scalar::all(127), cv::Scalar::all(40.f)); - cv::randn(in_mat_u, cv::Scalar::all(127), cv::Scalar::all(40.f)); - cv::randn(in_mat_v, cv::Scalar::all(127), cv::Scalar::all(40.f)); - - cv::Mat out_mat_gapi(cv::Mat::zeros(sz, CV_8UC3)); - cv::Mat out_mat_ocv (cv::Mat::zeros(sz, CV_8UC3)); - - // G-API code ////////////////////////////////////////////////////////////// - FluidI420toRGBComputation cc(to_test(in_mat_y), to_test(in_mat_u), to_test(in_mat_v), to_test(out_mat_gapi)); - cc.warmUp(); - -#if PERF_TEST - // iterate testing, and print performance - test_ms([&](){ cc.apply(); }, - 400, "I420toRGB GAPI %s %dx%d", typeToString(CV_8UC3).c_str(), sz.width, sz.height); -#endif - - // OpenCV code ///////////////////////////////////////////////////////////// - { - cv::Mat in_mat_uv = cv::Mat::zeros(in_mat_u.size(), CV_8UC2); - std::array in_uv = {in_mat_u, in_mat_v}; - cv::merge(in_uv, in_mat_uv); - //cvtColorTwoPlane supports NV12 only at the moment - cv::cvtColorTwoPlane(in_mat_y,in_mat_uv,out_mat_ocv,cv::COLOR_YUV2RGB_NV12); - } - // Comparison ////////////////////////////////////////////////////////////// - { - EXPECT_LE(cv::norm(out_mat_ocv, out_mat_gapi, cv::NORM_INF), tolerance); - EXPECT_EQ(sz, out_mat_gapi.size()); - } -} - -TEST_P(ConvertDepthTestGAPI, AccuracyTest) -{ - const auto params = GetParam(); - int in_depth = std::get<0>(params); - int out_depth = std::get<1>(params); - cv::Size sz = std::get<2>(params); - double tolerance = std::get<3>(params); - - const int out_type = CV_MAKETYPE(out_depth,1); - - initMatrixRandU(CV_MAKETYPE(in_depth,1), sz, out_type); - - // G-API code ////////////////////////////////////////////////////////////// - ConvertDepthComputation cc(to_test(in_mat1), to_test(out_mat_gapi), out_mat_gapi.depth()); - cc.warmUp(); - -#if PERF_TEST - // iterate testing, and print performance - test_ms([&](){ cc.apply(); }, - 400, "ConvDepth GAPI %s to %s %dx%d", depthToString(in_mat1.depth()).c_str(), depthToString(out_mat_gapi.depth()).c_str(), sz.width, sz.height); -#endif - - // OpenCV code ///////////////////////////////////////////////////////////// - { - in_mat1.convertTo(out_mat_ocv, out_type); - } - // Comparison ////////////////////////////////////////////////////////////// - { - EXPECT_LE(cv::norm(out_mat_ocv, out_mat_gapi, cv::NORM_INF), tolerance); - } -} - -TEST_P(DivCTestGAPI, AccuracyTest) -{ - const auto params = GetParam(); - const int in_depth = std::get<0>(params); - const int in_channels = std::get<1>(params); - const cv::Size sz = std::get<2>(params); - const cv::Scalar C = std::get<3>(params); - double tolerance = std::get<4>(params); - - const int in_type = CV_MAKETYPE(in_depth,in_channels); - - initMatrixRandU(in_type, sz, in_type); - - // G-API code - DivCComputation cc(to_test(in_mat1), to_test(out_mat_gapi), to_test(C)); - cc.warmUp(); - - // OpenCV code ///////////////////////////////////////////////////////////// - { - out_mat_ocv = in_mat1 / C; - } - // Comparison ////////////////////////////////////////////////////////////// - { - EXPECT_LE(cv::norm(out_mat_ocv, out_mat_gapi, cv::NORM_INF), tolerance); - } -} - -TEST_P(SubCTestGAPI, AccuracyTest) -{ - const auto params = GetParam(); - const int in_depth = std::get<0>(params); - const int in_channels = std::get<1>(params); - const cv::Size sz = std::get<2>(params); - const cv::Scalar C = std::get<3>(params); - const double tolerance = std::get<4>(params); - - const int in_type = CV_MAKETYPE(in_depth,in_channels); - - initMatrixRandU(in_type, sz, in_type); - - // G-API code - SubCComputation cc(to_test(in_mat1), to_test(out_mat_gapi), to_test(C)); - cc.warmUp(); - - // OpenCV code ///////////////////////////////////////////////////////////// - { - out_mat_ocv = in_mat1 - C; - } - // Comparison ////////////////////////////////////////////////////////////// - { - EXPECT_LE(cv::norm(out_mat_ocv, out_mat_gapi, cv::NORM_INF), tolerance); - } -} - -//---------------------------------------------------------------------- - -TEST_P(ResizeTestIE, AccuracyTest) -{ - int type = 0, interp = 0; - cv::Size sz_in, sz_out; - double tolerance = 0.0; - std::pair sizes; - std::tie(type, interp, sizes, tolerance) = GetParam(); - std::tie(sz_in, sz_out) = sizes; - - cv::Mat in_mat1(sz_in, type ); - cv::Scalar mean = cv::Scalar::all(127); - cv::Scalar stddev = cv::Scalar::all(40.f); - - cv::randn(in_mat1, mean, stddev); - - cv::Mat out_mat(sz_out, type); - cv::Mat out_mat_ocv(sz_out, type); - - // Inference Engine code /////////////////////////////////////////////////// - - size_t channels = out_mat.channels(); - CV_Assert(1 == channels || 3 == channels); - - int depth = CV_MAT_DEPTH(type); - CV_Assert(CV_8U == depth || CV_32F == depth); - - CV_Assert(cv::INTER_AREA == interp || cv::INTER_LINEAR == interp); - - ASSERT_TRUE(in_mat1.isContinuous() && out_mat.isContinuous()); - - using namespace InferenceEngine; - - size_t in_height = in_mat1.rows, in_width = in_mat1.cols; - size_t out_height = out_mat.rows, out_width = out_mat.cols; - InferenceEngine::SizeVector in_sv = { 1, channels, in_height, in_width }; - InferenceEngine::SizeVector out_sv = { 1, channels, out_height, out_width }; - - // HWC blob: channels are interleaved - Precision precision = CV_8U == depth ? Precision::U8 : Precision::FP32; - TensorDesc in_desc(precision, in_sv, Layout::NHWC); - TensorDesc out_desc(precision, out_sv, Layout::NHWC); - - Blob::Ptr in_blob, out_blob; - in_blob = make_blob_with_precision(in_desc , in_mat1.data); - out_blob = make_blob_with_precision(out_desc, out_mat.data); - - PreProcessDataPtr preprocess = CreatePreprocDataHelper(); - preprocess->setRoiBlob(in_blob); - - ResizeAlgorithm algorithm = cv::INTER_AREA == interp ? RESIZE_AREA : RESIZE_BILINEAR; - PreProcessInfo info; - info.setResizeAlgorithm(algorithm); - - // test once to warm-up cache - preprocess->execute(out_blob, info, false); - -#if PERF_TEST - // iterate testing, and print performance - test_ms([&](){ preprocess->execute(out_blob, info, false); }, - 100, "Resize IE %s %s %dx%d -> %dx%d", - interpToString(interp).c_str(), typeToString(type).c_str(), - sz_in.width, sz_in.height, sz_out.width, sz_out.height); -#endif - - // OpenCV code ///////////////////////////////////////////////////////////// - { - cv::resize(in_mat1, out_mat_ocv, sz_out, 0, 0, interp); - } - // Comparison ////////////////////////////////////////////////////////////// - { - EXPECT_LE(cv::norm(out_mat_ocv, out_mat, cv::NORM_INF), tolerance); - } -} - -IE_SUPPRESS_DEPRECATED_START - -TEST_P(ColorConvertTestIE, AccuracyTest) -{ - using namespace InferenceEngine; - int depth = 0; - auto in_fmt = ColorFormat::RAW; - auto out_fmt = ColorFormat::BGR; // for now, always BGR - auto in_layout = Layout::ANY; - auto out_layout = Layout::ANY; - cv::Size size; - double tolerance = 0.0; - std::tie(depth, in_fmt, in_layout, out_layout, size, tolerance) = GetParam(); - - int in_type = CV_MAKE_TYPE(depth, numChannels(in_fmt)); - int out_type = CV_MAKE_TYPE(depth, numChannels(out_fmt)); - - cv::Mat in_mat1(size, in_type); - cv::Scalar mean = cv::Scalar::all(127); - cv::Scalar stddev = cv::Scalar::all(40.f); - - if (depth != CV_16F) - cv::randn(in_mat1, mean, stddev); - - cv::Mat out_mat(size, out_type); - cv::Mat out_mat_ocv(size, out_type); - - // Inference Engine code /////////////////////////////////////////////////// - - if (in_fmt != ColorFormat::RAW && in_fmt != ColorFormat::BGR) { - cv::cvtColor(in_mat1, in_mat1, toCvtColorCode(in_fmt)); - } - - size_t in_channels = in_mat1.channels(); - CV_Assert(3 == in_channels || 4 == in_channels); - - size_t out_channels = out_mat.channels(); - CV_Assert(3 == out_channels || 4 == out_channels); - - CV_Assert(CV_8U == depth || CV_32F == depth || depth == CV_16S || depth == CV_16F); - - ASSERT_TRUE(in_mat1.isContinuous() && out_mat.isContinuous()); - - size_t in_height = in_mat1.rows, in_width = in_mat1.cols; - size_t out_height = out_mat.rows, out_width = out_mat.cols; - InferenceEngine::SizeVector in_sv = { 1, in_channels, in_height, in_width }; - InferenceEngine::SizeVector out_sv = { 1, out_channels, out_height, out_width }; - - auto depth_to_precision = [](int depth) -> Precision::ePrecision { - switch (depth) - { - case CV_8U: return Precision::U8; - case CV_16S: return Precision::I16; - case CV_16F: return Precision::FP16; - case CV_32F: return Precision::FP32; - default: - throw std::logic_error("Unsupported configuration"); - } - return Precision::UNSPECIFIED; - }; - - // HWC blob: channels are interleaved - Precision precision = depth_to_precision(depth); - - Blob::Ptr in_blob, out_blob; - switch (precision) - { - case Precision::U8: - in_blob = img2Blob(in_mat1, in_layout); - out_blob = img2Blob(out_mat, out_layout); - break; - - case Precision::FP32: - in_blob = img2Blob(in_mat1, in_layout); - out_blob = img2Blob(out_mat, out_layout); - break; - - case Precision::I16: - in_blob = img2Blob(in_mat1, in_layout); - out_blob = img2Blob(out_mat, out_layout); - break; - - case Precision::FP16: - in_blob = img2Blob(in_mat1, in_layout); - out_blob = img2Blob(out_mat, out_layout); - - break; - - - default: - FAIL() << "Unsupported configuration"; - } - - PreProcessDataPtr preprocess = CreatePreprocDataHelper(); - preprocess->setRoiBlob(in_blob); - - PreProcessInfo info; - info.setColorFormat(in_fmt); - - // test once to warm-up cache - preprocess->execute(out_blob, info, false); - - switch (precision) - { - case Precision::U8: Blob2Img (out_blob, out_mat, out_layout); break; - case Precision::FP32: Blob2Img(out_blob, out_mat, out_layout); break; - case Precision::I16: Blob2Img (out_blob, out_mat, out_layout); break; - case Precision::FP16: Blob2Img (out_blob, out_mat, out_layout); break; - default: FAIL() << "Unsupported configuration"; - } - -#if PERF_TEST - // iterate testing, and print performance - test_ms([&](){ preprocess->execute(out_blob, info, false); }, - 100, "Color Convert IE %s %s %s %dx%d %s->%s", - depthToString(depth).c_str(), - layoutToString(in_layout).c_str(), layoutToString(out_layout).c_str(), - size.width, size.height, - colorFormatToString(in_fmt).c_str(), colorFormatToString(out_fmt).c_str()); -#endif - - // OpenCV code ///////////////////////////////////////////////////////////// - { - if (in_fmt != out_fmt) { - cv::cvtColor(in_mat1, out_mat_ocv, toCvtColorCode(in_fmt, out_fmt)); - } else { - // only reorder is done - out_mat_ocv = in_mat1; - } - } - - // Comparison ////////////////////////////////////////////////////////////// - { - EXPECT_LE(cv::norm(out_mat_ocv, out_mat, cv::NORM_INF), tolerance); - } -} - -TEST_P(SplitTestIE, AccuracyTest) -{ - const auto params = GetParam(); - int type = std::get<0>(params); - cv::Size size = std::get<1>(params); - double tolerance = std::get<2>(params); - - int depth = CV_MAT_DEPTH(type); - CV_Assert(CV_8U == depth || CV_32F == depth); - - int type1 = CV_MAKE_TYPE(depth, 1); - int type4 = CV_MAKE_TYPE(depth, 4); - - cv::Scalar mean = cv::Scalar::all(127); - cv::Scalar stddev = cv::Scalar::all(40.f); - - cv::Mat in_mat(size, type); - cv::randn(in_mat, mean, stddev); - - int channels = in_mat.channels(); - CV_Assert(2 == channels || 3 == channels || 4 == channels); - - size_t elemsize1 = in_mat.elemSize1(); - int total = in_mat.total(); - - cv::Mat out_mat(size, type4); - CV_Assert(in_mat.isContinuous() && out_mat.isContinuous()); - - cv::Mat out_mat0(size, type1, out_mat.data + 0*total*elemsize1); - cv::Mat out_mat1(size, type1, out_mat.data + 1*total*elemsize1); - cv::Mat out_mat2(size, type1, out_mat.data + 2*total*elemsize1); - cv::Mat out_mat3(size, type1, out_mat.data + 3*total*elemsize1); - - cv::Mat out_mats[] = {out_mat0, out_mat1, out_mat2, out_mat3}; - - std::vector out_mats_ocv(channels); - - // Inference Engine code /////////////////////////////////////////////////// - - using namespace InferenceEngine; - - size_t width = size.width; - size_t height = size.height; - InferenceEngine::SizeVector sv = { 1, (size_t)channels, height, width }; - - Precision precision = CV_8U == depth ? Precision::U8 : Precision::FP32; - TensorDesc in_desc(precision, sv, Layout::NHWC); // interleaved - TensorDesc out_desc(precision, sv, Layout::NCHW); // color planes - - Blob::Ptr in_blob, out_blob; - in_blob = make_blob_with_precision( in_desc, in_mat.data); - out_blob = make_blob_with_precision(out_desc, out_mat.data); - - // test once - blob_copy(in_blob, out_blob); - -#if PERF_TEST - // iterate testing, and print performance - test_ms([&]() { blob_copy(in_blob, out_blob); }, - 400, "Split IE %s %dx%d", typeToString(type).c_str(), size.width, size.height); -#endif - - // OpenCV code ///////////////////////////////////////////////////////////// - - cv::split(in_mat, out_mats_ocv); - - // Comparison ////////////////////////////////////////////////////////////// - - for (int i = 0; i < channels; i++) - { - EXPECT_LE(cv::norm(out_mats_ocv[i], out_mats[i], cv::NORM_INF), tolerance); - } -} - -TEST_P(MergeTestIE, AccuracyTest) -{ - const auto params = GetParam(); - int type = std::get<0>(params); - cv::Size size = std::get<1>(params); - double tolerance = std::get<2>(params); - - int depth = CV_MAT_DEPTH(type); - CV_Assert(CV_8U == depth || CV_32F == depth); - - int type1 = CV_MAKE_TYPE(depth, 1); - int type4 = CV_MAKE_TYPE(depth, 4); - - cv::Mat out_mat(size, type), out_mat_ocv; - - cv::Mat in_mat(size, type4); - - int channels = out_mat.channels(); - CV_Assert(2 == channels || 3 == channels || 4 == channels); - - size_t elemsize1 = out_mat.elemSize1(); - int total = out_mat.total(); - - cv::Mat in_mat0(size, type1, in_mat.data + 0*total*elemsize1); - cv::Mat in_mat1(size, type1, in_mat.data + 1*total*elemsize1); - cv::Mat in_mat2(size, type1, in_mat.data + 2*total*elemsize1); - cv::Mat in_mat3(size, type1, in_mat.data + 3*total*elemsize1); - - cv::Mat in_mats[] = { in_mat0, in_mat1, in_mat2, in_mat3 }; - - cv::Scalar mean = cv::Scalar::all(127); - cv::Scalar stddev = cv::Scalar::all(40.f); - - for (int i = 0; i < 4 ; i++) - { - cv::randn(in_mats[i], mean, stddev); - } - - CV_Assert(in_mat.isContinuous() && out_mat.isContinuous()); - - // Inference Engine code /////////////////////////////////////////////////// - - using namespace InferenceEngine; - - size_t width = size.width; - size_t height = size.height; - InferenceEngine::SizeVector sv = { 1, (size_t)channels, height, width }; - - Precision precision = CV_8U == depth ? Precision::U8 : Precision::FP32; - TensorDesc in_desc(precision, sv, Layout::NCHW); // color planes - TensorDesc out_desc(precision, sv, Layout::NHWC); // interleaved - - Blob::Ptr in_blob, out_blob; - in_blob = make_blob_with_precision( in_desc, in_mat.data); - out_blob = make_blob_with_precision(out_desc, out_mat.data); - - // test once - blob_copy(in_blob, out_blob); - -#if PERF_TEST - // iterate testing, and print performance - test_ms([&]() { blob_copy(in_blob, out_blob); }, - 400, "Merge IE %s %dx%d", typeToString(type).c_str(), size.width, size.height); -#endif - - // OpenCV code ///////////////////////////////////////////////////////////// - - cv::merge(in_mats, channels, out_mat_ocv); - - // Comparison ////////////////////////////////////////////////////////////// - - EXPECT_LE(cv::norm(out_mat_ocv, out_mat, cv::NORM_INF), tolerance); -} - -TEST_P(MeanValueGAPI, AccuracyTest) -{ - const auto params = GetParam(); - cv::Size sz = std::get<0>(params); - double tolerance = std::get<1>(params); - - initMatrixRandU(CV_32FC1, sz, CV_32FC1); - - const cv::Scalar mean = { 0.485, 0.456, 0.406 }; - const cv::Scalar std = { 0.229, 0.224, 0.225 }; - - // G-API code - MeanValueSubtractComputation cc(to_test(in_mat1), to_test(out_mat_gapi), to_test(mean), to_test(std)); - cc.warmUp(); - - // OpenCV code ///////////////////////////////////////////////////////////// - { - out_mat_ocv = (in_mat1 - mean) / std; - } - // Comparison ////////////////////////////////////////////////////////////// - { - EXPECT_LE(cv::norm(out_mat_ocv, out_mat_gapi, cv::NORM_INF), tolerance); - } - -} - diff --git a/src/common/preprocessing/tests/common/fluid_tests.hpp b/src/common/preprocessing/tests/common/fluid_tests.hpp deleted file mode 100644 index 3ee98527250..00000000000 --- a/src/common/preprocessing/tests/common/fluid_tests.hpp +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright (C) 2018-2023 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#ifndef FLUID_TESTS_HPP -#define FLUID_TESTS_HPP - -#include "fluid_tests_common.hpp" -#include "ie_preprocess.hpp" - -#include - -struct ResizeTestGAPI: public testing::TestWithParam, double>> {}; -struct ResizeRGB8UTestGAPI: public testing::TestWithParam, double>> {}; -struct SplitTestGAPI: public TestParams> {}; -struct ChanToPlaneTestGAPI: public TestParams> {}; -struct MergeTestGAPI: public TestParams> {}; -struct NV12toRGBTestGAPI: public TestParams> {}; -struct I420toRGBTestGAPI: public TestParams> {}; -struct ResizeRoiTestGAPI: public testing::TestWithParam, cv::Rect, double>> {}; -struct ResizeRGB8URoiTestGAPI: public testing::TestWithParam, cv::Rect, double>> {}; -struct ConvertDepthTestGAPI: public TestParams> // tolerance -{}; -struct DivCTestGAPI: public TestParams> // tolerance -{}; - -struct SubCTestGAPI : public DivCTestGAPI -{}; - -struct MeanValueGAPI: public TestParams> {}; -//------------------------------------------------------------------------------ - -struct ResizeTestIE: public testing::TestWithParam, double>> {}; - -struct SplitTestIE: public TestParams> {}; -struct MergeTestIE: public TestParams> {}; - -struct ColorConvertTestIE: - public testing::TestWithParam> // tolerance -{}; - -struct ColorConvertYUV420TestIE: - public testing::TestWithParam> // tolerance -{}; - -struct PrecisionConvertTestIE: public TestParams> // tolerance -{}; - -//------------------------------------------------------------------------------ - -using PreprocParams = std::tuple< std::pair // output data type - , InferenceEngine::ResizeAlgorithm // resize algorithm, if needed - , InferenceEngine::ColorFormat // input color format, if needed - , InferenceEngine::Layout // input tensor layout - , InferenceEngine::Layout // output tensor layout - , std::pair // number of input and output channels - , std::pair - >; - -struct PreprocTest: public TestParams {}; - -#endif //FLUID_TESTS_HPP diff --git a/src/common/preprocessing/tests/common/fluid_tests_common.hpp b/src/common/preprocessing/tests/common/fluid_tests_common.hpp deleted file mode 100644 index 69b62f5c87e..00000000000 --- a/src/common/preprocessing/tests/common/fluid_tests_common.hpp +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright (C) 2018-2023 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#pragma once - -#include - -#include "opencv2/core.hpp" -#include "opencv2/gapi/cpu/core.hpp" - -#include - -class TestFunctional -{ -public: - cv::Mat in_mat1; - cv::Mat in_mat2; - cv::Mat out_mat_gapi; - cv::Mat out_mat_ocv; - - cv::Scalar sc; - - void initMatsRandU(int type, cv::Size sz_in, int dtype, bool createOutputMatrices = true) - { - in_mat1 = cv::Mat(sz_in, type); - in_mat2 = cv::Mat(sz_in, type); - - auto& rng = cv::theRNG(); - sc = cv::Scalar(rng(100),rng(100),rng(100),rng(100)); - cv::randu(in_mat1, cv::Scalar::all(0), cv::Scalar::all(255)); - cv::randu(in_mat2, cv::Scalar::all(0), cv::Scalar::all(255)); - - if (createOutputMatrices && dtype != -1) - { - out_mat_gapi = cv::Mat (sz_in, dtype); - out_mat_ocv = cv::Mat (sz_in, dtype); - } - } - - void initMatrixRandU(int type, cv::Size sz_in, int dtype, bool createOutputMatrices = true) - { - in_mat1 = cv::Mat(sz_in, type); - - auto& rng = cv::theRNG(); - sc = cv::Scalar(rng(100),rng(100),rng(100),rng(100)); - - cv::randu(in_mat1, cv::Scalar::all(0), cv::Scalar::all(255)); - - if (createOutputMatrices && dtype != -1) - { - out_mat_gapi = cv::Mat (sz_in, dtype); - out_mat_ocv = cv::Mat (sz_in, dtype); - } - } - - void initMatsRandN(int type, cv::Size sz_in, int dtype, bool createOutputMatrices = true) - { - in_mat1 = cv::Mat(sz_in, type); - cv::randn(in_mat1, cv::Scalar::all(127), cv::Scalar::all(40.f)); - - if (createOutputMatrices && dtype != -1) - { - out_mat_gapi = cv::Mat(sz_in, dtype); - out_mat_ocv = cv::Mat(sz_in, dtype); - } - } -}; - -template -class TestParams: public TestFunctional, public testing::TestWithParam{}; diff --git a/src/common/preprocessing/tests/cpu/fluid_tests_cpu.cpp b/src/common/preprocessing/tests/cpu/fluid_tests_cpu.cpp deleted file mode 100644 index 3f125a14dae..00000000000 --- a/src/common/preprocessing/tests/cpu/fluid_tests_cpu.cpp +++ /dev/null @@ -1,413 +0,0 @@ -// Copyright (C) 2018-2023 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#include "fluid_tests.hpp" - -#include - -#include - -#define TEST_SIZES \ - cv::Size(3840, 2160), \ - cv::Size(1920, 1080), \ - cv::Size(1280, 720), \ - cv::Size(1280, 960), \ - cv::Size( 960, 720), \ - cv::Size( 640, 480), \ - cv::Size( 320, 200), \ - cv::Size( 113, 71) - -#define TEST_RESIZE_DOWN \ - std::make_pair(cv::Size(3840, 2160), cv::Size(1920, 1080)), \ - std::make_pair(cv::Size(3840, 2160), cv::Size(1280, 720)), \ - std::make_pair(cv::Size(1920, 1080), cv::Size(1280, 720)), \ - std::make_pair(cv::Size(1920, 1080), cv::Size( 640, 480)), \ - std::make_pair(cv::Size(1280, 720), cv::Size( 640, 480)), \ - std::make_pair(cv::Size(1280, 720), cv::Size( 320, 200)), \ - std::make_pair(cv::Size( 640, 480), cv::Size( 320, 200)), \ - std::make_pair(cv::Size( 640, 480), cv::Size( 113, 71)), \ - std::make_pair(cv::Size( 320, 200), cv::Size( 113, 71)) - -#define TEST_RESIZE_UP \ - std::make_pair(cv::Size(1920, 1080), cv::Size(3840, 2160)), \ - std::make_pair(cv::Size(1280, 720), cv::Size(3840, 2160)), \ - std::make_pair(cv::Size(1280, 720), cv::Size(1920, 1080)), \ - std::make_pair(cv::Size( 640, 480), cv::Size(1920, 1080)), \ - std::make_pair(cv::Size( 640, 480), cv::Size(1280, 720)), \ - std::make_pair(cv::Size( 320, 200), cv::Size(1280, 720)), \ - std::make_pair(cv::Size( 320, 200), cv::Size( 640, 480)), \ - std::make_pair(cv::Size( 113, 71), cv::Size( 640, 480)), \ - std::make_pair(cv::Size( 113, 71), cv::Size( 320, 200)) - -#define TEST_RESIZE_HORZ \ - std::make_pair(cv::Size(3840, 2160), cv::Size(1920, 2160)), \ - std::make_pair(cv::Size(1920, 1080), cv::Size(3840, 1080)), \ - std::make_pair(cv::Size(1920, 1080), cv::Size(1280, 1080)), \ - std::make_pair(cv::Size(1280, 720), cv::Size(1920, 720)), \ - std::make_pair(cv::Size(1280, 720), cv::Size( 640, 720)), \ - std::make_pair(cv::Size( 640, 480), cv::Size(1280, 480)), \ - std::make_pair(cv::Size( 640, 480), cv::Size( 320, 480)), \ - std::make_pair(cv::Size( 320, 200), cv::Size( 640, 200)), \ - std::make_pair(cv::Size( 320, 200), cv::Size( 113, 200)), \ - std::make_pair(cv::Size( 113, 71), cv::Size( 320, 71)) - -#define TEST_RESIZE_VERT \ - std::make_pair(cv::Size(3840, 2160), cv::Size(3840, 1080)), \ - std::make_pair(cv::Size(1920, 1080), cv::Size(1920, 2160)), \ - std::make_pair(cv::Size(1920, 1080), cv::Size(1920, 720)), \ - std::make_pair(cv::Size(1280, 720), cv::Size(1280, 1080)), \ - std::make_pair(cv::Size(1280, 720), cv::Size(1280, 480)), \ - std::make_pair(cv::Size( 640, 480), cv::Size( 640, 720)), \ - std::make_pair(cv::Size( 640, 480), cv::Size( 640, 200)), \ - std::make_pair(cv::Size( 320, 200), cv::Size( 320, 480)), \ - std::make_pair(cv::Size( 320, 200), cv::Size( 320, 71)), \ - std::make_pair(cv::Size( 113, 71), cv::Size( 113, 200)) - -#define TEST_RESIZE_COPY \ - std::make_pair(cv::Size(3840, 2160), cv::Size(3840, 2160)), \ - std::make_pair(cv::Size(1920, 1080), cv::Size(1920, 1080)), \ - std::make_pair(cv::Size(1280, 720), cv::Size(1280, 720)), \ - std::make_pair(cv::Size( 640, 480), cv::Size( 640, 480)), \ - std::make_pair(cv::Size( 320, 200), cv::Size( 320, 200)), \ - std::make_pair(cv::Size( 113, 71), cv::Size( 113, 71)) - -#define TEST_RESIZE_SPECIAL \ - std::make_pair(cv::Size(300, 300), cv::Size(300, 199)), \ - std::make_pair(cv::Size(300, 300), cv::Size(199, 300)), \ - std::make_pair(cv::Size(300, 300), cv::Size(199, 199)), \ - std::make_pair(cv::Size(199, 199), cv::Size(300, 300)), \ - std::make_pair(cv::Size(199, 300), cv::Size(300, 300)), \ - std::make_pair(cv::Size(300, 199), cv::Size(300, 300)) - -#define TEST_RESIZE_PAIRS \ - TEST_RESIZE_DOWN, \ - TEST_RESIZE_UP, \ - TEST_RESIZE_HORZ, \ - TEST_RESIZE_VERT, \ - TEST_RESIZE_COPY, \ - TEST_RESIZE_SPECIAL - -#define TEST_SIZES_PREPROC \ - std::make_pair(cv::Size(1920, 1080), cv::Size(1024, 1024)), \ - std::make_pair(cv::Size(1280, 720), cv::Size( 544, 320)), \ - std::make_pair(cv::Size( 640, 480), cv::Size( 896, 512)), \ - std::make_pair(cv::Size( 200, 400), cv::Size( 128, 384)), \ - std::make_pair(cv::Size( 256, 256), cv::Size( 72, 72)), \ - std::make_pair(cv::Size( 96, 256), cv::Size( 128, 384)) - -using namespace testing; -#if defined(__arm__) || defined(__aarch64__) -INSTANTIATE_TEST_SUITE_P(ResizeTestFluid_U8, ResizeTestGAPI, - Combine(Values(CV_8UC1, CV_8UC3), - Values(cv::INTER_LINEAR, cv::INTER_AREA), - Values(TEST_RESIZE_PAIRS), - Values(4))); // error not more than 4 unit - -INSTANTIATE_TEST_SUITE_P(ResizeRGB8UTestFluid_U8, ResizeRGB8UTestGAPI, - Combine(Values(CV_8UC3, CV_8UC4), - Values(cv::INTER_LINEAR), - Values(TEST_RESIZE_PAIRS), - Values(4))); // error not more than 4 unit -#else -INSTANTIATE_TEST_SUITE_P(ResizeTestFluid_U8, ResizeTestGAPI, - Combine(Values(CV_8UC1, CV_8UC3), - Values(cv::INTER_LINEAR, cv::INTER_AREA), - Values(TEST_RESIZE_PAIRS), - Values(1))); // error not more than 1 unit - -INSTANTIATE_TEST_SUITE_P(ResizeRGB8UTestFluid_U8, ResizeRGB8UTestGAPI, - Combine(Values(CV_8UC3, CV_8UC4), - Values(cv::INTER_LINEAR), - Values(TEST_RESIZE_PAIRS), - Values(1))); // error not more than 1 unit -#endif - -INSTANTIATE_TEST_SUITE_P(ResizeTestFluid_F32, ResizeTestGAPI, - Combine(Values(CV_32FC1, CV_32FC3), - Values(cv::INTER_LINEAR, cv::INTER_AREA), - Values(TEST_RESIZE_PAIRS), - Values(0.015))); // accuracy like ~1.5% - - -INSTANTIATE_TEST_SUITE_P(SplitTestFluid, SplitTestGAPI, - Combine(Values(2, 3, 4), - Values(CV_8U, CV_8S, CV_16U, CV_16S, CV_16F, CV_32F, CV_32S), - Values(TEST_SIZES), - Values(0))); - -INSTANTIATE_TEST_SUITE_P(ChanToPlaneTestFluid, ChanToPlaneTestGAPI, - Combine(Values(1, 3), - Values(CV_8U, CV_32F), - Values(TEST_SIZES), - Values(0))); - -INSTANTIATE_TEST_SUITE_P(MergeTestFluid, MergeTestGAPI, - Combine(Values(2, 3, 4), - Values(CV_8U, CV_8S, CV_16U, CV_16S, CV_16F, CV_32F, CV_32S), - Values(TEST_SIZES), - Values(0))); - -INSTANTIATE_TEST_SUITE_P(NV12toRGBTestFluid, NV12toRGBTestGAPI, - Combine(Values(cv::Size(3840, 2160), - cv::Size(1920, 1080), - cv::Size(1280, 720), - cv::Size(1280, 960), - cv::Size( 960, 720), - cv::Size( 640, 480), - cv::Size( 300, 300), - cv::Size( 320, 200)), - Values(1))); - -INSTANTIATE_TEST_SUITE_P(I420toRGBTestFluid, I420toRGBTestGAPI, - Combine(Values(cv::Size(3840, 2160), - cv::Size(1920, 1080), - cv::Size(1280, 720), - cv::Size(1280, 960), - cv::Size( 960, 720), - cv::Size( 640, 480), - cv::Size( 300, 300), - cv::Size( 320, 200)), - Values(1))); - -INSTANTIATE_TEST_SUITE_P(ConvertDepthFluid, ConvertDepthTestGAPI, - Combine(Values(CV_16U, CV_32F, CV_8U), - Values(CV_32F, CV_16U, CV_8U), - Values(cv::Size(3840, 2160), - cv::Size(1920, 1080), - cv::Size(1280, 720), - cv::Size(1280, 960), - cv::Size( 960, 720), - cv::Size( 640, 480), - cv::Size( 300, 300), - cv::Size( 320, 200)), - Values(1))); - -INSTANTIATE_TEST_SUITE_P(DivCFluid, DivCTestGAPI, - Combine(Values(CV_32F), - Values(1), //channels - Values(TEST_SIZES), - Values(cv::Scalar{0.229}), - Values(0))); - -INSTANTIATE_TEST_SUITE_P(SubCFluid, SubCTestGAPI, - Combine(Values(CV_32F), - Values(1), //channels - Values(TEST_SIZES), - Values(cv::Scalar{0.229}), - Values(0.00001))); - -INSTANTIATE_TEST_SUITE_P(ResizeRoiTestFluid, ResizeRoiTestGAPI, - Combine(Values(CV_8UC1, CV_8UC3), - Values(cv::INTER_LINEAR), - Values(std::make_pair(cv::Size(24, 24), cv::Size(12, 12))), - Values(cv::Rect{0, 0, 12, 3}, - cv::Rect{0, 3, 12, 3}, - cv::Rect{0, 6, 12, 3}, - cv::Rect{0, 9, 12, 3}), - Values(1))); // error not more than 1 unit - -INSTANTIATE_TEST_SUITE_P(ResizeRGB8URoiTestFluid, ResizeRGB8URoiTestGAPI, - Combine(Values(CV_8UC3, CV_8UC4), - Values(cv::INTER_LINEAR), - Values(std::make_pair(cv::Size(24, 24), cv::Size(12, 12))), - Values(cv::Rect{0, 0, 12, 3}, - cv::Rect{0, 3, 12, 3}, - cv::Rect{0, 6, 12, 3}, - cv::Rect{0, 9, 12, 3}), - Values(1))); // error not more than 1 unit - -//---------------------------------------------------------------------- - -#if defined(__arm__) || defined(__aarch64__) -INSTANTIATE_TEST_SUITE_P(ResizeTestFluid_U8, ResizeTestIE, - Combine(Values(CV_8UC1, CV_8UC3), - Values(cv::INTER_LINEAR, cv::INTER_AREA), - Values(TEST_RESIZE_PAIRS), - Values(4))); // error not more than 4 unit -#else -INSTANTIATE_TEST_SUITE_P(ResizeTestFluid_U8, ResizeTestIE, - Combine(Values(CV_8UC1, CV_8UC3), - Values(cv::INTER_LINEAR, cv::INTER_AREA), - Values(TEST_RESIZE_PAIRS), - Values(1))); // error not more than 1 unit -#endif - -INSTANTIATE_TEST_SUITE_P(ResizeTestFluid_F32, ResizeTestIE, - Combine(Values(CV_32FC1, CV_32FC3), - Values(cv::INTER_LINEAR, cv::INTER_AREA), - Values(TEST_RESIZE_PAIRS), - Values(0.05))); // error within 0.05 units - -INSTANTIATE_TEST_SUITE_P(SplitTestFluid, SplitTestIE, - Combine(Values(CV_8UC2, CV_8UC3, CV_8UC4, - CV_32FC2, CV_32FC3, CV_32FC4), - Values(TEST_SIZES), - Values(0))); - -INSTANTIATE_TEST_SUITE_P(MergeTestFluid, MergeTestIE, - Combine(Values(CV_8UC2, CV_8UC3, CV_8UC4, - CV_32FC2, CV_32FC3, CV_32FC4), - Values(TEST_SIZES), - Values(0))); - -INSTANTIATE_TEST_SUITE_P(ColorConvertFluid_3ch, ColorConvertTestIE, - Combine(Values(CV_8U, CV_32F), - Values(InferenceEngine::ColorFormat::RGB), - Values(InferenceEngine::NHWC, InferenceEngine::NCHW), - Values(InferenceEngine::NHWC, InferenceEngine::NCHW), - Values(TEST_SIZES), - Values(0))); - -INSTANTIATE_TEST_SUITE_P(ColorConvertFluid_4ch, ColorConvertTestIE, - Combine(Values(CV_8U, CV_32F), - Values(InferenceEngine::ColorFormat::RGBX, - InferenceEngine::ColorFormat::BGRX), - Values(InferenceEngine::NHWC), - Values(InferenceEngine::NHWC, InferenceEngine::NCHW), - Values(TEST_SIZES), - Values(0))); - -INSTANTIATE_TEST_SUITE_P(Reorder_HWC2CHW, ColorConvertTestIE, - Combine(Values(CV_8U, CV_32F, CV_16S, CV_16F), - Values(InferenceEngine::ColorFormat::BGR), - Values(InferenceEngine::NHWC), - Values(InferenceEngine::NCHW), - Values(TEST_SIZES), - Values(0))); - -INSTANTIATE_TEST_SUITE_P(Reorder_CHW2HWC, ColorConvertTestIE, - Combine(Values(CV_8U, CV_32F, CV_16S, CV_16F), - Values(InferenceEngine::ColorFormat::BGR), - Values(InferenceEngine::NCHW), - Values(InferenceEngine::NHWC), - Values(TEST_SIZES), - Values(0))); - -INSTANTIATE_TEST_SUITE_P(MeanValueGAPI32F, MeanValueGAPI, - Combine(Values(TEST_SIZES), - Values(0.00001))); - -//------------------------------------------------------------------------------ - -namespace IE = InferenceEngine; - -static const auto FRAME_SIZES = - Values(std::make_pair(cv::Size(1920,1080), - cv::Size(1024,1024)), // person-vehicle-bike-detection-crossroad-0078 - std::make_pair(cv::Size(1024, 768), - cv::Size( 992, 544)), // person-detection-retail-0001 - std::make_pair(cv::Size(1280, 720), - cv::Size( 896, 512)), // road-segmentation-adas-0001 - std::make_pair(cv::Size(3840, 2160), - cv::Size(2048, 1024)), // semantic-segmentation-adas-0001 - std::make_pair(cv::Size(1270, 720), - cv::Size(2048, 1024)), // semantic-segmentation-adas-0001 (UPSCALE) - std::make_pair(cv::Size( 640, 480), - cv::Size( 544, 320))); // 320 - face-person-detection-retail-0002, - // 320 - person-detection-retail-10013 - // 300 - face-detection-retail-0004 - -static const auto PATCH_SIZES = - Values(std::make_pair(cv::Size(200,400), - cv::Size(128,384)), // person-reidentification-retail-0076 - std::make_pair(cv::Size( 96,256), - cv::Size(128,384)), // person-reidentification-retail-0076 (UPSCALE) - std::make_pair(cv::Size(340,340), - cv::Size(320,256)), // vehicle-license-plate-detection-barrier-0007 - std::make_pair(cv::Size(256,256), - cv::Size( 72,72)), // vehicle-attributes-recognition-barrier-0039 - std::make_pair(cv::Size(96,96), - cv::Size(64,64)), // 60 - head-pose-estimation-adas-0001 - // 62 - age-gender-recognition-retail-0013 - // 64 - emotions-recognition-retail-0003 - std::make_pair(cv::Size(128,48), - cv::Size( 94,24)), // license-plate-recognition-barrier-0001 - std::make_pair(cv::Size(120,200), - cv::Size(80, 160))); // 80 - person-attributes-recognition-crossroad-0031 - // 64 - person-reidentification-retail-0079 - -static const auto U8toU8 = std::make_pair(IE::Precision::U8,IE::Precision::U8); - -static const auto PRECISIONS = Values(U8toU8, std::make_pair(IE::Precision::FP32,IE::Precision::FP32)); - -INSTANTIATE_TEST_SUITE_P(ReorderResize_Frame, PreprocTest, - Combine(PRECISIONS, - Values(IE::ResizeAlgorithm::RESIZE_BILINEAR), // AREA is not there yet - Values(IE::ColorFormat::RAW), - Values(IE::Layout::NHWC), - Values(IE::Layout::NCHW), - Values(std::make_pair(1, 1), std::make_pair(3, 3)), - FRAME_SIZES)); - -INSTANTIATE_TEST_SUITE_P(Scale3ch_Frame, PreprocTest, - Combine(PRECISIONS, - Values(IE::ResizeAlgorithm::RESIZE_BILINEAR), // AREA is not there yet - Values(IE::ColorFormat::RAW), - Values(IE::Layout::NHWC), - Values(IE::Layout::NHWC), - Values(std::make_pair(3, 3)), - FRAME_SIZES)); - -INSTANTIATE_TEST_SUITE_P(ReorderResize_Patch, PreprocTest, - Combine(PRECISIONS, - Values(IE::ResizeAlgorithm::RESIZE_BILINEAR), // AREA is not there yet - Values(IE::ColorFormat::RAW), - Values(IE::Layout::NHWC), - Values(IE::Layout::NCHW, IE::Layout::NCHW), - Values(std::make_pair(1, 1), std::make_pair(3, 3)), - PATCH_SIZES)); - -INSTANTIATE_TEST_SUITE_P(Everything_Resize, PreprocTest, - Combine(PRECISIONS, - Values(IE::ResizeAlgorithm::RESIZE_BILINEAR, IE::ResizeAlgorithm::RESIZE_AREA), - Values(IE::ColorFormat::RAW), - Values(IE::Layout::NHWC, IE::Layout::NCHW), - Values(IE::Layout::NHWC, IE::Layout::NCHW), - Values(std::make_pair(1, 1), - std::make_pair(2, 2), - std::make_pair(3, 3), - std::make_pair(4, 4)), - Values(TEST_SIZES_PREPROC))); - -INSTANTIATE_TEST_SUITE_P(ColorFormats_3ch, PreprocTest, - Combine(PRECISIONS, - Values(IE::ResizeAlgorithm::RESIZE_BILINEAR, IE::ResizeAlgorithm::RESIZE_AREA), - Values(IE::ColorFormat::RGB), - Values(IE::Layout::NHWC, IE::Layout::NCHW), - Values(IE::Layout::NHWC, IE::Layout::NCHW), - Values(std::make_pair(3, 3)), - Values(TEST_SIZES_PREPROC))); - -INSTANTIATE_TEST_SUITE_P(ColorFormats_4ch, PreprocTest, - Combine(PRECISIONS, - Values(IE::ResizeAlgorithm::RESIZE_BILINEAR, IE::ResizeAlgorithm::RESIZE_AREA), - Values(IE::ColorFormat::BGRX, IE::ColorFormat::RGBX), - Values(IE::Layout::NHWC), - Values(IE::Layout::NHWC, IE::Layout::NCHW), - Values(std::make_pair(4, 3)), - Values(TEST_SIZES_PREPROC))); - -INSTANTIATE_TEST_SUITE_P(PlainPrecisionConversions, PreprocTest, - Combine(Values(std::make_pair(IE::Precision::U16,IE::Precision::FP32), - std::make_pair(IE::Precision::FP32,IE::Precision::U16) - ), - Values(IE::ResizeAlgorithm::NO_RESIZE), - Values(IE::ColorFormat::RAW), - Values(IE::Layout::NHWC), - Values(IE::Layout::NHWC), - Values(std::make_pair(1, 1)), - Values(std::make_pair(cv::Size(640,480), cv::Size(640,480))))); - - -INSTANTIATE_TEST_SUITE_P(PrecisionConversionsPipelines, PreprocTest, - Combine(Values(std::make_pair(IE::Precision::U16, IE::Precision::FP32), - std::make_pair(IE::Precision::FP32,IE::Precision::U8), - std::make_pair(IE::Precision::U8, IE::Precision::FP32) - ), - Values(IE::ResizeAlgorithm::RESIZE_BILINEAR), - Values(IE::ColorFormat::RAW), - Values(IE::Layout::NHWC, IE::Layout::NCHW), - Values(IE::Layout::NHWC, IE::Layout::NCHW), - Values(std::make_pair(1, 1), std::make_pair(3, 3)), - Values(TEST_SIZES_PREPROC))); diff --git a/src/common/preprocessing/tests/fluid_test_computations/CMakeLists.txt b/src/common/preprocessing/tests/fluid_test_computations/CMakeLists.txt deleted file mode 100644 index e7ccc606610..00000000000 --- a/src/common/preprocessing/tests/fluid_test_computations/CMakeLists.txt +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright (C) 2018-2023 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 -# - -file(GLOB SRC *.cpp) -file(GLOB HDR *.hpp) - -set(TARGET_NAME fluid_test_computations) - -add_library(${TARGET_NAME} STATIC ${SRC} ${HDR}) - -# Workaround to avoid warnings caused with bug in the avx512intrin.h of GCC5 -if((CMAKE_CXX_COMPILER_ID STREQUAL "GNU") AND - (CMAKE_CXX_COMPILER_VERSION VERSION_LESS_EQUAL 5.5)) - set_target_properties(${TARGET_NAME} PROPERTIES LINK_FLAGS_RELEASE "-Wno-error=maybe-uninitialized -Wno-maybe-uninitialized") -endif() - -target_include_directories(${TARGET_NAME} PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}") - -target_link_libraries(${TARGET_NAME} PRIVATE openvino_gapi_preproc_s inference_engine fluid opencv_gapi) - -add_dependencies(${TARGET_NAME} openvino_gapi_preproc) diff --git a/src/common/preprocessing/tests/fluid_test_computations/fluid_test_computations.cpp b/src/common/preprocessing/tests/fluid_test_computations/fluid_test_computations.cpp deleted file mode 100644 index 3210beb83a1..00000000000 --- a/src/common/preprocessing/tests/fluid_test_computations/fluid_test_computations.cpp +++ /dev/null @@ -1,314 +0,0 @@ -// Copyright (C) 2018-2023 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#include -#include -#include -#include - -#define CV_MAT_CHANNELS(flags) (((flags) >> CV_CN_SHIFT) + 1) - -struct FluidComputation::Priv -{ - cv::GComputation m_c; - cv::GRunArgs m_v_in; - std::vector m_v_out; - - Priv(cv::GComputation && c, std::vector&& v_in, std::vector&& v_out) - : m_c(std::move(c)), - m_v_in(v_in.begin(), v_in.end()), - m_v_out(std::move(v_out)) - {} - - Priv(cv::GComputation && c, cv::gapi::own::Mat&& v_in, cv::gapi::own::Mat&& v_out) - : m_c(std::move(c)), - m_v_in{std::move(v_in)}, - m_v_out{std::move(v_out)} - {} - - Priv(cv::GComputation && c, cv::gapi::own::Mat&& v_in, std::vector&& v_out) - : m_c(std::move(c)), - m_v_in{std::move(v_in)}, - m_v_out(std::move(v_out)) - {} - - Priv(cv::GComputation && c, cv::GRunArgs&& v_in, std::vector&& v_out) - : m_c(std::move(c)), - m_v_in(std::move(v_in)), - m_v_out(std::move(v_out)) - {} - - cv::GRunArgs ins() { return m_v_in;} - cv::GRunArgsP outs() { - cv::GRunArgsP call_outs; - - for (auto &m : m_v_out) { call_outs.emplace_back(&m); } - - return call_outs; - } - -}; - -FluidComputation::FluidComputation(Priv *priv) - : m_priv(priv) -{} - -namespace -{ - cv::gapi::own::Rect to_own(test::Rect rect) { return {rect.x, rect.y, rect.width, rect.height}; } -} - -void FluidComputation::warmUp(test::Rect roi) -{ - auto compile_args = roi.empty() ? cv::compile_args(InferenceEngine::gapi::preprocKernels()) - : cv::compile_args(InferenceEngine::gapi::preprocKernels(), - cv::GFluidOutputRois{{to_own(roi)}}); - - m_priv->m_c.apply(m_priv->ins(), m_priv->outs(), std::move(compile_args)); -} - -void FluidComputation::apply() -{ - m_priv->m_c.apply(m_priv->ins(), m_priv->outs()); -} - -namespace -{ -cv::gapi::own::Scalar to_own(test::Scalar const& s) { - return {s.v[0], s.v[1], s.v[2], s.v[3]}; -} - -cv::gapi::own::Mat to_own(test::Mat mat) { - return {mat.rows, mat.cols, mat.type, mat.data, mat.step}; -} - -std::vector to_own(std::vector mats) -{ - std::vector own_mats(mats.size()); - for (size_t i = 0; i < mats.size(); i++) { - own_mats[i] = to_own(mats[i]); - } - return own_mats; -} - -template -std::vector to_vec_impl(std::tuple &&gmats, cv::detail::Seq) { - return { std::get(gmats)... }; -} - -template -std::vector to_vec(std::tuple &&gmats) { - return to_vec_impl(std::move(gmats), typename cv::detail::MkSeq::type()); -} -} // anonymous namespace - -static cv::GComputation buildResizeComputation(test::Mat inMat, test::Mat outMat, int interp) -{ - cv::gapi::own::Size sz_in { inMat.cols, inMat.rows}; - cv::gapi::own::Size sz_out {outMat.cols, outMat.rows}; - int type = outMat.type; - cv::GMat in, out; - switch (CV_MAT_CHANNELS(type)) { - case 1: - out = InferenceEngine::gapi::ScalePlane::on(in, type, sz_in, sz_out, interp); - break; - case 3: - { - int depth = CV_MAT_DEPTH(type); - int type1 = CV_MAKE_TYPE(depth, 1); - cv::GMat in0, in1, in2, out0, out1, out2; - std::tie(in0, in1, in2) = InferenceEngine::gapi::Split3::on(in); - out0 = InferenceEngine::gapi::ScalePlane::on(in0, type1, sz_in, sz_out, interp); - out1 = InferenceEngine::gapi::ScalePlane::on(in1, type1, sz_in, sz_out, interp); - out2 = InferenceEngine::gapi::ScalePlane::on(in2, type1, sz_in, sz_out, interp); - out = InferenceEngine::gapi::Merge3::on(out0, out1, out2); - } - break; - default: GAPI_Assert(!"ERROR: unsupported number of channels!"); - } - - return cv::GComputation(in, out); -} - -FluidResizeComputation::FluidResizeComputation(test::Mat inMat, test::Mat outMat, int interp) - : FluidComputation(new Priv{buildResizeComputation(inMat, outMat, interp) - ,to_own(inMat) - ,to_own(outMat) - }) -{} - -static cv::GComputation buildResizeRGB8UComputation(test::Mat inMat, test::Mat outMat, int interp) -{ - cv::gapi::own::Size sz_in { inMat.cols, inMat.rows}; - cv::gapi::own::Size sz_out {outMat.cols, outMat.rows}; - int type = outMat.type; - cv::GMat in, out, out_r, out_g, out_b, out_x; - - if (type == CV_8UC3) { - std::tie(out_r, out_g, out_b) = InferenceEngine::gapi::ScalePlanes::on(in, type, sz_in, sz_out, interp); - out = InferenceEngine::gapi::Merge3::on(out_r, out_g, out_b); - } - else if (type == CV_8UC4) { - std::tie(out_r, out_g, out_b, out_x) = InferenceEngine::gapi::ScalePlanes4::on(in, type, sz_in, sz_out, interp); - out = InferenceEngine::gapi::Merge4::on(out_r, out_g, out_b, out_x); - } else { - GAPI_Assert(!"ERROR: unsupported number of channels!"); - } - - return cv::GComputation(in, out); -} - -FluidResizeRGB8UComputation::FluidResizeRGB8UComputation(test::Mat inMat, test::Mat outMat, int interp) - : FluidComputation(new Priv{buildResizeRGB8UComputation(inMat, outMat, interp) - ,to_own(inMat) - ,to_own(outMat) - }) -{} - -static cv::GComputation buildSplitComputation(int planes) -{ - std::vector ins(1); - std::vector outs(planes); - - switch (planes) { - case 2: outs = to_vec(InferenceEngine::gapi::Split2::on(ins[0])); break; - case 3: outs = to_vec(InferenceEngine::gapi::Split3::on(ins[0])); break; - case 4: outs = to_vec(InferenceEngine::gapi::Split4::on(ins[0])); break; - default: GAPI_Assert(false); - } - - return cv::GComputation(ins, outs); -} - -FluidSplitComputation::FluidSplitComputation(test::Mat inMat, std::vector outMats) - : FluidComputation(new Priv{buildSplitComputation(outMats.size()) - ,to_own(inMat) - ,to_own(outMats) - }) -{} - -static cv::GComputation buildChanToPlaneComputation(int chan) -{ - cv::GMat in, out; - out = InferenceEngine::gapi::ChanToPlane::on(in, chan); - return cv::GComputation(in, out); -} - -FluidChanToPlaneComputation::FluidChanToPlaneComputation(test::Mat inMat, test::Mat outMat, int chan) - : FluidComputation(new Priv{buildChanToPlaneComputation(chan) - ,to_own(inMat) - ,to_own(outMat) - }) -{} - -static cv::GComputation buildMergeComputation(int planes) -{ - std::vector ins(planes); - std::vector outs(1); - - switch (planes) { - case 2: outs[0] = InferenceEngine::gapi::Merge2::on(ins[0], ins[1]); break; - case 3: outs[0] = InferenceEngine::gapi::Merge3::on(ins[0], ins[1], ins[2]); break; - case 4: outs[0] = InferenceEngine::gapi::Merge4::on(ins[0], ins[1], ins[2], ins[3]); break; - default: GAPI_Assert(false); - } - - return cv::GComputation(ins, outs); -} - -FluidMergeComputation::FluidMergeComputation(std::vector inMats, test::Mat outMat) - : FluidComputation(new Priv{buildMergeComputation(inMats.size()) - ,to_own(inMats) - ,{to_own(outMat)} - }) -{} - -static cv::GComputation buildFluidNV12toRGBComputation() -{ - cv::GMat in_y, in_uv; - cv::GMat out = InferenceEngine::gapi::NV12toRGB::on(in_y,in_uv); - return cv::GComputation(cv::GIn(in_y,in_uv), cv::GOut(out)); -} - -FluidNV12toRGBComputation::FluidNV12toRGBComputation(test::Mat inMat_y, test::Mat inMat_uv, test::Mat outMat) - : FluidComputation(new Priv{buildFluidNV12toRGBComputation() - ,to_own({inMat_y,inMat_uv}) - ,{to_own(outMat)} - }) -{} - -static cv::GComputation buildFluidI420toRGBComputation() -{ - cv::GMat in_y, in_u, in_v; - cv::GMat out = InferenceEngine::gapi::I420toRGB::on(in_y, in_u, in_v); - return cv::GComputation(cv::GIn(in_y, in_u, in_v), cv::GOut(out)); -} - - -FluidI420toRGBComputation::FluidI420toRGBComputation(test::Mat inMat_y, test::Mat inMat_u, test::Mat inMat_v, test::Mat outMat) - : FluidComputation(new Priv{buildFluidI420toRGBComputation() - ,to_own({inMat_y,inMat_u, inMat_v}) - ,{to_own(outMat)} - }) -{} - -ConvertDepthComputation::ConvertDepthComputation(test::Mat inMat, test::Mat outMat, int depth) - : FluidComputation(new Priv{ [depth]()-> cv::GComputation { - cv::GMat in; - cv::GMat out = InferenceEngine::gapi::ConvertDepth::on(in, depth); - return cv::GComputation(cv::GIn(in), cv::GOut(out)); - }() - , to_own(inMat) - , to_own(outMat) - }) -{} - -DivCComputation::DivCComputation(test::Mat inMat, test::Mat outMat, test::Scalar const& c) - : FluidComputation(new Priv{ []()-> cv::GComputation { - cv::GMat in; - cv::GScalar C; - cv::GMat out = in / C; - return cv::GComputation(cv::GIn(in, C), cv::GOut(out)); - }() - , cv::GRunArgs{cv::GRunArg{to_own(inMat)}, cv::GRunArg{to_own(c)}} - , {to_own(outMat)} - }) -{} - -SubCComputation::SubCComputation(test::Mat inMat, test::Mat outMat, test::Scalar const& c) - : FluidComputation(new Priv{ []()-> cv::GComputation{ - cv::GMat in; - cv::GScalar C; - cv::GMat out = in - C; - return cv::GComputation(cv::GIn(in, C), cv::GOut(out)); - }() - , cv::GRunArgs{cv::GRunArg{to_own(inMat)}, cv::GRunArg{to_own(c)}} - , {to_own(outMat)} - }) -{} - -MeanValueSubtractComputation::MeanValueSubtractComputation(test::Mat inMat, test::Mat outMat, test::Scalar const& mean, test::Scalar const& std) - : FluidComputation(new Priv{ []()-> cv::GComputation{ - cv::GMat in; - cv::GScalar _mean; - cv::GScalar _std; - cv::GMat out = (in - _mean) / _std; - return cv::GComputation(cv::GIn(in, _mean, _std), cv::GOut(out)); - }() - , cv::GRunArgs{cv::GRunArg{to_own(inMat)}, cv::GRunArg{to_own(mean)}, cv::GRunArg{to_own(std)}} - , {to_own(outMat)} - }) -{} - -namespace cv { -cv::GMat operator-(const cv::GMat& lhs, const cv::GScalar& rhs) -{ - return InferenceEngine::gapi::GSubC::on(lhs, rhs, -1); -} -cv::GMat operator/(const cv::GMat& lhs, const cv::GScalar& rhs) -{ - return InferenceEngine::gapi::GDivC::on(lhs, rhs, 1.0, -1); -} - -} diff --git a/src/common/preprocessing/tests/fluid_test_computations/fluid_test_computations.hpp b/src/common/preprocessing/tests/fluid_test_computations/fluid_test_computations.hpp deleted file mode 100644 index d9f3e7c5513..00000000000 --- a/src/common/preprocessing/tests/fluid_test_computations/fluid_test_computations.hpp +++ /dev/null @@ -1,117 +0,0 @@ -// Copyright (C) 2018-2023 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#ifndef FLUID_TEST_COMPUTATIONS_HPP -#define FLUID_TEST_COMPUTATIONS_HPP - -#include - -#include -#include -#include - -namespace test -{ -struct Mat -{ - int rows; - int cols; - int type; - void* data; - size_t step; -}; -struct Rect{ - int x; - int y; - int width; - int height; - bool empty(){ - return width == 0 && height == 0; - }; -}; -struct Scalar -{ - std::array v; -}; - -} - -class FluidComputation -{ -protected: - struct Priv; - std::shared_ptr m_priv; -public: - FluidComputation(Priv* priv); - void warmUp(test::Rect roi = {}); - void apply(); -}; - -class FluidResizeComputation : public FluidComputation -{ -public: - FluidResizeComputation(test::Mat inMat, test::Mat outMat, int interp); -}; - -class FluidResizeRGB8UComputation : public FluidComputation -{ -public: - FluidResizeRGB8UComputation(test::Mat inMat, test::Mat outMat, int interp); -}; - -class FluidSplitComputation : public FluidComputation -{ -public: - FluidSplitComputation(test::Mat inMat, std::vector outMats); -}; - -class FluidChanToPlaneComputation : public FluidComputation -{ -public: - FluidChanToPlaneComputation(test::Mat inMat, test::Mat outMat, int chan); -}; - -class FluidMergeComputation : public FluidComputation -{ -public: - FluidMergeComputation(std::vector inMats, test::Mat outMat); -}; - -class FluidNV12toRGBComputation : public FluidComputation -{ -public: - FluidNV12toRGBComputation(test::Mat inMat_y, test::Mat inMat_uv, test::Mat outMat); -}; - -class FluidI420toRGBComputation : public FluidComputation -{ -public: - FluidI420toRGBComputation(test::Mat inMat_y, test::Mat inMat_u, test::Mat inMat_v, test::Mat outMat); -}; - -class ConvertDepthComputation : public FluidComputation -{ -public: - ConvertDepthComputation(test::Mat inMat, test::Mat outMat, int depth); -}; - -class DivCComputation : public FluidComputation -{ -public: - DivCComputation(test::Mat inMat, test::Mat outMat, test::Scalar const& c); -}; - -class SubCComputation : public FluidComputation -{ -public: - SubCComputation(test::Mat inMat, test::Mat outMat, test::Scalar const& c); -}; - -class MeanValueSubtractComputation : public FluidComputation -{ -public: - MeanValueSubtractComputation(test::Mat inMat, test::Mat outMat, test::Scalar const& mean, test::Scalar const& std); -}; - -#endif // FLUID_TEST_COMPUTATIONS_HPP diff --git a/src/inference/CMakeLists.txt b/src/inference/CMakeLists.txt index 6525d181c02..2e2c7028788 100644 --- a/src/inference/CMakeLists.txt +++ b/src/inference/CMakeLists.txt @@ -85,15 +85,11 @@ source_group("include" FILES ${LIBRARY_HEADERS} ${PUBLIC_HEADERS}) add_library(${TARGET_NAME}_plugin_api INTERFACE) target_include_directories(${TARGET_NAME}_plugin_api INTERFACE - $ $ $ $ $) -target_compile_definitions(${TARGET_NAME}_plugin_api INTERFACE - $) - target_link_libraries(${TARGET_NAME}_plugin_api INTERFACE openvino::pugixml openvino::itt openvino::util) ov_set_threading_interface_for(${TARGET_NAME}_plugin_api) @@ -128,8 +124,7 @@ ov_build_target_faster(${TARGET_NAME}_obj target_compile_definitions(${TARGET_NAME}_obj PRIVATE IMPLEMENT_INFERENCE_ENGINE_API $<$:PROXY_PLUGIN_ENABLED> - $ - $) + $) target_include_directories(${TARGET_NAME}_obj SYSTEM PRIVATE $ diff --git a/src/inference/dev_api/cpp_interfaces/interface/ie_iinfer_request_internal.hpp b/src/inference/dev_api/cpp_interfaces/interface/ie_iinfer_request_internal.hpp index 0bd5402c6d1..4b22faa5339 100644 --- a/src/inference/dev_api/cpp_interfaces/interface/ie_iinfer_request_internal.hpp +++ b/src/inference/dev_api/cpp_interfaces/interface/ie_iinfer_request_internal.hpp @@ -4,6 +4,7 @@ #pragma once +#include #include #include #include @@ -12,7 +13,6 @@ #include "ie_common.h" #include "ie_compound_blob.h" #include "ie_input_info.hpp" -#include "ie_preprocess_data.hpp" #include "openvino/core/node_output.hpp" #include "so_ptr.hpp" @@ -328,9 +328,22 @@ protected: InferenceEngine::BlobMap _outputs; //!< A map of user passed blobs for network outputs std::vector> _parameters; //!< A vector of function inputs std::vector> _results; //!< A vector of function outputs - std::map _preProcData; //!< A map of pre-process data per input std::map _batched_inputs; //!< A map of user passed blobs for network inputs + class PreProcessDataPlugin { + public: + void setRoiBlob(const Blob::Ptr& blob) {} + + Blob::Ptr getRoiBlob() const { + return nullptr; + } + + void execute(Blob::Ptr& preprocessedBlob, const PreProcessInfo& info, bool serial, int batchSize = -1) {} + + void isApplicable(const Blob::Ptr& src, const Blob::Ptr& dst) {} + }; + std::map> _preProcData; //!< A map of pre-process data per input + /** * @brief A shared pointer to IInferRequestInternal * @note Needed to correctly handle ownership between objects. diff --git a/src/inference/src/cpp_interfaces/interface/ie_iinfer_request_internal.cpp b/src/inference/src/cpp_interfaces/interface/ie_iinfer_request_internal.cpp index af12db6c429..adf7a0a7dbd 100644 --- a/src/inference/src/cpp_interfaces/interface/ie_iinfer_request_internal.cpp +++ b/src/inference/src/cpp_interfaces/interface/ie_iinfer_request_internal.cpp @@ -143,24 +143,19 @@ void IInferRequestInternal::SetBlob(const std::string& name, const Blob::Ptr& us } auto& devBlob = _deviceInputs[name]; - const bool preProcRequired = preProcessingRequired(foundInput, userBlob, devBlob); - if (compoundBlobPassed && !preProcRequired) { + if (compoundBlobPassed) { IE_THROW(NotImplemented) << "cannot set compound blob: supported only for input pre-processing"; } - if (preProcRequired) { - addInputPreProcessingFor(name, userBlob, devBlob ? devBlob : _inputs[name]); - } else { - size_t inputSize = foundInput->getTensorDesc().getLayout() != InferenceEngine::Layout::SCALAR - ? InferenceEngine::details::product(foundInput->getTensorDesc().getDims()) - : 1; - if (!isInputDynamic && dataSize != inputSize) { - IE_THROW() << "Input tensor size is not equal network input size (" << dataSize << "!=" << inputSize - << ")."; - } - _inputs[name] = userBlob; - devBlob = userBlob; + size_t inputSize = foundInput->getTensorDesc().getLayout() != InferenceEngine::Layout::SCALAR + ? InferenceEngine::details::product(foundInput->getTensorDesc().getDims()) + : 1; + if (!isInputDynamic && dataSize != inputSize) { + IE_THROW() << "Input tensor size is not equal network input size (" << dataSize << "!=" << inputSize + << ")."; } + _inputs[name] = userBlob; + devBlob = userBlob; _batched_inputs.erase(name); } else { if (compoundBlobPassed) { @@ -369,24 +364,12 @@ Blob::Ptr IInferRequestInternal::GetBlob(const std::string& name) { if (findInputAndOutputBlobByName(name, foundInput, foundOutput)) { const auto input = findInputByNodeName(name); const bool isInputDynamic = input && input->get_output_partial_shape(0).is_dynamic(); - // ROI blob is returned only if it was set previously. Otherwise default blob is returned. - auto it = _preProcData.find(name); - if (it != _preProcData.end()) { - data = it->second->getRoiBlob(); - } else { - data = _inputs[name]; - const auto& dims = foundInput->getTensorDesc().getDims(); - if (isInputDynamic) - checkBlob(data, name, true); - else - checkBlob(data, name, true, foundInput->getTensorDesc().getLayout() != SCALAR ? dims : oneVector); - - auto& devBlob = _deviceInputs[name]; - if (preProcessingRequired(foundInput, data, devBlob)) { - // if no devBlob, performs inplace - addInputPreProcessingFor(name, data, devBlob ? devBlob : _inputs[name]); - } - } + data = _inputs[name]; + const auto& dims = foundInput->getTensorDesc().getDims(); + if (isInputDynamic) + checkBlob(data, name, true); + else + checkBlob(data, name, true, foundInput->getTensorDesc().getLayout() != SCALAR ? dims : oneVector); } else { const auto output = findOutputByNodeName(name); const bool isOutputDynamic = output && output->get_output_partial_shape(0).is_dynamic(); @@ -438,16 +421,7 @@ void IInferRequestInternal::SetCallback(Callback callback) { _callback = std::move(callback); } -void IInferRequestInternal::execDataPreprocessing(InferenceEngine::BlobMap& preprocessedBlobs, bool serial) { - for (auto& input : preprocessedBlobs) { - // If there is a pre-process entry for an input then it must be pre-processed - // using preconfigured resize algorithm. - auto it = _preProcData.find(input.first); - if (it != _preProcData.end()) { - it->second->execute(input.second, _networkInputs[input.first]->getPreProcess(), serial, -1); - } - } -} +void IInferRequestInternal::execDataPreprocessing(InferenceEngine::BlobMap& preprocessedBlobs, bool serial) {} bool IInferRequestInternal::findInputAndOutputBlobByName(const std::string& name, InputInfo::Ptr& foundInput, @@ -569,56 +543,6 @@ std::shared_ptr IInferRequestInternal::getPointerToSo() const { return _so; } -bool IInferRequestInternal::preProcessingRequired(const InputInfo::Ptr& info, - const Blob::Ptr& userBlob, - const Blob::Ptr& deviceBlob) { - // pre-processing is required if: - // 1. resize algorithm is specified (resize required) - // 2. color format specified: - // 2.a. color format is not equal to network's expected (color conversion required) - // 2.b. network's layout != blob's layout (reorder required) - // 3. precision conversion is required - - const auto& preProcessInfo = info->getPreProcess(); - const auto inputColorFormat = preProcessInfo.getColorFormat(); - // FIXME: support other network's input formats once the API is ready. Assuming input is in - // the BGR format by default - const auto networkColorFormat = ColorFormat::BGR; - const bool colorFormatSpecified = inputColorFormat != ColorFormat::RAW; - - auto blob_layout = [](const Blob::Ptr& b) { - return b->getTensorDesc().getLayout(); - }; - auto blob_prec = [](const Blob::Ptr& b) { - return b->getTensorDesc().getPrecision(); - }; - - auto dst_layout = deviceBlob ? blob_layout(deviceBlob) : info->getLayout(); - auto dst_prec = deviceBlob ? blob_prec(deviceBlob) : info->getPrecision(); - - // FIXME: remove the first part to allow any needed conversion? - const bool need_layout_conv = (colorFormatSpecified || deviceBlob) && (blob_layout(userBlob) != dst_layout); - - return preProcessInfo.getResizeAlgorithm() != ResizeAlgorithm::NO_RESIZE || - (colorFormatSpecified && inputColorFormat != networkColorFormat) || need_layout_conv || - (blob_prec(userBlob) != dst_prec); -} - -void IInferRequestInternal::addInputPreProcessingFor(const std::string& name, - const Blob::Ptr& from, - const Blob::Ptr& to) { - auto ppDataIt = _preProcData.find(name); - if (ppDataIt == _preProcData.end()) { - ppDataIt = (_preProcData.emplace(name, CreatePreprocDataHelper())).first; - } - - auto& preproc_ptr = ppDataIt->second; - preproc_ptr->isApplicable(from, to); - // Stores the given blob as ROI blob. It will be used to fill in network input - // during pre-processing - preproc_ptr->setRoiBlob(from); -} - void* IInferRequestInternal::GetUserData() noexcept { return _userData; } diff --git a/src/inference/src/dev/core_impl.cpp b/src/inference/src/dev/core_impl.cpp index 66627f58e63..93c6c0c54fe 100644 --- a/src/inference/src/dev/core_impl.cpp +++ b/src/inference/src/dev/core_impl.cpp @@ -803,8 +803,7 @@ ov::SoPtr ov::CoreImpl::compile_model_with_preprocess(ov::Pl std::shared_ptr preprocessed_model = model; // Disable conversion for proxy plugin and virtual devices to add pre-processing based on API of internal plugins - if (!is_new_api() && !std::dynamic_pointer_cast(plugin.m_ptr) && - !is_virtual_device(plugin.get_name()) && !is_proxy_device(plugin)) { + if (!is_new_api() && !is_virtual_device(plugin.get_name()) && !is_proxy_device(plugin)) { ov::pass::Manager manager; manager.register_pass(); diff --git a/src/plugins/auto/tests/functional/shared_tests_instances/behavior/plugin/set_preprocess.cpp b/src/plugins/auto/tests/functional/shared_tests_instances/behavior/plugin/set_preprocess.cpp deleted file mode 100644 index b75c4a4a93c..00000000000 --- a/src/plugins/auto/tests/functional/shared_tests_instances/behavior/plugin/set_preprocess.cpp +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright (C) 2018-2023 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#include "behavior/plugin/set_preprocess.hpp" - -#ifdef ENABLE_GAPI_PREPROCESSING - -using namespace BehaviorTestsDefinitions; -namespace { -const std::vector netPrecisions = {InferenceEngine::Precision::FP32, - InferenceEngine::Precision::FP16}; - -const std::vector> multiConfigs = { - {{InferenceEngine::MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES, ov::test::utils::DEVICE_TEMPLATE}}}; - -const std::vector ioPrecisions = {InferenceEngine::Precision::FP32, - InferenceEngine::Precision::U8}; -const std::vector netLayouts = { - InferenceEngine::Layout::NCHW, - // InferenceEngine::Layout::NHWC -}; - -const std::vector ioLayouts = {InferenceEngine::Layout::NCHW, InferenceEngine::Layout::NHWC}; - -INSTANTIATE_TEST_SUITE_P(smoke_Multi_BehaviorTests, - InferRequestPreprocessConversionTest, - ::testing::Combine(::testing::ValuesIn(netPrecisions), - ::testing::ValuesIn(ioPrecisions), - ::testing::ValuesIn(ioPrecisions), - ::testing::ValuesIn(netLayouts), - ::testing::ValuesIn(ioLayouts), - ::testing::ValuesIn(ioLayouts), - ::testing::Bool(), - ::testing::Bool(), - ::testing::Values(ov::test::utils::DEVICE_MULTI), - ::testing::ValuesIn(multiConfigs)), - InferRequestPreprocessConversionTest::getTestCaseName); - -INSTANTIATE_TEST_SUITE_P(smoke_Multi_BehaviorTests, - InferRequestPreprocessDynamicallyInSetBlobTest, - ::testing::Combine(::testing::ValuesIn(netPrecisions), - ::testing::Bool(), - ::testing::Bool(), - ::testing::ValuesIn(netLayouts), - ::testing::Bool(), - ::testing::Bool(), - ::testing::Values(true), // only SetBlob - ::testing::Values(true), // only SetBlob - ::testing::Values(ov::test::utils::DEVICE_MULTI), - ::testing::ValuesIn(multiConfigs)), - InferRequestPreprocessDynamicallyInSetBlobTest::getTestCaseName); - -INSTANTIATE_TEST_SUITE_P(smoke_Auto_BehaviorTests, - InferRequestPreprocessConversionTest, - ::testing::Combine(::testing::ValuesIn(netPrecisions), - ::testing::ValuesIn(ioPrecisions), - ::testing::ValuesIn(ioPrecisions), - ::testing::ValuesIn(netLayouts), - ::testing::ValuesIn(ioLayouts), - ::testing::ValuesIn(ioLayouts), - ::testing::Bool(), - ::testing::Bool(), - ::testing::Values(ov::test::utils::DEVICE_AUTO), - ::testing::ValuesIn(multiConfigs)), - InferRequestPreprocessConversionTest::getTestCaseName); - -INSTANTIATE_TEST_SUITE_P(smoke_Auto_BehaviorTests, - InferRequestPreprocessDynamicallyInSetBlobTest, - ::testing::Combine(::testing::ValuesIn(netPrecisions), - ::testing::Bool(), - ::testing::Bool(), - ::testing::ValuesIn(netLayouts), - ::testing::Bool(), - ::testing::Bool(), - ::testing::Values(true), // only SetBlob - ::testing::Values(true), // only SetBlob - ::testing::Values(ov::test::utils::DEVICE_AUTO), - ::testing::ValuesIn(multiConfigs)), - InferRequestPreprocessDynamicallyInSetBlobTest::getTestCaseName); - -} // namespace - -#endif // ENABLE_GAPI_PREPROCESSING diff --git a/src/plugins/intel_cpu/tests/functional/shared_tests_instances/behavior/plugin/preprocessing.cpp b/src/plugins/intel_cpu/tests/functional/shared_tests_instances/behavior/plugin/preprocessing.cpp deleted file mode 100644 index e0b1ac0f607..00000000000 --- a/src/plugins/intel_cpu/tests/functional/shared_tests_instances/behavior/plugin/preprocessing.cpp +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright (C) 2018-2023 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#include "behavior/plugin/preprocessing.hpp" - -#ifdef ENABLE_GAPI_PREPROCESSING - -using namespace BehaviorTestsDefinitions; - -namespace { - -const std::vector inputPrecisions = { - InferenceEngine::Precision::U16, - InferenceEngine::Precision::FP32 -}; - -const std::vector> configs = { - {} -}; - -INSTANTIATE_TEST_SUITE_P(BehaviourPreprocessingTestsViaSetInput, PreprocessingPrecisionConvertTest, - ::testing::Combine( - ::testing::ValuesIn(inputPrecisions), - ::testing::Values(1, 2, 3, 4, 5), // Number of input tensor channels - ::testing::Values(true), // Use SetInput - ::testing::Values(ov::test::utils::DEVICE_CPU), - ::testing::ValuesIn(configs)), - PreprocessingPrecisionConvertTest::getTestCaseName); - -INSTANTIATE_TEST_SUITE_P(BehaviourPreprocessingTestsViaGetBlob, PreprocessingPrecisionConvertTest, - ::testing::Combine( - ::testing::ValuesIn(inputPrecisions), - ::testing::Values(4, 5), // Number of input tensor channels (blob_copy only supports 4d and 5d tensors) - ::testing::Values(false), // use GetBlob - ::testing::Values(ov::test::utils::DEVICE_CPU), - ::testing::ValuesIn(configs)), - PreprocessingPrecisionConvertTest::getTestCaseName); -} // namespace - -#endif // ENABLE_GAPI_PREPROCESSING diff --git a/src/plugins/intel_cpu/tests/functional/shared_tests_instances/behavior/plugin/set_preprocess.cpp b/src/plugins/intel_cpu/tests/functional/shared_tests_instances/behavior/plugin/set_preprocess.cpp deleted file mode 100644 index 49664de10b4..00000000000 --- a/src/plugins/intel_cpu/tests/functional/shared_tests_instances/behavior/plugin/set_preprocess.cpp +++ /dev/null @@ -1,111 +0,0 @@ -// Copyright (C) 2018-2023 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#include "behavior/plugin/set_preprocess.hpp" - -#ifdef ENABLE_GAPI_PREPROCESSING - -using namespace BehaviorTestsDefinitions; -namespace { - const std::vector netPrecisions = { - InferenceEngine::Precision::FP32, - InferenceEngine::Precision::FP16 - }; - - const std::vector> configs = { - {}, - {{InferenceEngine::PluginConfigParams::KEY_CPU_THROUGHPUT_STREAMS, InferenceEngine::PluginConfigParams::CPU_THROUGHPUT_AUTO}}, - {{InferenceEngine::PluginConfigParams::KEY_CPU_THROUGHPUT_STREAMS, "0"}, {InferenceEngine::PluginConfigParams::KEY_CPU_THREADS_NUM, "1"}} - }; - - const std::vector> heteroConfigs = { - {{ "TARGET_FALLBACK" , ov::test::utils::DEVICE_CPU}} - }; - - INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, InferRequestPreprocessTest, - ::testing::Combine( - ::testing::ValuesIn(netPrecisions), - ::testing::Values(ov::test::utils::DEVICE_CPU), - ::testing::ValuesIn(configs)), - InferRequestPreprocessTest::getTestCaseName); - - INSTANTIATE_TEST_SUITE_P(smoke_Hetero_BehaviorTests, InferRequestPreprocessTest, - ::testing::Combine( - ::testing::ValuesIn(netPrecisions), - ::testing::Values(ov::test::utils::DEVICE_HETERO), - ::testing::ValuesIn(heteroConfigs)), - InferRequestPreprocessTest::getTestCaseName); - - const std::vector ioPrecisions = { - InferenceEngine::Precision::FP32, - InferenceEngine::Precision::U8 - }; - const std::vector netLayouts = { - InferenceEngine::Layout::NCHW, - // InferenceEngine::Layout::NHWC - }; - - const std::vector ioLayouts = { - InferenceEngine::Layout::NCHW, - InferenceEngine::Layout::NHWC - }; - - INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, InferRequestPreprocessConversionTest, - ::testing::Combine( - ::testing::ValuesIn(netPrecisions), - ::testing::ValuesIn(ioPrecisions), - ::testing::ValuesIn(ioPrecisions), - ::testing::ValuesIn(netLayouts), - ::testing::ValuesIn(ioLayouts), - ::testing::ValuesIn(ioLayouts), - ::testing::Bool(), - ::testing::Bool(), - ::testing::Values(ov::test::utils::DEVICE_CPU), - ::testing::ValuesIn(configs)), - InferRequestPreprocessConversionTest::getTestCaseName); - - INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, InferRequestPreprocessDynamicallyInSetBlobTest, - ::testing::Combine( - ::testing::ValuesIn(netPrecisions), - ::testing::Bool(), - ::testing::Bool(), - ::testing::ValuesIn(netLayouts), - ::testing::Bool(), - ::testing::Bool(), - ::testing::Values(true), // only SetBlob - ::testing::Values(true), // only SetBlob - ::testing::Values(ov::test::utils::DEVICE_CPU), - ::testing::ValuesIn(configs)), - InferRequestPreprocessDynamicallyInSetBlobTest::getTestCaseName); - - INSTANTIATE_TEST_SUITE_P(smoke_Hetero_BehaviorTests, InferRequestPreprocessConversionTest, - ::testing::Combine( - ::testing::ValuesIn(netPrecisions), - ::testing::ValuesIn(ioPrecisions), - ::testing::ValuesIn(ioPrecisions), - ::testing::ValuesIn(netLayouts), - ::testing::ValuesIn(ioLayouts), - ::testing::ValuesIn(ioLayouts), - ::testing::Bool(), - ::testing::Bool(), - ::testing::Values(ov::test::utils::DEVICE_HETERO), - ::testing::ValuesIn(heteroConfigs)), - InferRequestPreprocessConversionTest::getTestCaseName); - - INSTANTIATE_TEST_SUITE_P(smoke_Hetero_BehaviorTests, InferRequestPreprocessDynamicallyInSetBlobTest, - ::testing::Combine( - ::testing::ValuesIn(netPrecisions), - ::testing::Bool(), - ::testing::Bool(), - ::testing::ValuesIn(netLayouts), - ::testing::Bool(), - ::testing::Bool(), - ::testing::Values(true), // only SetBlob - ::testing::Values(true), // only SetBlob - ::testing::Values(ov::test::utils::DEVICE_HETERO), - ::testing::ValuesIn(heteroConfigs)), - InferRequestPreprocessDynamicallyInSetBlobTest::getTestCaseName); -} // namespace - -#endif // ENABLE_GAPI_PREPROCESSING diff --git a/src/plugins/intel_gpu/tests/functional/shared_tests_instances/behavior/plugin/set_preprocess.cpp b/src/plugins/intel_gpu/tests/functional/shared_tests_instances/behavior/plugin/set_preprocess.cpp deleted file mode 100644 index 2ffabe771b6..00000000000 --- a/src/plugins/intel_gpu/tests/functional/shared_tests_instances/behavior/plugin/set_preprocess.cpp +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright (C) 2018-2023 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#include -#include "behavior/plugin/set_preprocess.hpp" - -#ifdef ENABLE_GAPI_PREPROCESSING - -using namespace BehaviorTestsDefinitions; -namespace { - using PreprocessBehTest = BehaviorTestsUtils::BehaviorTestsBasic; - - const std::vector netPrecisions = { - InferenceEngine::Precision::FP32, - InferenceEngine::Precision::FP16 - }; - - auto configs = []() { - return std::vector>{ - {}, - }; - }; - - INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, InferRequestPreprocessTest, - ::testing::Combine( - ::testing::ValuesIn(netPrecisions), - ::testing::Values(ov::test::utils::DEVICE_GPU), - ::testing::ValuesIn(configs())), - InferRequestPreprocessTest::getTestCaseName); - - const std::vector ioPrecisions = { - InferenceEngine::Precision::FP32, - InferenceEngine::Precision::U8 - }; - const std::vector netLayouts = { - InferenceEngine::Layout::NCHW, - // InferenceEngine::Layout::NHWC - }; - - const std::vector ioLayouts = { - InferenceEngine::Layout::NCHW, - InferenceEngine::Layout::NHWC - }; - - INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, InferRequestPreprocessConversionTest, - ::testing::Combine( - ::testing::ValuesIn(netPrecisions), - ::testing::ValuesIn(ioPrecisions), - ::testing::ValuesIn(ioPrecisions), - ::testing::ValuesIn(netLayouts), - ::testing::ValuesIn(ioLayouts), - ::testing::ValuesIn(ioLayouts), - ::testing::Bool(), - ::testing::Bool(), - ::testing::Values(ov::test::utils::DEVICE_GPU), - ::testing::ValuesIn(configs())), - InferRequestPreprocessConversionTest::getTestCaseName); - - INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, InferRequestPreprocessDynamicallyInSetBlobTest, - ::testing::Combine( - ::testing::ValuesIn(netPrecisions), - ::testing::Bool(), - ::testing::Bool(), - ::testing::ValuesIn(netLayouts), - ::testing::Bool(), - ::testing::Bool(), - ::testing::Values(true), // only SetBlob - ::testing::Values(true), // only SetBlob - ::testing::Values(ov::test::utils::DEVICE_GPU), - ::testing::ValuesIn(configs())), - InferRequestPreprocessDynamicallyInSetBlobTest::getTestCaseName); - -} // namespace - -#endif // ENABLE_GAPI_PREPROCESSING diff --git a/src/plugins/template/tests/functional/shared_tests_instances/behavior/plugin/preprocessing.cpp b/src/plugins/template/tests/functional/shared_tests_instances/behavior/plugin/preprocessing.cpp deleted file mode 100644 index ea8d9a2e622..00000000000 --- a/src/plugins/template/tests/functional/shared_tests_instances/behavior/plugin/preprocessing.cpp +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright (C) 2018-2023 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#include "behavior/plugin/preprocessing.hpp" - -#ifdef ENABLE_GAPI_PREPROCESSING - -using namespace BehaviorTestsDefinitions; - -namespace { - -const std::vector inputPrecisions = {InferenceEngine::Precision::U8, - InferenceEngine::Precision::FP32}; - -const std::vector> configs = {{}}; - -INSTANTIATE_TEST_SUITE_P(smoke_Behavior_PreprocessingPrecisionConvertTestsViaSetInput, - PreprocessingPrecisionConvertTest, - ::testing::Combine(::testing::ValuesIn(inputPrecisions), - ::testing::Values(4), // Number of input tensor channels - ::testing::Values(true), // Use SetInput - ::testing::Values(ov::test::utils::DEVICE_TEMPLATE), - ::testing::ValuesIn(configs)), - PreprocessingPrecisionConvertTest::getTestCaseName); - -INSTANTIATE_TEST_SUITE_P(smoke_Behavior_PreprocessingPrecisionConvertTestsViaGetBlob, - PreprocessingPrecisionConvertTest, - ::testing::Combine(::testing::ValuesIn(inputPrecisions), - ::testing::Values(4), // Number of input tensor channels (blob_copy only - // supports 4d and 5d tensors) - ::testing::Values(false), // use GetBlob - ::testing::Values(ov::test::utils::DEVICE_TEMPLATE), - ::testing::ValuesIn(configs)), - PreprocessingPrecisionConvertTest::getTestCaseName); - -} // namespace - -#endif // ENABLE_GAPI_PREPROCESSING diff --git a/src/plugins/template/tests/functional/shared_tests_instances/behavior/plugin/set_preprocess.cpp b/src/plugins/template/tests/functional/shared_tests_instances/behavior/plugin/set_preprocess.cpp deleted file mode 100644 index e570c6fc0f6..00000000000 --- a/src/plugins/template/tests/functional/shared_tests_instances/behavior/plugin/set_preprocess.cpp +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright (C) 2018-2023 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#include "behavior/plugin/set_preprocess.hpp" - -using namespace BehaviorTestsDefinitions; - -namespace { - -const std::vector netPrecisions = {InferenceEngine::Precision::FP32, - InferenceEngine::Precision::FP16}; - -const std::vector> configs = {{}}; - -const std::vector> multiConfigs = { - {{InferenceEngine::MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES, ov::test::utils::DEVICE_TEMPLATE}}}; - -const std::vector> heteroConfigs = { - {{"TARGET_FALLBACK", ov::test::utils::DEVICE_TEMPLATE}}}; - -INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, - InferRequestPreprocessTest, - ::testing::Combine(::testing::ValuesIn(netPrecisions), - ::testing::Values(ov::test::utils::DEVICE_TEMPLATE), - ::testing::ValuesIn(configs)), - InferRequestPreprocessTest::getTestCaseName); - -INSTANTIATE_TEST_SUITE_P(smoke_Multi_BehaviorTests, - InferRequestPreprocessTest, - ::testing::Combine(::testing::ValuesIn(netPrecisions), - ::testing::Values(ov::test::utils::DEVICE_MULTI), - ::testing::ValuesIn(multiConfigs)), - InferRequestPreprocessTest::getTestCaseName); - -INSTANTIATE_TEST_SUITE_P(smoke_Hetero_BehaviorTests, - InferRequestPreprocessTest, - ::testing::Combine(::testing::ValuesIn(netPrecisions), - ::testing::Values(ov::test::utils::DEVICE_HETERO), - ::testing::ValuesIn(heteroConfigs)), - InferRequestPreprocessTest::getTestCaseName); - -const std::vector ioPrecisions = {InferenceEngine::Precision::FP32, - InferenceEngine::Precision::U8}; -const std::vector netLayouts = { - InferenceEngine::Layout::NCHW, -}; - -const std::vector ioLayouts = { - InferenceEngine::Layout::NCHW, - InferenceEngine::Layout::NHWC, -}; - -INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, - InferRequestPreprocessConversionTest, - ::testing::Combine(::testing::ValuesIn(netPrecisions), - ::testing::ValuesIn(ioPrecisions), - ::testing::ValuesIn(ioPrecisions), - ::testing::ValuesIn(netLayouts), - ::testing::ValuesIn(ioLayouts), - ::testing::ValuesIn(ioLayouts), - ::testing::Bool(), - ::testing::Bool(), - ::testing::Values(ov::test::utils::DEVICE_TEMPLATE), - ::testing::ValuesIn(configs)), - InferRequestPreprocessConversionTest::getTestCaseName); - -} // namespace diff --git a/src/tests/functional/plugin/shared/CMakeLists.txt b/src/tests/functional/plugin/shared/CMakeLists.txt index 3715abb9bb0..d557f1a1a48 100644 --- a/src/tests/functional/plugin/shared/CMakeLists.txt +++ b/src/tests/functional/plugin/shared/CMakeLists.txt @@ -71,10 +71,6 @@ ov_add_target( ${DEPENDENCIES} ) -if(ENABLE_GAPI_PREPROCESSING) - target_compile_definitions(${TARGET_NAME} PUBLIC ENABLE_GAPI_PREPROCESSING) -endif() - ov_build_target_faster(${TARGET_NAME} PCH PRIVATE "src/precomp.hpp" ) diff --git a/thirdparty/ade b/thirdparty/ade deleted file mode 160000 index 0e8a2ccdd34..00000000000 --- a/thirdparty/ade +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 0e8a2ccdd34f29dba55894f5f3c5179809888b9e diff --git a/thirdparty/dependencies.cmake b/thirdparty/dependencies.cmake index 2f3a5383528..fc6ef40c64c 100644 --- a/thirdparty/dependencies.cmake +++ b/thirdparty/dependencies.cmake @@ -260,46 +260,6 @@ if(NOT TARGET openvino::pugixml) ov_build_pugixml() endif() -# -# Fluid, G-API, OpenCV HAL -# - -if(ENABLE_GAPI_PREPROCESSING) - add_library(ocv_hal INTERFACE) - target_include_directories(ocv_hal INTERFACE "${CMAKE_CURRENT_SOURCE_DIR}/thirdparty/ocv") - - # ade - find_package(ade 0.1.0 QUIET) - if(ade_FOUND) - # conan and vcpkg create 'ade' target - # we just need to remove non-numerical symbols from version, - # because conan defines it as 0.1.2a, which is invalid in cmake - string(REGEX REPLACE "[a-z]" "" ade_VERSION "${ade_VERSION}") - else() - add_subdirectory(thirdparty/ade EXCLUDE_FROM_ALL) - - set_target_properties(ade PROPERTIES FOLDER thirdparty) - ov_developer_package_export_targets(TARGET ade) - - ov_install_static_lib(ade ${OV_CPACK_COMP_CORE}) - endif() - - # fluid - add_subdirectory(thirdparty/fluid/modules/gapi EXCLUDE_FROM_ALL) - - if(CMAKE_COMPILER_IS_GNUCXX AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 11) - target_compile_options(fluid PRIVATE "-Wno-maybe-uninitialized") - endif() - if(UNUSED_BUT_SET_VARIABLE_SUPPORTED) - target_compile_options(fluid PRIVATE "-Wno-unused-but-set-variable") - endif() - - set_target_properties(fluid PROPERTIES FOLDER thirdparty) - ov_developer_package_export_targets(TARGET fluid) - - ov_install_static_lib(fluid ${OV_CPACK_COMP_CORE}) -endif() - # # Gflags # diff --git a/thirdparty/fluid/README.md b/thirdparty/fluid/README.md deleted file mode 100644 index 3b31555e529..00000000000 --- a/thirdparty/fluid/README.md +++ /dev/null @@ -1,45 +0,0 @@ -# OpenCV G-API (Fluid), standalone edition - -This subtree hosts sources of G-API - a new OpenCV module for -efficient image processing. G-API serves as a preprocessing vehicle -for Inference Engine. At the moment, only Fluid (CPU) backend is used. - -The sources are taken from OpenCV's [main repository](https://github.com/opencv). - -There are supplementary scripts which ease and verify the update -process. - -## Usage - -Updating to the latest `master`: - - ./update.sh - -Updating to a particular revision: - - ./update.sh COMMIT_HASH - -During update, this script checks if the source tree was modified -after the latest update. If it was, update fails -- we want to avoid -any diverge in the source so _no changes_ should be committed ever to -this copy of G-API. - -One can check manually if sources were diverged from its last "valid" -copy by running - - ./check.sh - -An error message and non-zero exit code indicate possible inconsitency -with this source copy. - -One updated, all changes will be automatically staged. - -## Files - -In addition to the source tree, the above two scripts maintain two -files: -- `revision.txt` -- the OpenCV's revision used to produce this source - copy. If the code was taken from `master`, a timestamp is stored - otherwise. -- `checksum.txt` -- latest valid copy's check sum. Don't update this - file manually. diff --git a/thirdparty/fluid/check.sh b/thirdparty/fluid/check.sh deleted file mode 100755 index e222af6116a..00000000000 --- a/thirdparty/fluid/check.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env bash - -if [ ! -f modules ] && [ ! -f checksum.txt ]; then - exit 0 -fi - -THIS_HASH=$(./checksum.sh) -OLD_HASH=$(cat checksum.txt) - -if [ $THIS_HASH != $OLD_HASH ]; then - echo "Invalid checksum -- any changes were done to the source tree here?" - exit 1 -fi - -echo "Check done." diff --git a/thirdparty/fluid/checksum.sh b/thirdparty/fluid/checksum.sh deleted file mode 100755 index 3c8236177f3..00000000000 --- a/thirdparty/fluid/checksum.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env bash - -find modules/ -type f -exec md5sum {} \; | sort -k 2 | md5sum | cut -d' ' -f 1 diff --git a/thirdparty/fluid/checksum.txt b/thirdparty/fluid/checksum.txt deleted file mode 100644 index d4ac1a3744b..00000000000 --- a/thirdparty/fluid/checksum.txt +++ /dev/null @@ -1 +0,0 @@ -52e7351a888ee42076be4db81d9ac992 diff --git a/thirdparty/fluid/revision.txt b/thirdparty/fluid/revision.txt deleted file mode 100644 index aa3d773c1c9..00000000000 --- a/thirdparty/fluid/revision.txt +++ /dev/null @@ -1 +0,0 @@ -4.x diff --git a/thirdparty/fluid/update.sh b/thirdparty/fluid/update.sh deleted file mode 100755 index 5f4c0536c11..00000000000 --- a/thirdparty/fluid/update.sh +++ /dev/null @@ -1,73 +0,0 @@ -#!/usr/bin/env bash - -REVISION="" -TARGET_DIR=$(pwd) - -case "$#" in - "0") echo "Using latest master..." - REVISION="master" - ;; - "1") REVISION=$1 - echo "Using revision ${REVISION}..." - ;; - *) echo "Usage: ${0} [REVISION] - - Update Fluid to OpenCV source tree at the given REVISION. - If no revision specified, the most recent 'master' commit is used. -" - exit 1 ;; -esac - -# Before doing anything, check if this snapshot was not modified -./check.sh -if [ $? -ne 0 ]; then - echo "Consistency check failed, please reset this subtree to its initial state first!" - exit 1 -fi - -# Download the stuff... -URL="https://github.com/opencv/opencv/archive/${REVISION}.zip" -wget -c ${URL} -if [ $? -ne 0 ]; then - echo "Failed to download ${URL}!" - exit 1 -fi - -unzip -qq ${REVISION}.zip - -# Remove current files -if [ -f modules ]; then - find modules -type f | xargs git rm - find modules -type f | xargs rm - rm -vd modules -fi - -# Put a new copy. Extend this section if needed -# BOM thing might help here, probably -pushd "opencv-${REVISION}" -cp -rv --parent modules/gapi ${TARGET_DIR} -popd -# Note "-f" is used to add files like doc/ which are omitted -# now by IE's current .gitignore - it breaks checksum otherwise. -git add -f modules/gapi - -# Clean-up files -rm -rf "opencv-${REVISION}" -rm "${REVISION}.zip" - -# Calculate and store checksum -./checksum.sh > checksum.txt -git add checksum.txt - -# Store revision -if [ ${REVISION} == "master" ]; then - REVISION="${REVISION} / $(date +%F)" -fi -echo ${REVISION} > revision.txt -git add revision.txt - -# Display status -git status - -# Fin -echo "Done!" diff --git a/thirdparty/ocv/opencv_hal.hpp b/thirdparty/ocv/opencv_hal.hpp deleted file mode 100644 index 0d68078d98d..00000000000 --- a/thirdparty/ocv/opencv_hal.hpp +++ /dev/null @@ -1,256 +0,0 @@ -/*M/////////////////////////////////////////////////////////////////////////////////////// -// -// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. -// -// By downloading, copying, installing or using the software you agree to this license. -// If you do not agree to this license, do not download, install, -// copy or use the software. -// -// -// License Agreement -// For Open Source Computer Vision Library -// -// Copyright (C) 2000-2008, Intel Corporation, all rights reserved. -// Copyright (C) 2009, Willow Garage Inc., all rights reserved. -// Copyright (C) 2013, OpenCV Foundation, all rights reserved. -// Copyright (C) 2015, Itseez Inc., all rights reserved. -// Third party copyrights are property of their respective owners. -// -// Redistribution and use in source and binary forms, with or without modification, -// are permitted provided that the following conditions are met: -// -// * Redistribution's of source code must retain the above copyright notice, -// this list of conditions and the following disclaimer. -// -// * Redistribution's in binary form must reproduce the above copyright notice, -// this list of conditions and the following disclaimer in the documentation -// and/or other materials provided with the distribution. -// -// * The name of the copyright holders may not be used to endorse or promote products -// derived from this software without specific prior written permission. -// -// This software is provided by the copyright holders and contributors "as is" and -// any express or implied warranties, including, but not limited to, the implied -// warranties of merchantability and fitness for a particular purpose are disclaimed. -// In no event shall the Intel Corporation or contributors be liable for any direct, -// indirect, incidental, special, exemplary, or consequential damages -// (including, but not limited to, procurement of substitute goods or services; -// loss of use, data, or profits; or business interruption) however caused -// and on any theory of liability, whether in contract, strict liability, -// or tort (including negligence or otherwise) arising in any way out of -// the use of this software, even if advised of the possibility of such damage. -// -//M*/ - -#ifndef OPENCV_HAL_HPP -#define OPENCV_HAL_HPP - -#include "opencv2/core/cvdef.h" -#include "opencv2/core/cvstd.hpp" -#include "opencv2/core/hal/interface.h" - -namespace cv { namespace hal { - -//! @addtogroup core_hal_functions -//! @{ - -CV_EXPORTS int normHamming(const uchar* a, int n); -CV_EXPORTS int normHamming(const uchar* a, const uchar* b, int n); - -CV_EXPORTS int normHamming(const uchar* a, int n, int cellSize); -CV_EXPORTS int normHamming(const uchar* a, const uchar* b, int n, int cellSize); - -CV_EXPORTS int LU32f(float* A, size_t astep, int m, float* b, size_t bstep, int n); -CV_EXPORTS int LU64f(double* A, size_t astep, int m, double* b, size_t bstep, int n); -CV_EXPORTS bool Cholesky32f(float* A, size_t astep, int m, float* b, size_t bstep, int n); -CV_EXPORTS bool Cholesky64f(double* A, size_t astep, int m, double* b, size_t bstep, int n); -CV_EXPORTS void SVD32f(float* At, size_t astep, float* W, float* U, size_t ustep, float* Vt, size_t vstep, int m, int n, int flags); -CV_EXPORTS void SVD64f(double* At, size_t astep, double* W, double* U, size_t ustep, double* Vt, size_t vstep, int m, int n, int flags); -CV_EXPORTS int QR32f(float* A, size_t astep, int m, int n, int k, float* b, size_t bstep, float* hFactors); -CV_EXPORTS int QR64f(double* A, size_t astep, int m, int n, int k, double* b, size_t bstep, double* hFactors); - -CV_EXPORTS void gemm32f(const float* src1, size_t src1_step, const float* src2, size_t src2_step, - float alpha, const float* src3, size_t src3_step, float beta, float* dst, size_t dst_step, - int m_a, int n_a, int n_d, int flags); -CV_EXPORTS void gemm64f(const double* src1, size_t src1_step, const double* src2, size_t src2_step, - double alpha, const double* src3, size_t src3_step, double beta, double* dst, size_t dst_step, - int m_a, int n_a, int n_d, int flags); -CV_EXPORTS void gemm32fc(const float* src1, size_t src1_step, const float* src2, size_t src2_step, - float alpha, const float* src3, size_t src3_step, float beta, float* dst, size_t dst_step, - int m_a, int n_a, int n_d, int flags); -CV_EXPORTS void gemm64fc(const double* src1, size_t src1_step, const double* src2, size_t src2_step, - double alpha, const double* src3, size_t src3_step, double beta, double* dst, size_t dst_step, - int m_a, int n_a, int n_d, int flags); - -CV_EXPORTS int normL1_(const uchar* a, const uchar* b, int n); -CV_EXPORTS float normL1_(const float* a, const float* b, int n); -CV_EXPORTS float normL2Sqr_(const float* a, const float* b, int n); - -CV_EXPORTS void exp32f(const float* src, float* dst, int n); -CV_EXPORTS void exp64f(const double* src, double* dst, int n); -CV_EXPORTS void log32f(const float* src, float* dst, int n); -CV_EXPORTS void log64f(const double* src, double* dst, int n); - -CV_EXPORTS void fastAtan32f(const float* y, const float* x, float* dst, int n, bool angleInDegrees); -CV_EXPORTS void fastAtan64f(const double* y, const double* x, double* dst, int n, bool angleInDegrees); -CV_EXPORTS void magnitude32f(const float* x, const float* y, float* dst, int n); -CV_EXPORTS void magnitude64f(const double* x, const double* y, double* dst, int n); -CV_EXPORTS void sqrt32f(const float* src, float* dst, int len); -CV_EXPORTS void sqrt64f(const double* src, double* dst, int len); -CV_EXPORTS void invSqrt32f(const float* src, float* dst, int len); -CV_EXPORTS void invSqrt64f(const double* src, double* dst, int len); - -CV_EXPORTS void split8u(const uchar* src, uchar** dst, int len, int cn ); -CV_EXPORTS void split16u(const ushort* src, ushort** dst, int len, int cn ); -CV_EXPORTS void split32s(const int* src, int** dst, int len, int cn ); -CV_EXPORTS void split64s(const int64* src, int64** dst, int len, int cn ); - -CV_EXPORTS void merge8u(const uchar** src, uchar* dst, int len, int cn ); -CV_EXPORTS void merge16u(const ushort** src, ushort* dst, int len, int cn ); -CV_EXPORTS void merge32s(const int** src, int* dst, int len, int cn ); -CV_EXPORTS void merge64s(const int64** src, int64* dst, int len, int cn ); - -CV_EXPORTS void add8u( const uchar* src1, size_t step1, const uchar* src2, size_t step2, uchar* dst, size_t step, int width, int height, void* ); -CV_EXPORTS void add8s( const schar* src1, size_t step1, const schar* src2, size_t step2, schar* dst, size_t step, int width, int height, void* ); -CV_EXPORTS void add16u( const ushort* src1, size_t step1, const ushort* src2, size_t step2, ushort* dst, size_t step, int width, int height, void* ); -CV_EXPORTS void add16s( const short* src1, size_t step1, const short* src2, size_t step2, short* dst, size_t step, int width, int height, void* ); -CV_EXPORTS void add32s( const int* src1, size_t step1, const int* src2, size_t step2, int* dst, size_t step, int width, int height, void* ); -CV_EXPORTS void add32f( const float* src1, size_t step1, const float* src2, size_t step2, float* dst, size_t step, int width, int height, void* ); -CV_EXPORTS void add64f( const double* src1, size_t step1, const double* src2, size_t step2, double* dst, size_t step, int width, int height, void* ); - -CV_EXPORTS void sub8u( const uchar* src1, size_t step1, const uchar* src2, size_t step2, uchar* dst, size_t step, int width, int height, void* ); -CV_EXPORTS void sub8s( const schar* src1, size_t step1, const schar* src2, size_t step2, schar* dst, size_t step, int width, int height, void* ); -CV_EXPORTS void sub16u( const ushort* src1, size_t step1, const ushort* src2, size_t step2, ushort* dst, size_t step, int width, int height, void* ); -CV_EXPORTS void sub16s( const short* src1, size_t step1, const short* src2, size_t step2, short* dst, size_t step, int width, int height, void* ); -CV_EXPORTS void sub32s( const int* src1, size_t step1, const int* src2, size_t step2, int* dst, size_t step, int width, int height, void* ); -CV_EXPORTS void sub32f( const float* src1, size_t step1, const float* src2, size_t step2, float* dst, size_t step, int width, int height, void* ); -CV_EXPORTS void sub64f( const double* src1, size_t step1, const double* src2, size_t step2, double* dst, size_t step, int width, int height, void* ); - -CV_EXPORTS void max8u( const uchar* src1, size_t step1, const uchar* src2, size_t step2, uchar* dst, size_t step, int width, int height, void* ); -CV_EXPORTS void max8s( const schar* src1, size_t step1, const schar* src2, size_t step2, schar* dst, size_t step, int width, int height, void* ); -CV_EXPORTS void max16u( const ushort* src1, size_t step1, const ushort* src2, size_t step2, ushort* dst, size_t step, int width, int height, void* ); -CV_EXPORTS void max16s( const short* src1, size_t step1, const short* src2, size_t step2, short* dst, size_t step, int width, int height, void* ); -CV_EXPORTS void max32s( const int* src1, size_t step1, const int* src2, size_t step2, int* dst, size_t step, int width, int height, void* ); -CV_EXPORTS void max32f( const float* src1, size_t step1, const float* src2, size_t step2, float* dst, size_t step, int width, int height, void* ); -CV_EXPORTS void max64f( const double* src1, size_t step1, const double* src2, size_t step2, double* dst, size_t step, int width, int height, void* ); - -CV_EXPORTS void min8u( const uchar* src1, size_t step1, const uchar* src2, size_t step2, uchar* dst, size_t step, int width, int height, void* ); -CV_EXPORTS void min8s( const schar* src1, size_t step1, const schar* src2, size_t step2, schar* dst, size_t step, int width, int height, void* ); -CV_EXPORTS void min16u( const ushort* src1, size_t step1, const ushort* src2, size_t step2, ushort* dst, size_t step, int width, int height, void* ); -CV_EXPORTS void min16s( const short* src1, size_t step1, const short* src2, size_t step2, short* dst, size_t step, int width, int height, void* ); -CV_EXPORTS void min32s( const int* src1, size_t step1, const int* src2, size_t step2, int* dst, size_t step, int width, int height, void* ); -CV_EXPORTS void min32f( const float* src1, size_t step1, const float* src2, size_t step2, float* dst, size_t step, int width, int height, void* ); -CV_EXPORTS void min64f( const double* src1, size_t step1, const double* src2, size_t step2, double* dst, size_t step, int width, int height, void* ); - -CV_EXPORTS void absdiff8u( const uchar* src1, size_t step1, const uchar* src2, size_t step2, uchar* dst, size_t step, int width, int height, void* ); -CV_EXPORTS void absdiff8s( const schar* src1, size_t step1, const schar* src2, size_t step2, schar* dst, size_t step, int width, int height, void* ); -CV_EXPORTS void absdiff16u( const ushort* src1, size_t step1, const ushort* src2, size_t step2, ushort* dst, size_t step, int width, int height, void* ); -CV_EXPORTS void absdiff16s( const short* src1, size_t step1, const short* src2, size_t step2, short* dst, size_t step, int width, int height, void* ); -CV_EXPORTS void absdiff32s( const int* src1, size_t step1, const int* src2, size_t step2, int* dst, size_t step, int width, int height, void* ); -CV_EXPORTS void absdiff32f( const float* src1, size_t step1, const float* src2, size_t step2, float* dst, size_t step, int width, int height, void* ); -CV_EXPORTS void absdiff64f( const double* src1, size_t step1, const double* src2, size_t step2, double* dst, size_t step, int width, int height, void* ); - -CV_EXPORTS void and8u( const uchar* src1, size_t step1, const uchar* src2, size_t step2, uchar* dst, size_t step, int width, int height, void* ); -CV_EXPORTS void or8u( const uchar* src1, size_t step1, const uchar* src2, size_t step2, uchar* dst, size_t step, int width, int height, void* ); -CV_EXPORTS void xor8u( const uchar* src1, size_t step1, const uchar* src2, size_t step2, uchar* dst, size_t step, int width, int height, void* ); -CV_EXPORTS void not8u( const uchar* src1, size_t step1, const uchar* src2, size_t step2, uchar* dst, size_t step, int width, int height, void* ); - -CV_EXPORTS void cmp8u(const uchar* src1, size_t step1, const uchar* src2, size_t step2, uchar* dst, size_t step, int width, int height, void* _cmpop); -CV_EXPORTS void cmp8s(const schar* src1, size_t step1, const schar* src2, size_t step2, uchar* dst, size_t step, int width, int height, void* _cmpop); -CV_EXPORTS void cmp16u(const ushort* src1, size_t step1, const ushort* src2, size_t step2, uchar* dst, size_t step, int width, int height, void* _cmpop); -CV_EXPORTS void cmp16s(const short* src1, size_t step1, const short* src2, size_t step2, uchar* dst, size_t step, int width, int height, void* _cmpop); -CV_EXPORTS void cmp32s(const int* src1, size_t step1, const int* src2, size_t step2, uchar* dst, size_t step, int width, int height, void* _cmpop); -CV_EXPORTS void cmp32f(const float* src1, size_t step1, const float* src2, size_t step2, uchar* dst, size_t step, int width, int height, void* _cmpop); -CV_EXPORTS void cmp64f(const double* src1, size_t step1, const double* src2, size_t step2, uchar* dst, size_t step, int width, int height, void* _cmpop); - -CV_EXPORTS void mul8u( const uchar* src1, size_t step1, const uchar* src2, size_t step2, uchar* dst, size_t step, int width, int height, void* scale); -CV_EXPORTS void mul8s( const schar* src1, size_t step1, const schar* src2, size_t step2, schar* dst, size_t step, int width, int height, void* scale); -CV_EXPORTS void mul16u( const ushort* src1, size_t step1, const ushort* src2, size_t step2, ushort* dst, size_t step, int width, int height, void* scale); -CV_EXPORTS void mul16s( const short* src1, size_t step1, const short* src2, size_t step2, short* dst, size_t step, int width, int height, void* scale); -CV_EXPORTS void mul32s( const int* src1, size_t step1, const int* src2, size_t step2, int* dst, size_t step, int width, int height, void* scale); -CV_EXPORTS void mul32f( const float* src1, size_t step1, const float* src2, size_t step2, float* dst, size_t step, int width, int height, void* scale); -CV_EXPORTS void mul64f( const double* src1, size_t step1, const double* src2, size_t step2, double* dst, size_t step, int width, int height, void* scale); - -CV_EXPORTS void div8u( const uchar* src1, size_t step1, const uchar* src2, size_t step2, uchar* dst, size_t step, int width, int height, void* scale); -CV_EXPORTS void div8s( const schar* src1, size_t step1, const schar* src2, size_t step2, schar* dst, size_t step, int width, int height, void* scale); -CV_EXPORTS void div16u( const ushort* src1, size_t step1, const ushort* src2, size_t step2, ushort* dst, size_t step, int width, int height, void* scale); -CV_EXPORTS void div16s( const short* src1, size_t step1, const short* src2, size_t step2, short* dst, size_t step, int width, int height, void* scale); -CV_EXPORTS void div32s( const int* src1, size_t step1, const int* src2, size_t step2, int* dst, size_t step, int width, int height, void* scale); -CV_EXPORTS void div32f( const float* src1, size_t step1, const float* src2, size_t step2, float* dst, size_t step, int width, int height, void* scale); -CV_EXPORTS void div64f( const double* src1, size_t step1, const double* src2, size_t step2, double* dst, size_t step, int width, int height, void* scale); - -CV_EXPORTS void recip8u( const uchar *, size_t, const uchar * src2, size_t step2, uchar* dst, size_t step, int width, int height, void* scale); -CV_EXPORTS void recip8s( const schar *, size_t, const schar * src2, size_t step2, schar* dst, size_t step, int width, int height, void* scale); -CV_EXPORTS void recip16u( const ushort *, size_t, const ushort * src2, size_t step2, ushort* dst, size_t step, int width, int height, void* scale); -CV_EXPORTS void recip16s( const short *, size_t, const short * src2, size_t step2, short* dst, size_t step, int width, int height, void* scale); -CV_EXPORTS void recip32s( const int *, size_t, const int * src2, size_t step2, int* dst, size_t step, int width, int height, void* scale); -CV_EXPORTS void recip32f( const float *, size_t, const float * src2, size_t step2, float* dst, size_t step, int width, int height, void* scale); -CV_EXPORTS void recip64f( const double *, size_t, const double * src2, size_t step2, double* dst, size_t step, int width, int height, void* scale); - -CV_EXPORTS void addWeighted8u( const uchar* src1, size_t step1, const uchar* src2, size_t step2, uchar* dst, size_t step, int width, int height, void* _scalars ); -CV_EXPORTS void addWeighted8s( const schar* src1, size_t step1, const schar* src2, size_t step2, schar* dst, size_t step, int width, int height, void* scalars ); -CV_EXPORTS void addWeighted16u( const ushort* src1, size_t step1, const ushort* src2, size_t step2, ushort* dst, size_t step, int width, int height, void* scalars ); -CV_EXPORTS void addWeighted16s( const short* src1, size_t step1, const short* src2, size_t step2, short* dst, size_t step, int width, int height, void* scalars ); -CV_EXPORTS void addWeighted32s( const int* src1, size_t step1, const int* src2, size_t step2, int* dst, size_t step, int width, int height, void* scalars ); -CV_EXPORTS void addWeighted32f( const float* src1, size_t step1, const float* src2, size_t step2, float* dst, size_t step, int width, int height, void* scalars ); -CV_EXPORTS void addWeighted64f( const double* src1, size_t step1, const double* src2, size_t step2, double* dst, size_t step, int width, int height, void* scalars ); - -CV_EXPORTS void cvt16f32f( const float16_t* src, float* dst, int len ); -CV_EXPORTS void cvt32f16f( const float* src, float16_t* dst, int len ); - -CV_EXPORTS void addRNGBias32f( float* arr, const float* scaleBiasPairs, int len ); -CV_EXPORTS void addRNGBias64f( double* arr, const double* scaleBiasPairs, int len ); - -struct CV_EXPORTS DFT1D -{ - static Ptr create(int len, int count, int depth, int flags, bool * useBuffer = 0); - virtual void apply(const uchar *src, uchar *dst) = 0; - virtual ~DFT1D() {} -}; - -struct CV_EXPORTS DFT2D -{ - static Ptr create(int width, int height, int depth, - int src_channels, int dst_channels, - int flags, int nonzero_rows = 0); - virtual void apply(const uchar *src_data, size_t src_step, uchar *dst_data, size_t dst_step) = 0; - virtual ~DFT2D() {} -}; - -struct CV_EXPORTS DCT2D -{ - static Ptr create(int width, int height, int depth, int flags); - virtual void apply(const uchar *src_data, size_t src_step, uchar *dst_data, size_t dst_step) = 0; - virtual ~DCT2D() {} -}; - -//! @} core_hal - -//============================================================================= -// for binary compatibility with 3.0 - -//! @cond IGNORED - -CV_EXPORTS int LU(float* A, size_t astep, int m, float* b, size_t bstep, int n); -CV_EXPORTS int LU(double* A, size_t astep, int m, double* b, size_t bstep, int n); -CV_EXPORTS bool Cholesky(float* A, size_t astep, int m, float* b, size_t bstep, int n); -CV_EXPORTS bool Cholesky(double* A, size_t astep, int m, double* b, size_t bstep, int n); - -CV_EXPORTS void exp(const float* src, float* dst, int n); -CV_EXPORTS void exp(const double* src, double* dst, int n); -CV_EXPORTS void log(const float* src, float* dst, int n); -CV_EXPORTS void log(const double* src, double* dst, int n); - -CV_EXPORTS void fastAtan2(const float* y, const float* x, float* dst, int n, bool angleInDegrees); -CV_EXPORTS void magnitude(const float* x, const float* y, float* dst, int n); -CV_EXPORTS void magnitude(const double* x, const double* y, double* dst, int n); -CV_EXPORTS void sqrt(const float* src, float* dst, int len); -CV_EXPORTS void sqrt(const double* src, double* dst, int len); -CV_EXPORTS void invSqrt(const float* src, float* dst, int len); -CV_EXPORTS void invSqrt(const double* src, double* dst, int len); - -//! @endcond - -}} //cv::hal - -#endif //OPENCV_HAL_HPP diff --git a/thirdparty/ocv/opencv_hal_avx.hpp b/thirdparty/ocv/opencv_hal_avx.hpp deleted file mode 100644 index d13d0db59af..00000000000 --- a/thirdparty/ocv/opencv_hal_avx.hpp +++ /dev/null @@ -1,3274 +0,0 @@ -// This file is part of OpenCV project. -// It is subject to the license terms in the LICENSE file found in the top-level directory -// of this distribution and at http://opencv.org/license.html - -#ifndef OPENCV_HAL_INTRIN_AVX_HPP -#define OPENCV_HAL_INTRIN_AVX_HPP - -#include - -namespace cv -{ - -//! @cond IGNORED - - -CV_CPU_OPTIMIZATION_HAL_NAMESPACE_BEGIN - -///////// Utils //////////// - -inline __m256i _v256_combine(const __m128i& lo, const __m128i& hi) -{ return _mm256_inserti128_si256(_mm256_castsi128_si256(lo), hi, 1); } - -inline __m256 _v256_combine(const __m128& lo, const __m128& hi) -{ return _mm256_insertf128_ps(_mm256_castps128_ps256(lo), hi, 1); } - -inline __m256d _v256_combine(const __m128d& lo, const __m128d& hi) -{ return _mm256_insertf128_pd(_mm256_castpd128_pd256(lo), hi, 1); } - -inline int _v_cvtsi256_si32(const __m256i& a) -{ return _mm_cvtsi128_si32(_mm256_castsi256_si128(a)); } - -inline __m256i _v256_shuffle_odd_64(const __m256i& v) -{ return _mm256_permute4x64_epi64(v, _MM_SHUFFLE(3, 1, 2, 0)); } - -inline __m256d _v256_shuffle_odd_64(const __m256d& v) -{ return _mm256_permute4x64_pd(v, _MM_SHUFFLE(3, 1, 2, 0)); } - -template -static inline __m256i _v256_permute2x128(const __m256i& a, const __m256i& b) -{ return _mm256_permute2x128_si256(a, b, imm); } - -template -inline __m256 _v256_permute2x128(const __m256& a, const __m256& b) -{ return _mm256_permute2f128_ps(a, b, imm); } - -template -inline __m256d _v256_permute2x128(const __m256d& a, const __m256d& b) -{ return _mm256_permute2f128_pd(a, b, imm); } - -template -static inline _Tpvec v256_permute2x128(const _Tpvec& a, const _Tpvec& b) -{ return _Tpvec(_v256_permute2x128(a.val, b.val)); } - -template -static inline __m256i _v256_permute4x64(const __m256i& a) -{ return _mm256_permute4x64_epi64(a, imm); } - -template -inline __m256d _v256_permute4x64(const __m256d& a) -{ return _mm256_permute4x64_pd(a, imm); } - -template -static inline _Tpvec v256_permute4x64(const _Tpvec& a) -{ return _Tpvec(_v256_permute4x64(a.val)); } - -inline __m128i _v256_extract_high(const __m256i& v) -{ return _mm256_extracti128_si256(v, 1); } - -inline __m128 _v256_extract_high(const __m256& v) -{ return _mm256_extractf128_ps(v, 1); } - -inline __m128d _v256_extract_high(const __m256d& v) -{ return _mm256_extractf128_pd(v, 1); } - -inline __m128i _v256_extract_low(const __m256i& v) -{ return _mm256_castsi256_si128(v); } - -inline __m128 _v256_extract_low(const __m256& v) -{ return _mm256_castps256_ps128(v); } - -inline __m128d _v256_extract_low(const __m256d& v) -{ return _mm256_castpd256_pd128(v); } - -inline __m256i _v256_packs_epu32(const __m256i& a, const __m256i& b) -{ - const __m256i m = _mm256_set1_epi32(65535); - __m256i am = _mm256_min_epu32(a, m); - __m256i bm = _mm256_min_epu32(b, m); - return _mm256_packus_epi32(am, bm); -} - -///////// Types //////////// - -struct v_uint8x32 -{ - typedef uchar lane_type; - enum { nlanes = 32 }; - __m256i val; - - explicit v_uint8x32(__m256i v) : val(v) {} - v_uint8x32(uchar v0, uchar v1, uchar v2, uchar v3, - uchar v4, uchar v5, uchar v6, uchar v7, - uchar v8, uchar v9, uchar v10, uchar v11, - uchar v12, uchar v13, uchar v14, uchar v15, - uchar v16, uchar v17, uchar v18, uchar v19, - uchar v20, uchar v21, uchar v22, uchar v23, - uchar v24, uchar v25, uchar v26, uchar v27, - uchar v28, uchar v29, uchar v30, uchar v31) - { - val = _mm256_setr_epi8((char)v0, (char)v1, (char)v2, (char)v3, - (char)v4, (char)v5, (char)v6 , (char)v7, (char)v8, (char)v9, - (char)v10, (char)v11, (char)v12, (char)v13, (char)v14, (char)v15, - (char)v16, (char)v17, (char)v18, (char)v19, (char)v20, (char)v21, - (char)v22, (char)v23, (char)v24, (char)v25, (char)v26, (char)v27, - (char)v28, (char)v29, (char)v30, (char)v31); - } - v_uint8x32() : val(_mm256_setzero_si256()) {} - uchar get0() const { return (uchar)_v_cvtsi256_si32(val); } -}; - -struct v_int8x32 -{ - typedef schar lane_type; - enum { nlanes = 32 }; - __m256i val; - - explicit v_int8x32(__m256i v) : val(v) {} - v_int8x32(schar v0, schar v1, schar v2, schar v3, - schar v4, schar v5, schar v6, schar v7, - schar v8, schar v9, schar v10, schar v11, - schar v12, schar v13, schar v14, schar v15, - schar v16, schar v17, schar v18, schar v19, - schar v20, schar v21, schar v22, schar v23, - schar v24, schar v25, schar v26, schar v27, - schar v28, schar v29, schar v30, schar v31) - { - val = _mm256_setr_epi8(v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, - v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, - v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31); - } - v_int8x32() : val(_mm256_setzero_si256()) {} - schar get0() const { return (schar)_v_cvtsi256_si32(val); } -}; - -struct v_uint16x16 -{ - typedef ushort lane_type; - enum { nlanes = 16 }; - __m256i val; - - explicit v_uint16x16(__m256i v) : val(v) {} - v_uint16x16(ushort v0, ushort v1, ushort v2, ushort v3, - ushort v4, ushort v5, ushort v6, ushort v7, - ushort v8, ushort v9, ushort v10, ushort v11, - ushort v12, ushort v13, ushort v14, ushort v15) - { - val = _mm256_setr_epi16((short)v0, (short)v1, (short)v2, (short)v3, - (short)v4, (short)v5, (short)v6, (short)v7, (short)v8, (short)v9, - (short)v10, (short)v11, (short)v12, (short)v13, (short)v14, (short)v15); - } - v_uint16x16() : val(_mm256_setzero_si256()) {} - ushort get0() const { return (ushort)_v_cvtsi256_si32(val); } -}; - -struct v_int16x16 -{ - typedef short lane_type; - enum { nlanes = 16 }; - __m256i val; - - explicit v_int16x16(__m256i v) : val(v) {} - v_int16x16(short v0, short v1, short v2, short v3, - short v4, short v5, short v6, short v7, - short v8, short v9, short v10, short v11, - short v12, short v13, short v14, short v15) - { - val = _mm256_setr_epi16(v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10, v11, v12, v13, v14, v15); - } - v_int16x16() : val(_mm256_setzero_si256()) {} - short get0() const { return (short)_v_cvtsi256_si32(val); } -}; - -struct v_uint32x8 -{ - typedef unsigned lane_type; - enum { nlanes = 8 }; - __m256i val; - - explicit v_uint32x8(__m256i v) : val(v) {} - v_uint32x8(unsigned v0, unsigned v1, unsigned v2, unsigned v3, - unsigned v4, unsigned v5, unsigned v6, unsigned v7) - { - val = _mm256_setr_epi32((unsigned)v0, (unsigned)v1, (unsigned)v2, - (unsigned)v3, (unsigned)v4, (unsigned)v5, (unsigned)v6, (unsigned)v7); - } - v_uint32x8() : val(_mm256_setzero_si256()) {} - unsigned get0() const { return (unsigned)_v_cvtsi256_si32(val); } -}; - -struct v_int32x8 -{ - typedef int lane_type; - enum { nlanes = 8 }; - __m256i val; - - explicit v_int32x8(__m256i v) : val(v) {} - v_int32x8(int v0, int v1, int v2, int v3, - int v4, int v5, int v6, int v7) - { - val = _mm256_setr_epi32(v0, v1, v2, v3, v4, v5, v6, v7); - } - v_int32x8() : val(_mm256_setzero_si256()) {} - int get0() const { return _v_cvtsi256_si32(val); } -}; - -struct v_float32x8 -{ - typedef float lane_type; - enum { nlanes = 8 }; - __m256 val; - - explicit v_float32x8(__m256 v) : val(v) {} - v_float32x8(float v0, float v1, float v2, float v3, - float v4, float v5, float v6, float v7) - { - val = _mm256_setr_ps(v0, v1, v2, v3, v4, v5, v6, v7); - } - v_float32x8() : val(_mm256_setzero_ps()) {} - float get0() const { return _mm_cvtss_f32(_mm256_castps256_ps128(val)); } -}; - -struct v_uint64x4 -{ - typedef uint64 lane_type; - enum { nlanes = 4 }; - __m256i val; - - explicit v_uint64x4(__m256i v) : val(v) {} - v_uint64x4(uint64 v0, uint64 v1, uint64 v2, uint64 v3) - { val = _mm256_setr_epi64x((int64)v0, (int64)v1, (int64)v2, (int64)v3); } - v_uint64x4() : val(_mm256_setzero_si256()) {} - uint64 get0() const - { - #if defined __x86_64__ || defined _M_X64 - return (uint64)_mm_cvtsi128_si64(_mm256_castsi256_si128(val)); - #else - int a = _mm_cvtsi128_si32(_mm256_castsi256_si128(val)); - int b = _mm_cvtsi128_si32(_mm256_castsi256_si128(_mm256_srli_epi64(val, 32))); - return (unsigned)a | ((uint64)(unsigned)b << 32); - #endif - } -}; - -struct v_int64x4 -{ - typedef int64 lane_type; - enum { nlanes = 4 }; - __m256i val; - - explicit v_int64x4(__m256i v) : val(v) {} - v_int64x4(int64 v0, int64 v1, int64 v2, int64 v3) - { val = _mm256_setr_epi64x(v0, v1, v2, v3); } - v_int64x4() : val(_mm256_setzero_si256()) {} - - int64 get0() const - { - #if defined __x86_64__ || defined _M_X64 - return (int64)_mm_cvtsi128_si64(_mm256_castsi256_si128(val)); - #else - int a = _mm_cvtsi128_si32(_mm256_castsi256_si128(val)); - int b = _mm_cvtsi128_si32(_mm256_castsi256_si128(_mm256_srli_epi64(val, 32))); - return (int64)((unsigned)a | ((uint64)(unsigned)b << 32)); - #endif - } -}; - -struct v_float64x4 -{ - typedef double lane_type; - enum { nlanes = 4 }; - __m256d val; - - explicit v_float64x4(__m256d v) : val(v) {} - v_float64x4(double v0, double v1, double v2, double v3) - { val = _mm256_setr_pd(v0, v1, v2, v3); } - v_float64x4() : val(_mm256_setzero_pd()) {} - double get0() const { return _mm_cvtsd_f64(_mm256_castpd256_pd128(val)); } -}; - -//////////////// Load and store operations /////////////// - -#define OPENCV_HAL_IMPL_AVX_LOADSTORE(_Tpvec, _Tp) \ - inline _Tpvec v256_load(const _Tp* ptr) \ - { return _Tpvec(_mm256_loadu_si256((const __m256i*)ptr)); } \ - inline _Tpvec v256_load_aligned(const _Tp* ptr) \ - { return _Tpvec(_mm256_load_si256((const __m256i*)ptr)); } \ - inline _Tpvec v256_load_low(const _Tp* ptr) \ - { \ - __m128i v128 = _mm_loadu_si128((const __m128i*)ptr); \ - return _Tpvec(_mm256_castsi128_si256(v128)); \ - } \ - inline _Tpvec v256_load_halves(const _Tp* ptr0, const _Tp* ptr1) \ - { \ - __m128i vlo = _mm_loadu_si128((const __m128i*)ptr0); \ - __m128i vhi = _mm_loadu_si128((const __m128i*)ptr1); \ - return _Tpvec(_v256_combine(vlo, vhi)); \ - } \ - inline void v_store(_Tp* ptr, const _Tpvec& a) \ - { _mm256_storeu_si256((__m256i*)ptr, a.val); } \ - inline void v_store_aligned(_Tp* ptr, const _Tpvec& a) \ - { _mm256_store_si256((__m256i*)ptr, a.val); } \ - inline void v_store_aligned_nocache(_Tp* ptr, const _Tpvec& a) \ - { _mm256_stream_si256((__m256i*)ptr, a.val); } \ - inline void v_store(_Tp* ptr, const _Tpvec& a, hal::StoreMode mode) \ - { \ - if( mode == hal::STORE_UNALIGNED ) \ - _mm256_storeu_si256((__m256i*)ptr, a.val); \ - else if( mode == hal::STORE_ALIGNED_NOCACHE ) \ - _mm256_stream_si256((__m256i*)ptr, a.val); \ - else \ - _mm256_store_si256((__m256i*)ptr, a.val); \ - } \ - inline void v_store_low(_Tp* ptr, const _Tpvec& a) \ - { _mm_storeu_si128((__m128i*)ptr, _v256_extract_low(a.val)); } \ - inline void v_store_high(_Tp* ptr, const _Tpvec& a) \ - { _mm_storeu_si128((__m128i*)ptr, _v256_extract_high(a.val)); } - -OPENCV_HAL_IMPL_AVX_LOADSTORE(v_uint8x32, uchar) -OPENCV_HAL_IMPL_AVX_LOADSTORE(v_int8x32, schar) -OPENCV_HAL_IMPL_AVX_LOADSTORE(v_uint16x16, ushort) -OPENCV_HAL_IMPL_AVX_LOADSTORE(v_int16x16, short) -OPENCV_HAL_IMPL_AVX_LOADSTORE(v_uint32x8, unsigned) -OPENCV_HAL_IMPL_AVX_LOADSTORE(v_int32x8, int) -OPENCV_HAL_IMPL_AVX_LOADSTORE(v_uint64x4, uint64) -OPENCV_HAL_IMPL_AVX_LOADSTORE(v_int64x4, int64) - -#define OPENCV_HAL_IMPL_AVX_LOADSTORE_FLT(_Tpvec, _Tp, suffix, halfreg) \ - inline _Tpvec v256_load(const _Tp* ptr) \ - { return _Tpvec(_mm256_loadu_##suffix(ptr)); } \ - inline _Tpvec v256_load_aligned(const _Tp* ptr) \ - { return _Tpvec(_mm256_load_##suffix(ptr)); } \ - inline _Tpvec v256_load_low(const _Tp* ptr) \ - { \ - return _Tpvec(_mm256_cast##suffix##128_##suffix##256 \ - (_mm_loadu_##suffix(ptr))); \ - } \ - inline _Tpvec v256_load_halves(const _Tp* ptr0, const _Tp* ptr1) \ - { \ - halfreg vlo = _mm_loadu_##suffix(ptr0); \ - halfreg vhi = _mm_loadu_##suffix(ptr1); \ - return _Tpvec(_v256_combine(vlo, vhi)); \ - } \ - inline void v_store(_Tp* ptr, const _Tpvec& a) \ - { _mm256_storeu_##suffix(ptr, a.val); } \ - inline void v_store_aligned(_Tp* ptr, const _Tpvec& a) \ - { _mm256_store_##suffix(ptr, a.val); } \ - inline void v_store_aligned_nocache(_Tp* ptr, const _Tpvec& a) \ - { _mm256_stream_##suffix(ptr, a.val); } \ - inline void v_store(_Tp* ptr, const _Tpvec& a, hal::StoreMode mode) \ - { \ - if( mode == hal::STORE_UNALIGNED ) \ - _mm256_storeu_##suffix(ptr, a.val); \ - else if( mode == hal::STORE_ALIGNED_NOCACHE ) \ - _mm256_stream_##suffix(ptr, a.val); \ - else \ - _mm256_store_##suffix(ptr, a.val); \ - } \ - inline void v_store_low(_Tp* ptr, const _Tpvec& a) \ - { _mm_storeu_##suffix(ptr, _v256_extract_low(a.val)); } \ - inline void v_store_high(_Tp* ptr, const _Tpvec& a) \ - { _mm_storeu_##suffix(ptr, _v256_extract_high(a.val)); } - -OPENCV_HAL_IMPL_AVX_LOADSTORE_FLT(v_float32x8, float, ps, __m128) -OPENCV_HAL_IMPL_AVX_LOADSTORE_FLT(v_float64x4, double, pd, __m128d) - -#define OPENCV_HAL_IMPL_AVX_CAST(_Tpvec, _Tpvecf, suffix, cast) \ - inline _Tpvec v_reinterpret_as_##suffix(const _Tpvecf& a) \ - { return _Tpvec(cast(a.val)); } - -#define OPENCV_HAL_IMPL_AVX_INIT(_Tpvec, _Tp, suffix, ssuffix, ctype_s) \ - inline _Tpvec v256_setzero_##suffix() \ - { return _Tpvec(_mm256_setzero_si256()); } \ - inline _Tpvec v256_setall_##suffix(_Tp v) \ - { return _Tpvec(_mm256_set1_##ssuffix((ctype_s)v)); } \ - OPENCV_HAL_IMPL_AVX_CAST(_Tpvec, v_uint8x32, suffix, OPENCV_HAL_NOP) \ - OPENCV_HAL_IMPL_AVX_CAST(_Tpvec, v_int8x32, suffix, OPENCV_HAL_NOP) \ - OPENCV_HAL_IMPL_AVX_CAST(_Tpvec, v_uint16x16, suffix, OPENCV_HAL_NOP) \ - OPENCV_HAL_IMPL_AVX_CAST(_Tpvec, v_int16x16, suffix, OPENCV_HAL_NOP) \ - OPENCV_HAL_IMPL_AVX_CAST(_Tpvec, v_uint32x8, suffix, OPENCV_HAL_NOP) \ - OPENCV_HAL_IMPL_AVX_CAST(_Tpvec, v_int32x8, suffix, OPENCV_HAL_NOP) \ - OPENCV_HAL_IMPL_AVX_CAST(_Tpvec, v_uint64x4, suffix, OPENCV_HAL_NOP) \ - OPENCV_HAL_IMPL_AVX_CAST(_Tpvec, v_int64x4, suffix, OPENCV_HAL_NOP) \ - OPENCV_HAL_IMPL_AVX_CAST(_Tpvec, v_float32x8, suffix, _mm256_castps_si256) \ - OPENCV_HAL_IMPL_AVX_CAST(_Tpvec, v_float64x4, suffix, _mm256_castpd_si256) - -OPENCV_HAL_IMPL_AVX_INIT(v_uint8x32, uchar, u8, epi8, char) -OPENCV_HAL_IMPL_AVX_INIT(v_int8x32, schar, s8, epi8, char) -OPENCV_HAL_IMPL_AVX_INIT(v_uint16x16, ushort, u16, epi16, short) -OPENCV_HAL_IMPL_AVX_INIT(v_int16x16, short, s16, epi16, short) -OPENCV_HAL_IMPL_AVX_INIT(v_uint32x8, unsigned, u32, epi32, int) -OPENCV_HAL_IMPL_AVX_INIT(v_int32x8, int, s32, epi32, int) -OPENCV_HAL_IMPL_AVX_INIT(v_uint64x4, uint64, u64, epi64x, int64) -OPENCV_HAL_IMPL_AVX_INIT(v_int64x4, int64, s64, epi64x, int64) - -#define OPENCV_HAL_IMPL_AVX_INIT_FLT(_Tpvec, _Tp, suffix, zsuffix, cast) \ - inline _Tpvec v256_setzero_##suffix() \ - { return _Tpvec(_mm256_setzero_##zsuffix()); } \ - inline _Tpvec v256_setall_##suffix(_Tp v) \ - { return _Tpvec(_mm256_set1_##zsuffix(v)); } \ - OPENCV_HAL_IMPL_AVX_CAST(_Tpvec, v_uint8x32, suffix, cast) \ - OPENCV_HAL_IMPL_AVX_CAST(_Tpvec, v_int8x32, suffix, cast) \ - OPENCV_HAL_IMPL_AVX_CAST(_Tpvec, v_uint16x16, suffix, cast) \ - OPENCV_HAL_IMPL_AVX_CAST(_Tpvec, v_int16x16, suffix, cast) \ - OPENCV_HAL_IMPL_AVX_CAST(_Tpvec, v_uint32x8, suffix, cast) \ - OPENCV_HAL_IMPL_AVX_CAST(_Tpvec, v_int32x8, suffix, cast) \ - OPENCV_HAL_IMPL_AVX_CAST(_Tpvec, v_uint64x4, suffix, cast) \ - OPENCV_HAL_IMPL_AVX_CAST(_Tpvec, v_int64x4, suffix, cast) - -OPENCV_HAL_IMPL_AVX_INIT_FLT(v_float32x8, float, f32, ps, _mm256_castsi256_ps) -OPENCV_HAL_IMPL_AVX_INIT_FLT(v_float64x4, double, f64, pd, _mm256_castsi256_pd) - -inline v_float32x8 v_reinterpret_as_f32(const v_float32x8& a) -{ return a; } -inline v_float32x8 v_reinterpret_as_f32(const v_float64x4& a) -{ return v_float32x8(_mm256_castpd_ps(a.val)); } - -inline v_float64x4 v_reinterpret_as_f64(const v_float64x4& a) -{ return a; } -inline v_float64x4 v_reinterpret_as_f64(const v_float32x8& a) -{ return v_float64x4(_mm256_castps_pd(a.val)); } - -/* Recombine */ -/*#define OPENCV_HAL_IMPL_AVX_COMBINE(_Tpvec, perm) \ - inline _Tpvec v_combine_low(const _Tpvec& a, const _Tpvec& b) \ - { return _Tpvec(perm(a.val, b.val, 0x20)); } \ - inline _Tpvec v_combine_high(const _Tpvec& a, const _Tpvec& b) \ - { return _Tpvec(perm(a.val, b.val, 0x31)); } \ - inline void v_recombine(const _Tpvec& a, const _Tpvec& b, \ - _Tpvec& c, _Tpvec& d) \ - { c = v_combine_low(a, b); d = v_combine_high(a, b); } - -#define OPENCV_HAL_IMPL_AVX_UNPACKS(_Tpvec, suffix) \ - OPENCV_HAL_IMPL_AVX_COMBINE(_Tpvec, _mm256_permute2x128_si256) \ - inline void v_zip(const _Tpvec& a0, const _Tpvec& a1, \ - _Tpvec& b0, _Tpvec& b1) \ - { \ - __m256i v0 = _v256_shuffle_odd_64(a0.val); \ - __m256i v1 = _v256_shuffle_odd_64(a1.val); \ - b0.val = _mm256_unpacklo_##suffix(v0, v1); \ - b1.val = _mm256_unpackhi_##suffix(v0, v1); \ - } - -OPENCV_HAL_IMPL_AVX_UNPACKS(v_uint8x32, epi8) -OPENCV_HAL_IMPL_AVX_UNPACKS(v_int8x32, epi8) -OPENCV_HAL_IMPL_AVX_UNPACKS(v_uint16x16, epi16) -OPENCV_HAL_IMPL_AVX_UNPACKS(v_int16x16, epi16) -OPENCV_HAL_IMPL_AVX_UNPACKS(v_uint32x8, epi32) -OPENCV_HAL_IMPL_AVX_UNPACKS(v_int32x8, epi32) -OPENCV_HAL_IMPL_AVX_UNPACKS(v_uint64x4, epi64) -OPENCV_HAL_IMPL_AVX_UNPACKS(v_int64x4, epi64) -OPENCV_HAL_IMPL_AVX_COMBINE(v_float32x8, _mm256_permute2f128_ps) -OPENCV_HAL_IMPL_AVX_COMBINE(v_float64x4, _mm256_permute2f128_pd) - -inline void v_zip(const v_float32x8& a0, const v_float32x8& a1, v_float32x8& b0, v_float32x8& b1) -{ - __m256 v0 = _mm256_unpacklo_ps(a0.val, a1.val); - __m256 v1 = _mm256_unpackhi_ps(a0.val, a1.val); - v_recombine(v_float32x8(v0), v_float32x8(v1), b0, b1); -} - -inline void v_zip(const v_float64x4& a0, const v_float64x4& a1, v_float64x4& b0, v_float64x4& b1) -{ - __m256d v0 = _v_shuffle_odd_64(a0.val); - __m256d v1 = _v_shuffle_odd_64(a1.val); - b0.val = _mm256_unpacklo_pd(v0, v1); - b1.val = _mm256_unpackhi_pd(v0, v1); -}*/ - -//////////////// Variant Value reordering /////////////// - -// unpacks -#define OPENCV_HAL_IMPL_AVX_UNPACK(_Tpvec, suffix) \ - inline _Tpvec v256_unpacklo(const _Tpvec& a, const _Tpvec& b) \ - { return _Tpvec(_mm256_unpacklo_##suffix(a.val, b.val)); } \ - inline _Tpvec v256_unpackhi(const _Tpvec& a, const _Tpvec& b) \ - { return _Tpvec(_mm256_unpackhi_##suffix(a.val, b.val)); } - -OPENCV_HAL_IMPL_AVX_UNPACK(v_uint8x32, epi8) -OPENCV_HAL_IMPL_AVX_UNPACK(v_int8x32, epi8) -OPENCV_HAL_IMPL_AVX_UNPACK(v_uint16x16, epi16) -OPENCV_HAL_IMPL_AVX_UNPACK(v_int16x16, epi16) -OPENCV_HAL_IMPL_AVX_UNPACK(v_uint32x8, epi32) -OPENCV_HAL_IMPL_AVX_UNPACK(v_int32x8, epi32) -OPENCV_HAL_IMPL_AVX_UNPACK(v_uint64x4, epi64) -OPENCV_HAL_IMPL_AVX_UNPACK(v_int64x4, epi64) -OPENCV_HAL_IMPL_AVX_UNPACK(v_float32x8, ps) -OPENCV_HAL_IMPL_AVX_UNPACK(v_float64x4, pd) - -// blend -#define OPENCV_HAL_IMPL_AVX_BLEND(_Tpvec, suffix) \ - template \ - inline _Tpvec v256_blend(const _Tpvec& a, const _Tpvec& b) \ - { return _Tpvec(_mm256_blend_##suffix(a.val, b.val, m)); } - -OPENCV_HAL_IMPL_AVX_BLEND(v_uint16x16, epi16) -OPENCV_HAL_IMPL_AVX_BLEND(v_int16x16, epi16) -OPENCV_HAL_IMPL_AVX_BLEND(v_uint32x8, epi32) -OPENCV_HAL_IMPL_AVX_BLEND(v_int32x8, epi32) -OPENCV_HAL_IMPL_AVX_BLEND(v_float32x8, ps) -OPENCV_HAL_IMPL_AVX_BLEND(v_float64x4, pd) - -template -inline v_uint64x4 v256_blend(const v_uint64x4& a, const v_uint64x4& b) -{ - enum {M0 = m}; - enum {M1 = (M0 | (M0 << 2)) & 0x33}; - enum {M2 = (M1 | (M1 << 1)) & 0x55}; - enum {MM = M2 | (M2 << 1)}; - return v_uint64x4(_mm256_blend_epi32(a.val, b.val, MM)); -} -template -inline v_int64x4 v256_blend(const v_int64x4& a, const v_int64x4& b) -{ return v_int64x4(v256_blend(v_uint64x4(a.val), v_uint64x4(b.val)).val); } - -// shuffle -// todo: emluate 64bit -#define OPENCV_HAL_IMPL_AVX_SHUFFLE(_Tpvec, intrin) \ - template \ - inline _Tpvec v256_shuffle(const _Tpvec& a) \ - { return _Tpvec(_mm256_##intrin(a.val, m)); } - -//OPENCV_HAL_IMPL_AVX_SHUFFLE(v_uint32x8, shuffle_epi8) -OPENCV_HAL_IMPL_AVX_SHUFFLE(v_uint32x8, shuffle_epi32) -OPENCV_HAL_IMPL_AVX_SHUFFLE(v_int32x8, shuffle_epi32) -OPENCV_HAL_IMPL_AVX_SHUFFLE(v_float32x8, permute_ps) -OPENCV_HAL_IMPL_AVX_SHUFFLE(v_float64x4, permute_pd) - -template -inline void v256_zip(const _Tpvec& a, const _Tpvec& b, _Tpvec& ab0, _Tpvec& ab1) -{ - ab0 = v256_unpacklo(a, b); - ab1 = v256_unpackhi(a, b); -} - -template -inline _Tpvec v256_combine_diagonal(const _Tpvec& a, const _Tpvec& b) -{ return _Tpvec(_mm256_blend_epi32(a.val, b.val, 0xf0)); } - -inline v_float32x8 v256_combine_diagonal(const v_float32x8& a, const v_float32x8& b) -{ return v256_blend<0xf0>(a, b); } - -inline v_float64x4 v256_combine_diagonal(const v_float64x4& a, const v_float64x4& b) -{ return v256_blend<0xc>(a, b); } - -template -inline _Tpvec v256_alignr_128(const _Tpvec& a, const _Tpvec& b) -{ return v256_permute2x128<0x21>(a, b); } - -template -inline _Tpvec v256_alignr_64(const _Tpvec& a, const _Tpvec& b) -{ return _Tpvec(_mm256_alignr_epi8(a.val, b.val, 8)); } -inline v_float64x4 v256_alignr_64(const v_float64x4& a, const v_float64x4& b) -{ return v_float64x4(_mm256_shuffle_pd(b.val, a.val, _MM_SHUFFLE(0, 0, 1, 1))); } -// todo: emulate float32 - -template -inline _Tpvec v256_swap_halves(const _Tpvec& a) -{ return v256_permute2x128<1>(a, a); } - -template -inline _Tpvec v256_reverse_64(const _Tpvec& a) -{ return v256_permute4x64<_MM_SHUFFLE(0, 1, 2, 3)>(a); } - -// ZIP -#define OPENCV_HAL_IMPL_AVX_ZIP(_Tpvec) \ - inline _Tpvec v_combine_low(const _Tpvec& a, const _Tpvec& b) \ - { return v256_permute2x128<0x20>(a, b); } \ - inline _Tpvec v_combine_high(const _Tpvec& a, const _Tpvec& b) \ - { return v256_permute2x128<0x31>(a, b); } \ - inline void v_recombine(const _Tpvec& a, const _Tpvec& b, \ - _Tpvec& c, _Tpvec& d) \ - { \ - _Tpvec a1b0 = v256_alignr_128(a, b); \ - c = v256_combine_diagonal(a, a1b0); \ - d = v256_combine_diagonal(a1b0, b); \ - } \ - inline void v_zip(const _Tpvec& a, const _Tpvec& b, \ - _Tpvec& ab0, _Tpvec& ab1) \ - { \ - _Tpvec ab0ab2, ab1ab3; \ - v256_zip(a, b, ab0ab2, ab1ab3); \ - v_recombine(ab0ab2, ab1ab3, ab0, ab1); \ - } - -OPENCV_HAL_IMPL_AVX_ZIP(v_uint8x32) -OPENCV_HAL_IMPL_AVX_ZIP(v_int8x32) -OPENCV_HAL_IMPL_AVX_ZIP(v_uint16x16) -OPENCV_HAL_IMPL_AVX_ZIP(v_int16x16) -OPENCV_HAL_IMPL_AVX_ZIP(v_uint32x8) -OPENCV_HAL_IMPL_AVX_ZIP(v_int32x8) -OPENCV_HAL_IMPL_AVX_ZIP(v_uint64x4) -OPENCV_HAL_IMPL_AVX_ZIP(v_int64x4) -OPENCV_HAL_IMPL_AVX_ZIP(v_float32x8) -OPENCV_HAL_IMPL_AVX_ZIP(v_float64x4) - -////////// Arithmetic, bitwise and comparison operations ///////// - -/* Element-wise binary and unary operations */ - -/** Arithmetics **/ -#define OPENCV_HAL_IMPL_AVX_BIN_OP(bin_op, _Tpvec, intrin) \ - inline _Tpvec operator bin_op (const _Tpvec& a, const _Tpvec& b) \ - { return _Tpvec(intrin(a.val, b.val)); } \ - inline _Tpvec& operator bin_op##= (_Tpvec& a, const _Tpvec& b) \ - { a.val = intrin(a.val, b.val); return a; } - -OPENCV_HAL_IMPL_AVX_BIN_OP(+, v_uint8x32, _mm256_adds_epu8) -OPENCV_HAL_IMPL_AVX_BIN_OP(-, v_uint8x32, _mm256_subs_epu8) -OPENCV_HAL_IMPL_AVX_BIN_OP(+, v_int8x32, _mm256_adds_epi8) -OPENCV_HAL_IMPL_AVX_BIN_OP(-, v_int8x32, _mm256_subs_epi8) -OPENCV_HAL_IMPL_AVX_BIN_OP(+, v_uint16x16, _mm256_adds_epu16) -OPENCV_HAL_IMPL_AVX_BIN_OP(-, v_uint16x16, _mm256_subs_epu16) -OPENCV_HAL_IMPL_AVX_BIN_OP(+, v_int16x16, _mm256_adds_epi16) -OPENCV_HAL_IMPL_AVX_BIN_OP(-, v_int16x16, _mm256_subs_epi16) -OPENCV_HAL_IMPL_AVX_BIN_OP(+, v_uint32x8, _mm256_add_epi32) -OPENCV_HAL_IMPL_AVX_BIN_OP(-, v_uint32x8, _mm256_sub_epi32) -OPENCV_HAL_IMPL_AVX_BIN_OP(*, v_uint32x8, _mm256_mullo_epi32) -OPENCV_HAL_IMPL_AVX_BIN_OP(+, v_int32x8, _mm256_add_epi32) -OPENCV_HAL_IMPL_AVX_BIN_OP(-, v_int32x8, _mm256_sub_epi32) -OPENCV_HAL_IMPL_AVX_BIN_OP(*, v_int32x8, _mm256_mullo_epi32) -OPENCV_HAL_IMPL_AVX_BIN_OP(+, v_uint64x4, _mm256_add_epi64) -OPENCV_HAL_IMPL_AVX_BIN_OP(-, v_uint64x4, _mm256_sub_epi64) -OPENCV_HAL_IMPL_AVX_BIN_OP(+, v_int64x4, _mm256_add_epi64) -OPENCV_HAL_IMPL_AVX_BIN_OP(-, v_int64x4, _mm256_sub_epi64) - -OPENCV_HAL_IMPL_AVX_BIN_OP(+, v_float32x8, _mm256_add_ps) -OPENCV_HAL_IMPL_AVX_BIN_OP(-, v_float32x8, _mm256_sub_ps) -OPENCV_HAL_IMPL_AVX_BIN_OP(*, v_float32x8, _mm256_mul_ps) -OPENCV_HAL_IMPL_AVX_BIN_OP(/, v_float32x8, _mm256_div_ps) -OPENCV_HAL_IMPL_AVX_BIN_OP(+, v_float64x4, _mm256_add_pd) -OPENCV_HAL_IMPL_AVX_BIN_OP(-, v_float64x4, _mm256_sub_pd) -OPENCV_HAL_IMPL_AVX_BIN_OP(*, v_float64x4, _mm256_mul_pd) -OPENCV_HAL_IMPL_AVX_BIN_OP(/, v_float64x4, _mm256_div_pd) - -inline void v_mul_expand(const v_uint8x32& a, const v_uint8x32& b, - v_uint16x16& c, v_uint16x16& d); -inline void v_mul_expand(const v_int8x32& a, const v_int8x32& b, - v_int16x16& c, v_int16x16& d); -inline v_uint8x32 v_pack(const v_uint16x16& a, const v_uint16x16& b); -inline v_int8x32 v_pack(const v_int16x16& a, const v_int16x16& b); - -// saturating multiply 8-bit, 16-bit -inline v_uint8x32 operator * (const v_uint8x32& a, const v_uint8x32& b) -{ - v_uint16x16 c, d; - v_mul_expand(a, b, c, d); - return v_pack(c, d); -} -inline v_int8x32 operator * (const v_int8x32& a, const v_int8x32& b) -{ - v_int16x16 c, d; - v_mul_expand(a, b, c, d); - return v_pack(c, d); -} -inline v_uint16x16 operator * (const v_uint16x16& a, const v_uint16x16& b) -{ - __m256i pl = _mm256_mullo_epi16(a.val, b.val); - __m256i ph = _mm256_mulhi_epu16(a.val, b.val); - __m256i p0 = _mm256_unpacklo_epi16(pl, ph); - __m256i p1 = _mm256_unpackhi_epi16(pl, ph); - return v_uint16x16(_v256_packs_epu32(p0, p1)); -} -inline v_int16x16 operator * (const v_int16x16& a, const v_int16x16& b) -{ - __m256i pl = _mm256_mullo_epi16(a.val, b.val); - __m256i ph = _mm256_mulhi_epi16(a.val, b.val); - __m256i p0 = _mm256_unpacklo_epi16(pl, ph); - __m256i p1 = _mm256_unpackhi_epi16(pl, ph); - return v_int16x16(_mm256_packs_epi32(p0, p1)); -} -inline v_uint8x32& operator *= (v_uint8x32& a, const v_uint8x32& b) -{ a = a * b; return a; } -inline v_int8x32& operator *= (v_int8x32& a, const v_int8x32& b) -{ a = a * b; return a; } -inline v_uint16x16& operator *= (v_uint16x16& a, const v_uint16x16& b) -{ a = a * b; return a; } -inline v_int16x16& operator *= (v_int16x16& a, const v_int16x16& b) -{ a = a * b; return a; } - -/** Non-saturating arithmetics **/ -#define OPENCV_HAL_IMPL_AVX_BIN_FUNC(func, _Tpvec, intrin) \ - inline _Tpvec func(const _Tpvec& a, const _Tpvec& b) \ - { return _Tpvec(intrin(a.val, b.val)); } - -OPENCV_HAL_IMPL_AVX_BIN_FUNC(v_add_wrap, v_uint8x32, _mm256_add_epi8) -OPENCV_HAL_IMPL_AVX_BIN_FUNC(v_add_wrap, v_int8x32, _mm256_add_epi8) -OPENCV_HAL_IMPL_AVX_BIN_FUNC(v_add_wrap, v_uint16x16, _mm256_add_epi16) -OPENCV_HAL_IMPL_AVX_BIN_FUNC(v_add_wrap, v_int16x16, _mm256_add_epi16) -OPENCV_HAL_IMPL_AVX_BIN_FUNC(v_sub_wrap, v_uint8x32, _mm256_sub_epi8) -OPENCV_HAL_IMPL_AVX_BIN_FUNC(v_sub_wrap, v_int8x32, _mm256_sub_epi8) -OPENCV_HAL_IMPL_AVX_BIN_FUNC(v_sub_wrap, v_uint16x16, _mm256_sub_epi16) -OPENCV_HAL_IMPL_AVX_BIN_FUNC(v_sub_wrap, v_int16x16, _mm256_sub_epi16) -OPENCV_HAL_IMPL_AVX_BIN_FUNC(v_mul_wrap, v_uint16x16, _mm256_mullo_epi16) -OPENCV_HAL_IMPL_AVX_BIN_FUNC(v_mul_wrap, v_int16x16, _mm256_mullo_epi16) - -//////////////// Value reordering /////////////// - -/* Expand */ -#define OPENCV_HAL_IMPL_AVX_EXPAND(_Tpvec, _Tpwvec, _Tp, intrin) \ - inline void v_expand(const _Tpvec& a, _Tpwvec& b0, _Tpwvec& b1) \ - { \ - b0.val = intrin(_v256_extract_low(a.val)); \ - b1.val = intrin(_v256_extract_high(a.val)); \ - } \ - inline _Tpwvec v_expand_low(const _Tpvec& a) \ - { return _Tpwvec(intrin(_v256_extract_low(a.val))); } \ - inline _Tpwvec v_expand_high(const _Tpvec& a) \ - { return _Tpwvec(intrin(_v256_extract_high(a.val))); } \ - inline _Tpwvec v256_load_expand(const _Tp* ptr) \ - { \ - __m128i a = _mm_loadu_si128((const __m128i*)ptr); \ - return _Tpwvec(intrin(a)); \ - } - -OPENCV_HAL_IMPL_AVX_EXPAND(v_uint8x32, v_uint16x16, uchar, _mm256_cvtepu8_epi16) -OPENCV_HAL_IMPL_AVX_EXPAND(v_int8x32, v_int16x16, schar, _mm256_cvtepi8_epi16) -OPENCV_HAL_IMPL_AVX_EXPAND(v_uint16x16, v_uint32x8, ushort, _mm256_cvtepu16_epi32) -OPENCV_HAL_IMPL_AVX_EXPAND(v_int16x16, v_int32x8, short, _mm256_cvtepi16_epi32) -OPENCV_HAL_IMPL_AVX_EXPAND(v_uint32x8, v_uint64x4, unsigned, _mm256_cvtepu32_epi64) -OPENCV_HAL_IMPL_AVX_EXPAND(v_int32x8, v_int64x4, int, _mm256_cvtepi32_epi64) - -static inline v_int16x16 v_load_ccache_expand(const uchar* ptr) -{ - return v_int16x16(_mm256_cvtepu8_epi16(_mm_lddqu_si128((const __m128i*)ptr))); -} - -inline v_uint8x32 v_mul_wrap(const v_uint8x32& a, const v_uint8x32& b) -{ - __m256i ad = _mm256_srai_epi16(a.val, 8); - __m256i bd = _mm256_srai_epi16(b.val, 8); - __m256i p0 = _mm256_mullo_epi16(a.val, b.val); // even - __m256i p1 = _mm256_slli_epi16(_mm256_mullo_epi16(ad, bd), 8); // odd - - const __m256i b01 = _mm256_set1_epi32(0xFF00FF00); - return v_uint8x32(_mm256_blendv_epi8(p0, p1, b01)); -} -inline v_int8x32 v_mul_wrap(const v_int8x32& a, const v_int8x32& b) -{ - return v_reinterpret_as_s8(v_mul_wrap(v_reinterpret_as_u8(a), v_reinterpret_as_u8(b))); -} - -// Multiply and expand -inline void v_mul_expand(const v_uint8x32& a, const v_uint8x32& b, - v_uint16x16& c, v_uint16x16& d) -{ - v_uint16x16 a0, a1, b0, b1; - v_expand(a, a0, a1); - v_expand(b, b0, b1); - c = v_mul_wrap(a0, b0); - d = v_mul_wrap(a1, b1); -} - -inline void v_mul_expand(const v_int8x32& a, const v_int8x32& b, - v_int16x16& c, v_int16x16& d) -{ - v_int16x16 a0, a1, b0, b1; - v_expand(a, a0, a1); - v_expand(b, b0, b1); - c = v_mul_wrap(a0, b0); - d = v_mul_wrap(a1, b1); -} - -inline void v_mul_expand(const v_int16x16& a, const v_int16x16& b, - v_int32x8& c, v_int32x8& d) -{ - v_int16x16 vhi = v_int16x16(_mm256_mulhi_epi16(a.val, b.val)); - - v_int16x16 v0, v1; - v_zip(v_mul_wrap(a, b), vhi, v0, v1); - - c = v_reinterpret_as_s32(v0); - d = v_reinterpret_as_s32(v1); -} - -inline void v_mul_expand(const v_uint16x16& a, const v_uint16x16& b, - v_uint32x8& c, v_uint32x8& d) -{ - v_uint16x16 vhi = v_uint16x16(_mm256_mulhi_epu16(a.val, b.val)); - - v_uint16x16 v0, v1; - v_zip(v_mul_wrap(a, b), vhi, v0, v1); - - c = v_reinterpret_as_u32(v0); - d = v_reinterpret_as_u32(v1); -} - -inline void v_mul_expand(const v_uint32x8& a, const v_uint32x8& b, - v_uint64x4& c, v_uint64x4& d) -{ - __m256i v0 = _mm256_mul_epu32(a.val, b.val); - __m256i v1 = _mm256_mul_epu32(_mm256_srli_epi64(a.val, 32), _mm256_srli_epi64(b.val, 32)); - v_zip(v_uint64x4(v0), v_uint64x4(v1), c, d); -} - -inline v_int16x16 v_mul_hi(const v_int16x16& a, const v_int16x16& b) { return v_int16x16(_mm256_mulhi_epi16(a.val, b.val)); } -inline v_uint16x16 v_mul_hi(const v_uint16x16& a, const v_uint16x16& b) { return v_uint16x16(_mm256_mulhi_epu16(a.val, b.val)); } - -/** Bitwise shifts **/ -#define OPENCV_HAL_IMPL_AVX_SHIFT_OP(_Tpuvec, _Tpsvec, suffix, srai) \ - inline _Tpuvec operator << (const _Tpuvec& a, int imm) \ - { return _Tpuvec(_mm256_slli_##suffix(a.val, imm)); } \ - inline _Tpsvec operator << (const _Tpsvec& a, int imm) \ - { return _Tpsvec(_mm256_slli_##suffix(a.val, imm)); } \ - inline _Tpuvec operator >> (const _Tpuvec& a, int imm) \ - { return _Tpuvec(_mm256_srli_##suffix(a.val, imm)); } \ - inline _Tpsvec operator >> (const _Tpsvec& a, int imm) \ - { return _Tpsvec(srai(a.val, imm)); } \ - template \ - inline _Tpuvec v_shl(const _Tpuvec& a) \ - { return _Tpuvec(_mm256_slli_##suffix(a.val, imm)); } \ - template \ - inline _Tpsvec v_shl(const _Tpsvec& a) \ - { return _Tpsvec(_mm256_slli_##suffix(a.val, imm)); } \ - template \ - inline _Tpuvec v_shr(const _Tpuvec& a) \ - { return _Tpuvec(_mm256_srli_##suffix(a.val, imm)); } \ - template \ - inline _Tpsvec v_shr(const _Tpsvec& a) \ - { return _Tpsvec(srai(a.val, imm)); } - -OPENCV_HAL_IMPL_AVX_SHIFT_OP(v_uint16x16, v_int16x16, epi16, _mm256_srai_epi16) -OPENCV_HAL_IMPL_AVX_SHIFT_OP(v_uint32x8, v_int32x8, epi32, _mm256_srai_epi32) - -inline __m256i _mm256_srai_epi64xx(const __m256i a, int imm) -{ - __m256i d = _mm256_set1_epi64x((int64)1 << 63); - __m256i r = _mm256_srli_epi64(_mm256_add_epi64(a, d), imm); - return _mm256_sub_epi64(r, _mm256_srli_epi64(d, imm)); -} -OPENCV_HAL_IMPL_AVX_SHIFT_OP(v_uint64x4, v_int64x4, epi64, _mm256_srai_epi64xx) - - -/** Bitwise logic **/ -#define OPENCV_HAL_IMPL_AVX_LOGIC_OP(_Tpvec, suffix, not_const) \ - OPENCV_HAL_IMPL_AVX_BIN_OP(&, _Tpvec, _mm256_and_##suffix) \ - OPENCV_HAL_IMPL_AVX_BIN_OP(|, _Tpvec, _mm256_or_##suffix) \ - OPENCV_HAL_IMPL_AVX_BIN_OP(^, _Tpvec, _mm256_xor_##suffix) \ - inline _Tpvec operator ~ (const _Tpvec& a) \ - { return _Tpvec(_mm256_xor_##suffix(a.val, not_const)); } - -OPENCV_HAL_IMPL_AVX_LOGIC_OP(v_uint8x32, si256, _mm256_set1_epi32(-1)) -OPENCV_HAL_IMPL_AVX_LOGIC_OP(v_int8x32, si256, _mm256_set1_epi32(-1)) -OPENCV_HAL_IMPL_AVX_LOGIC_OP(v_uint16x16, si256, _mm256_set1_epi32(-1)) -OPENCV_HAL_IMPL_AVX_LOGIC_OP(v_int16x16, si256, _mm256_set1_epi32(-1)) -OPENCV_HAL_IMPL_AVX_LOGIC_OP(v_uint32x8, si256, _mm256_set1_epi32(-1)) -OPENCV_HAL_IMPL_AVX_LOGIC_OP(v_int32x8, si256, _mm256_set1_epi32(-1)) -OPENCV_HAL_IMPL_AVX_LOGIC_OP(v_uint64x4, si256, _mm256_set1_epi64x(-1)) -OPENCV_HAL_IMPL_AVX_LOGIC_OP(v_int64x4, si256, _mm256_set1_epi64x(-1)) -OPENCV_HAL_IMPL_AVX_LOGIC_OP(v_float32x8, ps, _mm256_castsi256_ps(_mm256_set1_epi32(-1))) -OPENCV_HAL_IMPL_AVX_LOGIC_OP(v_float64x4, pd, _mm256_castsi256_pd(_mm256_set1_epi32(-1))) - -/** Select **/ -#define OPENCV_HAL_IMPL_AVX_SELECT(_Tpvec, suffix) \ - inline _Tpvec v_select(const _Tpvec& mask, const _Tpvec& a, const _Tpvec& b) \ - { return _Tpvec(_mm256_blendv_##suffix(b.val, a.val, mask.val)); } - -OPENCV_HAL_IMPL_AVX_SELECT(v_uint8x32, epi8) -OPENCV_HAL_IMPL_AVX_SELECT(v_int8x32, epi8) -OPENCV_HAL_IMPL_AVX_SELECT(v_uint16x16, epi8) -OPENCV_HAL_IMPL_AVX_SELECT(v_int16x16, epi8) -OPENCV_HAL_IMPL_AVX_SELECT(v_uint32x8, epi8) -OPENCV_HAL_IMPL_AVX_SELECT(v_int32x8, epi8) -OPENCV_HAL_IMPL_AVX_SELECT(v_float32x8, ps) -OPENCV_HAL_IMPL_AVX_SELECT(v_float64x4, pd) - -/** Comparison **/ -#define OPENCV_HAL_IMPL_AVX_CMP_OP_OV(_Tpvec) \ - inline _Tpvec operator != (const _Tpvec& a, const _Tpvec& b) \ - { return ~(a == b); } \ - inline _Tpvec operator < (const _Tpvec& a, const _Tpvec& b) \ - { return b > a; } \ - inline _Tpvec operator >= (const _Tpvec& a, const _Tpvec& b) \ - { return ~(a < b); } \ - inline _Tpvec operator <= (const _Tpvec& a, const _Tpvec& b) \ - { return b >= a; } - -#define OPENCV_HAL_IMPL_AVX_CMP_OP_INT(_Tpuvec, _Tpsvec, suffix, sbit) \ - inline _Tpuvec operator == (const _Tpuvec& a, const _Tpuvec& b) \ - { return _Tpuvec(_mm256_cmpeq_##suffix(a.val, b.val)); } \ - inline _Tpuvec operator > (const _Tpuvec& a, const _Tpuvec& b) \ - { \ - __m256i smask = _mm256_set1_##suffix(sbit); \ - return _Tpuvec(_mm256_cmpgt_##suffix( \ - _mm256_xor_si256(a.val, smask), \ - _mm256_xor_si256(b.val, smask))); \ - } \ - inline _Tpsvec operator == (const _Tpsvec& a, const _Tpsvec& b) \ - { return _Tpsvec(_mm256_cmpeq_##suffix(a.val, b.val)); } \ - inline _Tpsvec operator > (const _Tpsvec& a, const _Tpsvec& b) \ - { return _Tpsvec(_mm256_cmpgt_##suffix(a.val, b.val)); } \ - OPENCV_HAL_IMPL_AVX_CMP_OP_OV(_Tpuvec) \ - OPENCV_HAL_IMPL_AVX_CMP_OP_OV(_Tpsvec) - -OPENCV_HAL_IMPL_AVX_CMP_OP_INT(v_uint8x32, v_int8x32, epi8, (char)-128) -OPENCV_HAL_IMPL_AVX_CMP_OP_INT(v_uint16x16, v_int16x16, epi16, (short)-32768) -OPENCV_HAL_IMPL_AVX_CMP_OP_INT(v_uint32x8, v_int32x8, epi32, (int)0x80000000) - -#define OPENCV_HAL_IMPL_AVX_CMP_OP_64BIT(_Tpvec) \ - inline _Tpvec operator == (const _Tpvec& a, const _Tpvec& b) \ - { return _Tpvec(_mm256_cmpeq_epi64(a.val, b.val)); } \ - inline _Tpvec operator != (const _Tpvec& a, const _Tpvec& b) \ - { return ~(a == b); } - -OPENCV_HAL_IMPL_AVX_CMP_OP_64BIT(v_uint64x4) -OPENCV_HAL_IMPL_AVX_CMP_OP_64BIT(v_int64x4) - -#define OPENCV_HAL_IMPL_AVX_CMP_FLT(bin_op, imm8, _Tpvec, suffix) \ - inline _Tpvec operator bin_op (const _Tpvec& a, const _Tpvec& b) \ - { return _Tpvec(_mm256_cmp_##suffix(a.val, b.val, imm8)); } - -#define OPENCV_HAL_IMPL_AVX_CMP_OP_FLT(_Tpvec, suffix) \ - OPENCV_HAL_IMPL_AVX_CMP_FLT(==, _CMP_EQ_OQ, _Tpvec, suffix) \ - OPENCV_HAL_IMPL_AVX_CMP_FLT(!=, _CMP_NEQ_OQ, _Tpvec, suffix) \ - OPENCV_HAL_IMPL_AVX_CMP_FLT(<, _CMP_LT_OQ, _Tpvec, suffix) \ - OPENCV_HAL_IMPL_AVX_CMP_FLT(>, _CMP_GT_OQ, _Tpvec, suffix) \ - OPENCV_HAL_IMPL_AVX_CMP_FLT(<=, _CMP_LE_OQ, _Tpvec, suffix) \ - OPENCV_HAL_IMPL_AVX_CMP_FLT(>=, _CMP_GE_OQ, _Tpvec, suffix) - -OPENCV_HAL_IMPL_AVX_CMP_OP_FLT(v_float32x8, ps) -OPENCV_HAL_IMPL_AVX_CMP_OP_FLT(v_float64x4, pd) - -inline v_float32x8 v_not_nan(const v_float32x8& a) -{ return v_float32x8(_mm256_cmp_ps(a.val, a.val, _CMP_ORD_Q)); } -inline v_float64x4 v_not_nan(const v_float64x4& a) -{ return v_float64x4(_mm256_cmp_pd(a.val, a.val, _CMP_ORD_Q)); } - -/** min/max **/ -OPENCV_HAL_IMPL_AVX_BIN_FUNC(v_min, v_uint8x32, _mm256_min_epu8) -OPENCV_HAL_IMPL_AVX_BIN_FUNC(v_max, v_uint8x32, _mm256_max_epu8) -OPENCV_HAL_IMPL_AVX_BIN_FUNC(v_min, v_int8x32, _mm256_min_epi8) -OPENCV_HAL_IMPL_AVX_BIN_FUNC(v_max, v_int8x32, _mm256_max_epi8) -OPENCV_HAL_IMPL_AVX_BIN_FUNC(v_min, v_uint16x16, _mm256_min_epu16) -OPENCV_HAL_IMPL_AVX_BIN_FUNC(v_max, v_uint16x16, _mm256_max_epu16) -OPENCV_HAL_IMPL_AVX_BIN_FUNC(v_min, v_int16x16, _mm256_min_epi16) -OPENCV_HAL_IMPL_AVX_BIN_FUNC(v_max, v_int16x16, _mm256_max_epi16) -OPENCV_HAL_IMPL_AVX_BIN_FUNC(v_min, v_uint32x8, _mm256_min_epu32) -OPENCV_HAL_IMPL_AVX_BIN_FUNC(v_max, v_uint32x8, _mm256_max_epu32) -OPENCV_HAL_IMPL_AVX_BIN_FUNC(v_min, v_int32x8, _mm256_min_epi32) -OPENCV_HAL_IMPL_AVX_BIN_FUNC(v_max, v_int32x8, _mm256_max_epi32) -OPENCV_HAL_IMPL_AVX_BIN_FUNC(v_min, v_float32x8, _mm256_min_ps) -OPENCV_HAL_IMPL_AVX_BIN_FUNC(v_max, v_float32x8, _mm256_max_ps) -OPENCV_HAL_IMPL_AVX_BIN_FUNC(v_min, v_float64x4, _mm256_min_pd) -OPENCV_HAL_IMPL_AVX_BIN_FUNC(v_max, v_float64x4, _mm256_max_pd) - -/** Rotate **/ -template -inline v_uint8x32 v_rotate_left(const v_uint8x32& a, const v_uint8x32& b) -{ - enum {IMM_R = (16 - imm) & 0xFF}; - enum {IMM_R2 = (32 - imm) & 0xFF}; - - if (imm == 0) return a; - if (imm == 32) return b; - if (imm > 32) return v_uint8x32(); - - __m256i swap = _mm256_permute2x128_si256(a.val, b.val, 0x03); - if (imm == 16) return v_uint8x32(swap); - if (imm < 16) return v_uint8x32(_mm256_alignr_epi8(a.val, swap, IMM_R)); - return v_uint8x32(_mm256_alignr_epi8(swap, b.val, IMM_R2)); // imm < 32 -} - -template -inline v_uint8x32 v_rotate_right(const v_uint8x32& a, const v_uint8x32& b) -{ - enum {IMM_L = (imm - 16) & 0xFF}; - - if (imm == 0) return a; - if (imm == 32) return b; - if (imm > 32) return v_uint8x32(); - - __m256i swap = _mm256_permute2x128_si256(a.val, b.val, 0x21); - if (imm == 16) return v_uint8x32(swap); - if (imm < 16) return v_uint8x32(_mm256_alignr_epi8(swap, a.val, imm)); - return v_uint8x32(_mm256_alignr_epi8(b.val, swap, IMM_L)); -} - -template -inline v_uint8x32 v_rotate_left(const v_uint8x32& a) -{ - enum {IMM_L = (imm - 16) & 0xFF}; - enum {IMM_R = (16 - imm) & 0xFF}; - - if (imm == 0) return a; - if (imm > 32) return v_uint8x32(); - - // ESAC control[3] ? [127:0] = 0 - __m256i swapz = _mm256_permute2x128_si256(a.val, a.val, _MM_SHUFFLE(0, 0, 2, 0)); - if (imm == 16) return v_uint8x32(swapz); - if (imm < 16) return v_uint8x32(_mm256_alignr_epi8(a.val, swapz, IMM_R)); - return v_uint8x32(_mm256_slli_si256(swapz, IMM_L)); -} - -template -inline v_uint8x32 v_rotate_right(const v_uint8x32& a) -{ - enum {IMM_L = (imm - 16) & 0xFF}; - - if (imm == 0) return a; - if (imm > 32) return v_uint8x32(); - - // ESAC control[3] ? [127:0] = 0 - __m256i swapz = _mm256_permute2x128_si256(a.val, a.val, _MM_SHUFFLE(2, 0, 0, 1)); - if (imm == 16) return v_uint8x32(swapz); - if (imm < 16) return v_uint8x32(_mm256_alignr_epi8(swapz, a.val, imm)); - return v_uint8x32(_mm256_srli_si256(swapz, IMM_L)); -} - -#define OPENCV_HAL_IMPL_AVX_ROTATE_CAST(intrin, _Tpvec, cast) \ - template \ - inline _Tpvec intrin(const _Tpvec& a, const _Tpvec& b) \ - { \ - enum {IMMxW = imm * sizeof(typename _Tpvec::lane_type)}; \ - v_uint8x32 ret = intrin(v_reinterpret_as_u8(a), \ - v_reinterpret_as_u8(b)); \ - return _Tpvec(cast(ret.val)); \ - } \ - template \ - inline _Tpvec intrin(const _Tpvec& a) \ - { \ - enum {IMMxW = imm * sizeof(typename _Tpvec::lane_type)}; \ - v_uint8x32 ret = intrin(v_reinterpret_as_u8(a)); \ - return _Tpvec(cast(ret.val)); \ - } - -#define OPENCV_HAL_IMPL_AVX_ROTATE(_Tpvec) \ - OPENCV_HAL_IMPL_AVX_ROTATE_CAST(v_rotate_left, _Tpvec, OPENCV_HAL_NOP) \ - OPENCV_HAL_IMPL_AVX_ROTATE_CAST(v_rotate_right, _Tpvec, OPENCV_HAL_NOP) - -OPENCV_HAL_IMPL_AVX_ROTATE(v_int8x32) -OPENCV_HAL_IMPL_AVX_ROTATE(v_uint16x16) -OPENCV_HAL_IMPL_AVX_ROTATE(v_int16x16) -OPENCV_HAL_IMPL_AVX_ROTATE(v_uint32x8) -OPENCV_HAL_IMPL_AVX_ROTATE(v_int32x8) -OPENCV_HAL_IMPL_AVX_ROTATE(v_uint64x4) -OPENCV_HAL_IMPL_AVX_ROTATE(v_int64x4) - -OPENCV_HAL_IMPL_AVX_ROTATE_CAST(v_rotate_left, v_float32x8, _mm256_castsi256_ps) -OPENCV_HAL_IMPL_AVX_ROTATE_CAST(v_rotate_right, v_float32x8, _mm256_castsi256_ps) -OPENCV_HAL_IMPL_AVX_ROTATE_CAST(v_rotate_left, v_float64x4, _mm256_castsi256_pd) -OPENCV_HAL_IMPL_AVX_ROTATE_CAST(v_rotate_right, v_float64x4, _mm256_castsi256_pd) - -////////// Reduce and mask ///////// - -/** Reduce **/ -inline unsigned v_reduce_sum(const v_uint8x32& a) -{ - __m256i half = _mm256_sad_epu8(a.val, _mm256_setzero_si256()); - __m128i quarter = _mm_add_epi32(_v256_extract_low(half), _v256_extract_high(half)); - return (unsigned)_mm_cvtsi128_si32(_mm_add_epi32(quarter, _mm_unpackhi_epi64(quarter, quarter))); -} -inline int v_reduce_sum(const v_int8x32& a) -{ - __m256i half = _mm256_sad_epu8(_mm256_xor_si256(a.val, _mm256_set1_epi8((schar)-128)), _mm256_setzero_si256()); - __m128i quarter = _mm_add_epi32(_v256_extract_low(half), _v256_extract_high(half)); - return (unsigned)_mm_cvtsi128_si32(_mm_add_epi32(quarter, _mm_unpackhi_epi64(quarter, quarter))) - 4096; -} -#define OPENCV_HAL_IMPL_AVX_REDUCE_32(_Tpvec, sctype, func, intrin) \ - inline sctype v_reduce_##func(const _Tpvec& a) \ - { \ - __m128i val = intrin(_v256_extract_low(a.val), _v256_extract_high(a.val)); \ - val = intrin(val, _mm_srli_si128(val,8)); \ - val = intrin(val, _mm_srli_si128(val,4)); \ - val = intrin(val, _mm_srli_si128(val,2)); \ - val = intrin(val, _mm_srli_si128(val,1)); \ - return (sctype)_mm_cvtsi128_si32(val); \ - } - -OPENCV_HAL_IMPL_AVX_REDUCE_32(v_uint8x32, uchar, min, _mm_min_epu8) -OPENCV_HAL_IMPL_AVX_REDUCE_32(v_int8x32, schar, min, _mm_min_epi8) -OPENCV_HAL_IMPL_AVX_REDUCE_32(v_uint8x32, uchar, max, _mm_max_epu8) -OPENCV_HAL_IMPL_AVX_REDUCE_32(v_int8x32, schar, max, _mm_max_epi8) - -#define OPENCV_HAL_IMPL_AVX_REDUCE_16(_Tpvec, sctype, func, intrin) \ - inline sctype v_reduce_##func(const _Tpvec& a) \ - { \ - __m128i v0 = _v256_extract_low(a.val); \ - __m128i v1 = _v256_extract_high(a.val); \ - v0 = intrin(v0, v1); \ - v0 = intrin(v0, _mm_srli_si128(v0, 8)); \ - v0 = intrin(v0, _mm_srli_si128(v0, 4)); \ - v0 = intrin(v0, _mm_srli_si128(v0, 2)); \ - return (sctype) _mm_cvtsi128_si32(v0); \ - } - -OPENCV_HAL_IMPL_AVX_REDUCE_16(v_uint16x16, ushort, min, _mm_min_epu16) -OPENCV_HAL_IMPL_AVX_REDUCE_16(v_int16x16, short, min, _mm_min_epi16) -OPENCV_HAL_IMPL_AVX_REDUCE_16(v_uint16x16, ushort, max, _mm_max_epu16) -OPENCV_HAL_IMPL_AVX_REDUCE_16(v_int16x16, short, max, _mm_max_epi16) - -#define OPENCV_HAL_IMPL_AVX_REDUCE_8(_Tpvec, sctype, func, intrin) \ - inline sctype v_reduce_##func(const _Tpvec& a) \ - { \ - __m128i v0 = _v256_extract_low(a.val); \ - __m128i v1 = _v256_extract_high(a.val); \ - v0 = intrin(v0, v1); \ - v0 = intrin(v0, _mm_srli_si128(v0, 8)); \ - v0 = intrin(v0, _mm_srli_si128(v0, 4)); \ - return (sctype) _mm_cvtsi128_si32(v0); \ - } - -OPENCV_HAL_IMPL_AVX_REDUCE_8(v_uint32x8, unsigned, min, _mm_min_epu32) -OPENCV_HAL_IMPL_AVX_REDUCE_8(v_int32x8, int, min, _mm_min_epi32) -OPENCV_HAL_IMPL_AVX_REDUCE_8(v_uint32x8, unsigned, max, _mm_max_epu32) -OPENCV_HAL_IMPL_AVX_REDUCE_8(v_int32x8, int, max, _mm_max_epi32) - -#define OPENCV_HAL_IMPL_AVX_REDUCE_FLT(func, intrin) \ - inline float v_reduce_##func(const v_float32x8& a) \ - { \ - __m128 v0 = _v256_extract_low(a.val); \ - __m128 v1 = _v256_extract_high(a.val); \ - v0 = intrin(v0, v1); \ - v0 = intrin(v0, _mm_permute_ps(v0, _MM_SHUFFLE(0, 0, 3, 2))); \ - v0 = intrin(v0, _mm_permute_ps(v0, _MM_SHUFFLE(0, 0, 0, 1))); \ - return _mm_cvtss_f32(v0); \ - } - -OPENCV_HAL_IMPL_AVX_REDUCE_FLT(min, _mm_min_ps) -OPENCV_HAL_IMPL_AVX_REDUCE_FLT(max, _mm_max_ps) - -inline int v_reduce_sum(const v_int32x8& a) -{ - __m256i s0 = _mm256_hadd_epi32(a.val, a.val); - s0 = _mm256_hadd_epi32(s0, s0); - - __m128i s1 = _v256_extract_high(s0); - s1 = _mm_add_epi32(_v256_extract_low(s0), s1); - - return _mm_cvtsi128_si32(s1); -} - -inline unsigned v_reduce_sum(const v_uint32x8& a) -{ return v_reduce_sum(v_reinterpret_as_s32(a)); } - -inline int v_reduce_sum(const v_int16x16& a) -{ return v_reduce_sum(v_expand_low(a) + v_expand_high(a)); } -inline unsigned v_reduce_sum(const v_uint16x16& a) -{ return v_reduce_sum(v_expand_low(a) + v_expand_high(a)); } - -inline float v_reduce_sum(const v_float32x8& a) -{ - __m256 s0 = _mm256_hadd_ps(a.val, a.val); - s0 = _mm256_hadd_ps(s0, s0); - - __m128 s1 = _v256_extract_high(s0); - s1 = _mm_add_ps(_v256_extract_low(s0), s1); - - return _mm_cvtss_f32(s1); -} - -inline uint64 v_reduce_sum(const v_uint64x4& a) -{ - uint64 CV_DECL_ALIGNED(32) idx[2]; - _mm_store_si128((__m128i*)idx, _mm_add_epi64(_v256_extract_low(a.val), _v256_extract_high(a.val))); - return idx[0] + idx[1]; -} -inline int64 v_reduce_sum(const v_int64x4& a) -{ - int64 CV_DECL_ALIGNED(32) idx[2]; - _mm_store_si128((__m128i*)idx, _mm_add_epi64(_v256_extract_low(a.val), _v256_extract_high(a.val))); - return idx[0] + idx[1]; -} -inline double v_reduce_sum(const v_float64x4& a) -{ - __m256d s0 = _mm256_hadd_pd(a.val, a.val); - return _mm_cvtsd_f64(_mm_add_pd(_v256_extract_low(s0), _v256_extract_high(s0))); -} - -inline v_float32x8 v_reduce_sum4(const v_float32x8& a, const v_float32x8& b, - const v_float32x8& c, const v_float32x8& d) -{ - __m256 ab = _mm256_hadd_ps(a.val, b.val); - __m256 cd = _mm256_hadd_ps(c.val, d.val); - return v_float32x8(_mm256_hadd_ps(ab, cd)); -} - -inline unsigned v_reduce_sad(const v_uint8x32& a, const v_uint8x32& b) -{ - __m256i half = _mm256_sad_epu8(a.val, b.val); - __m128i quarter = _mm_add_epi32(_v256_extract_low(half), _v256_extract_high(half)); - return (unsigned)_mm_cvtsi128_si32(_mm_add_epi32(quarter, _mm_unpackhi_epi64(quarter, quarter))); -} -inline unsigned v_reduce_sad(const v_int8x32& a, const v_int8x32& b) -{ - __m256i half = _mm256_set1_epi8(0x7f); - half = _mm256_sad_epu8(_mm256_add_epi8(a.val, half), _mm256_add_epi8(b.val, half)); - __m128i quarter = _mm_add_epi32(_v256_extract_low(half), _v256_extract_high(half)); - return (unsigned)_mm_cvtsi128_si32(_mm_add_epi32(quarter, _mm_unpackhi_epi64(quarter, quarter))); -} -inline unsigned v_reduce_sad(const v_uint16x16& a, const v_uint16x16& b) -{ - v_uint32x8 l, h; - v_expand(v_add_wrap(a - b, b - a), l, h); - return v_reduce_sum(l + h); -} -inline unsigned v_reduce_sad(const v_int16x16& a, const v_int16x16& b) -{ - v_uint32x8 l, h; - v_expand(v_reinterpret_as_u16(v_sub_wrap(v_max(a, b), v_min(a, b))), l, h); - return v_reduce_sum(l + h); -} -inline unsigned v_reduce_sad(const v_uint32x8& a, const v_uint32x8& b) -{ - return v_reduce_sum(v_max(a, b) - v_min(a, b)); -} -inline unsigned v_reduce_sad(const v_int32x8& a, const v_int32x8& b) -{ - v_int32x8 m = a < b; - return v_reduce_sum(v_reinterpret_as_u32(((a - b) ^ m) - m)); -} -inline float v_reduce_sad(const v_float32x8& a, const v_float32x8& b) -{ - return v_reduce_sum((a - b) & v_float32x8(_mm256_castsi256_ps(_mm256_set1_epi32(0x7fffffff)))); -} - -/** Popcount **/ -inline v_uint8x32 v_popcount(const v_uint8x32& a) -{ - __m256i _popcnt_table = _mm256_setr_epi8(0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4, - 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4); - __m256i _popcnt_mask = _mm256_set1_epi8(0x0F); - return v_uint8x32(_mm256_add_epi8(_mm256_shuffle_epi8(_popcnt_table, _mm256_and_si256( a.val , _popcnt_mask)), - _mm256_shuffle_epi8(_popcnt_table, _mm256_and_si256(_mm256_srli_epi16(a.val, 4), _popcnt_mask)))); -} -inline v_uint16x16 v_popcount(const v_uint16x16& a) -{ - v_uint8x32 p = v_popcount(v_reinterpret_as_u8(a)); - p += v_rotate_right<1>(p); - return v_reinterpret_as_u16(p) & v256_setall_u16(0x00ff); -} -inline v_uint32x8 v_popcount(const v_uint32x8& a) -{ - v_uint8x32 p = v_popcount(v_reinterpret_as_u8(a)); - p += v_rotate_right<1>(p); - p += v_rotate_right<2>(p); - return v_reinterpret_as_u32(p) & v256_setall_u32(0x000000ff); -} -inline v_uint64x4 v_popcount(const v_uint64x4& a) -{ - return v_uint64x4(_mm256_sad_epu8(v_popcount(v_reinterpret_as_u8(a)).val, _mm256_setzero_si256())); -} -inline v_uint8x32 v_popcount(const v_int8x32& a) -{ return v_popcount(v_reinterpret_as_u8(a)); } -inline v_uint16x16 v_popcount(const v_int16x16& a) -{ return v_popcount(v_reinterpret_as_u16(a)); } -inline v_uint32x8 v_popcount(const v_int32x8& a) -{ return v_popcount(v_reinterpret_as_u32(a)); } -inline v_uint64x4 v_popcount(const v_int64x4& a) -{ return v_popcount(v_reinterpret_as_u64(a)); } - -/** Mask **/ -inline int v_signmask(const v_int8x32& a) -{ return _mm256_movemask_epi8(a.val); } -inline int v_signmask(const v_uint8x32& a) -{ return v_signmask(v_reinterpret_as_s8(a)); } - -inline int v_signmask(const v_int16x16& a) -{ return v_signmask(v_pack(a, a)) & 0xFFFF; } -inline int v_signmask(const v_uint16x16& a) -{ return v_signmask(v_reinterpret_as_s16(a)); } - -inline int v_signmask(const v_float32x8& a) -{ return _mm256_movemask_ps(a.val); } -inline int v_signmask(const v_float64x4& a) -{ return _mm256_movemask_pd(a.val); } - -inline int v_signmask(const v_int32x8& a) -{ return v_signmask(v_reinterpret_as_f32(a)); } -inline int v_signmask(const v_uint32x8& a) -{ return v_signmask(v_reinterpret_as_f32(a)); } - -inline int v_signmask(const v_int64x4& a) -{ return v_signmask(v_reinterpret_as_f64(a)); } -inline int v_signmask(const v_uint64x4& a) -{ return v_signmask(v_reinterpret_as_f64(a)); } - -inline int v_scan_forward(const v_int8x32& a) { return trailingZeros32(v_signmask(v_reinterpret_as_s8(a))); } -inline int v_scan_forward(const v_uint8x32& a) { return trailingZeros32(v_signmask(v_reinterpret_as_s8(a))); } -inline int v_scan_forward(const v_int16x16& a) { return trailingZeros32(v_signmask(v_reinterpret_as_s8(a))) / 2; } -inline int v_scan_forward(const v_uint16x16& a) { return trailingZeros32(v_signmask(v_reinterpret_as_s8(a))) / 2; } -inline int v_scan_forward(const v_int32x8& a) { return trailingZeros32(v_signmask(v_reinterpret_as_s8(a))) / 4; } -inline int v_scan_forward(const v_uint32x8& a) { return trailingZeros32(v_signmask(v_reinterpret_as_s8(a))) / 4; } -inline int v_scan_forward(const v_float32x8& a) { return trailingZeros32(v_signmask(v_reinterpret_as_s8(a))) / 4; } -inline int v_scan_forward(const v_int64x4& a) { return trailingZeros32(v_signmask(v_reinterpret_as_s8(a))) / 8; } -inline int v_scan_forward(const v_uint64x4& a) { return trailingZeros32(v_signmask(v_reinterpret_as_s8(a))) / 8; } -inline int v_scan_forward(const v_float64x4& a) { return trailingZeros32(v_signmask(v_reinterpret_as_s8(a))) / 8; } - -/** Checks **/ -#define OPENCV_HAL_IMPL_AVX_CHECK(_Tpvec, allmask) \ - inline bool v_check_all(const _Tpvec& a) { return v_signmask(a) == allmask; } \ - inline bool v_check_any(const _Tpvec& a) { return v_signmask(a) != 0; } -OPENCV_HAL_IMPL_AVX_CHECK(v_uint8x32, -1) -OPENCV_HAL_IMPL_AVX_CHECK(v_int8x32, -1) -OPENCV_HAL_IMPL_AVX_CHECK(v_uint32x8, 255) -OPENCV_HAL_IMPL_AVX_CHECK(v_int32x8, 255) -OPENCV_HAL_IMPL_AVX_CHECK(v_uint64x4, 15) -OPENCV_HAL_IMPL_AVX_CHECK(v_int64x4, 15) -OPENCV_HAL_IMPL_AVX_CHECK(v_float32x8, 255) -OPENCV_HAL_IMPL_AVX_CHECK(v_float64x4, 15) - -#define OPENCV_HAL_IMPL_AVX_CHECK_SHORT(_Tpvec) \ - inline bool v_check_all(const _Tpvec& a) { return (v_signmask(v_reinterpret_as_s8(a)) & 0xaaaaaaaa) == 0xaaaaaaaa; } \ - inline bool v_check_any(const _Tpvec& a) { return (v_signmask(v_reinterpret_as_s8(a)) & 0xaaaaaaaa) != 0; } -OPENCV_HAL_IMPL_AVX_CHECK_SHORT(v_uint16x16) -OPENCV_HAL_IMPL_AVX_CHECK_SHORT(v_int16x16) - -////////// Other math ///////// - -/** Some frequent operations **/ -#define OPENCV_HAL_IMPL_AVX_MULADD(_Tpvec, suffix) \ - static inline _Tpvec v_fma(const _Tpvec& a, const _Tpvec& b, const _Tpvec& c) \ - { return _Tpvec(_mm256_fmadd_##suffix(a.val, b.val, c.val));} \ - inline _Tpvec v_muladd(const _Tpvec& a, const _Tpvec& b, const _Tpvec& c) \ - { return _Tpvec(_mm256_fmadd_##suffix(a.val, b.val, c.val)); } \ - inline _Tpvec v_sqrt(const _Tpvec& x) \ - { return _Tpvec(_mm256_sqrt_##suffix(x.val)); } \ - inline _Tpvec v_sqr_magnitude(const _Tpvec& a, const _Tpvec& b) \ - { return v_fma(a, a, b * b); } \ - inline _Tpvec v_magnitude(const _Tpvec& a, const _Tpvec& b) \ - { return v_sqrt(v_fma(a, a, b*b)); } - -OPENCV_HAL_IMPL_AVX_MULADD(v_float32x8, ps) -OPENCV_HAL_IMPL_AVX_MULADD(v_float64x4, pd) - -inline v_int32x8 v_fma(const v_int32x8& a, const v_int32x8& b, const v_int32x8& c) -{ - return a * b + c; -} - -inline v_int32x8 v_muladd(const v_int32x8& a, const v_int32x8& b, const v_int32x8& c) -{ - return v_fma(a, b, c); -} - -inline v_float32x8 v_invsqrt(const v_float32x8& x) -{ - v_float32x8 half = x * v256_setall_f32(0.5); - v_float32x8 t = v_float32x8(_mm256_rsqrt_ps(x.val)); - // todo: _mm256_fnmsub_ps - t *= v256_setall_f32(1.5) - ((t * t) * half); - return t; -} - -inline v_float64x4 v_invsqrt(const v_float64x4& x) -{ - return v256_setall_f64(1.) / v_sqrt(x); -} - -/** Absolute values **/ -#define OPENCV_HAL_IMPL_AVX_ABS(_Tpvec, suffix) \ - inline v_u##_Tpvec v_abs(const v_##_Tpvec& x) \ - { return v_u##_Tpvec(_mm256_abs_##suffix(x.val)); } - -OPENCV_HAL_IMPL_AVX_ABS(int8x32, epi8) -OPENCV_HAL_IMPL_AVX_ABS(int16x16, epi16) -OPENCV_HAL_IMPL_AVX_ABS(int32x8, epi32) - -inline v_float32x8 v_abs(const v_float32x8& x) -{ return x & v_float32x8(_mm256_castsi256_ps(_mm256_set1_epi32(0x7fffffff))); } -inline v_float64x4 v_abs(const v_float64x4& x) -{ return x & v_float64x4(_mm256_castsi256_pd(_mm256_srli_epi64(_mm256_set1_epi64x(-1), 1))); } - -/** Absolute difference **/ -inline v_uint8x32 v_absdiff(const v_uint8x32& a, const v_uint8x32& b) -{ return v_add_wrap(a - b, b - a); } -inline v_uint16x16 v_absdiff(const v_uint16x16& a, const v_uint16x16& b) -{ return v_add_wrap(a - b, b - a); } -inline v_uint32x8 v_absdiff(const v_uint32x8& a, const v_uint32x8& b) -{ return v_max(a, b) - v_min(a, b); } - -inline v_uint8x32 v_absdiff(const v_int8x32& a, const v_int8x32& b) -{ - v_int8x32 d = v_sub_wrap(a, b); - v_int8x32 m = a < b; - return v_reinterpret_as_u8(v_sub_wrap(d ^ m, m)); -} - -inline v_uint16x16 v_absdiff(const v_int16x16& a, const v_int16x16& b) -{ return v_reinterpret_as_u16(v_sub_wrap(v_max(a, b), v_min(a, b))); } - -inline v_uint32x8 v_absdiff(const v_int32x8& a, const v_int32x8& b) -{ - v_int32x8 d = a - b; - v_int32x8 m = a < b; - return v_reinterpret_as_u32((d ^ m) - m); -} - -inline v_float32x8 v_absdiff(const v_float32x8& a, const v_float32x8& b) -{ return v_abs(a - b); } - -inline v_float64x4 v_absdiff(const v_float64x4& a, const v_float64x4& b) -{ return v_abs(a - b); } - -/** Saturating absolute difference **/ -inline v_int8x32 v_absdiffs(const v_int8x32& a, const v_int8x32& b) -{ - v_int8x32 d = a - b; - v_int8x32 m = a < b; - return (d ^ m) - m; -} -inline v_int16x16 v_absdiffs(const v_int16x16& a, const v_int16x16& b) -{ return v_max(a, b) - v_min(a, b); } - -////////// Conversions ///////// - -/** Rounding **/ -inline v_int32x8 v_round(const v_float32x8& a) -{ return v_int32x8(_mm256_cvtps_epi32(a.val)); } - -inline v_int32x8 v_round(const v_float64x4& a) -{ return v_int32x8(_mm256_castsi128_si256(_mm256_cvtpd_epi32(a.val))); } - -inline v_int32x8 v_round(const v_float64x4& a, const v_float64x4& b) -{ - __m128i ai = _mm256_cvtpd_epi32(a.val), bi = _mm256_cvtpd_epi32(b.val); - return v_int32x8(_v256_combine(ai, bi)); -} - -inline v_int32x8 v_trunc(const v_float32x8& a) -{ return v_int32x8(_mm256_cvttps_epi32(a.val)); } - -inline v_int32x8 v_trunc(const v_float64x4& a) -{ return v_int32x8(_mm256_castsi128_si256(_mm256_cvttpd_epi32(a.val))); } - -inline v_int32x8 v_floor(const v_float32x8& a) -{ return v_int32x8(_mm256_cvttps_epi32(_mm256_floor_ps(a.val))); } - -inline v_int32x8 v_floor(const v_float64x4& a) -{ return v_trunc(v_float64x4(_mm256_floor_pd(a.val))); } - -inline v_int32x8 v_ceil(const v_float32x8& a) -{ return v_int32x8(_mm256_cvttps_epi32(_mm256_ceil_ps(a.val))); } - -inline v_int32x8 v_ceil(const v_float64x4& a) -{ return v_trunc(v_float64x4(_mm256_ceil_pd(a.val))); } - -/** To float **/ -inline v_float32x8 v_cvt_f32(const v_int32x8& a) -{ return v_float32x8(_mm256_cvtepi32_ps(a.val)); } - -inline v_float32x8 v_cvt_f32(const v_float64x4& a) -{ return v_float32x8(_mm256_castps128_ps256(_mm256_cvtpd_ps(a.val))); } - -inline v_float32x8 v_cvt_f32(const v_float64x4& a, const v_float64x4& b) -{ - __m128 af = _mm256_cvtpd_ps(a.val), bf = _mm256_cvtpd_ps(b.val); - return v_float32x8(_v256_combine(af, bf)); -} - -inline v_float64x4 v_cvt_f64(const v_int32x8& a) -{ return v_float64x4(_mm256_cvtepi32_pd(_v256_extract_low(a.val))); } - -inline v_float64x4 v_cvt_f64_high(const v_int32x8& a) -{ return v_float64x4(_mm256_cvtepi32_pd(_v256_extract_high(a.val))); } - -inline v_float64x4 v_cvt_f64(const v_float32x8& a) -{ return v_float64x4(_mm256_cvtps_pd(_v256_extract_low(a.val))); } - -inline v_float64x4 v_cvt_f64_high(const v_float32x8& a) -{ return v_float64x4(_mm256_cvtps_pd(_v256_extract_high(a.val))); } - -// from (Mysticial and wim) https://stackoverflow.com/q/41144668 -inline v_float64x4 v_cvt_f64(const v_int64x4& v) -{ - // constants encoded as floating-point - __m256i magic_i_lo = _mm256_set1_epi64x(0x4330000000000000); // 2^52 - __m256i magic_i_hi32 = _mm256_set1_epi64x(0x4530000080000000); // 2^84 + 2^63 - __m256i magic_i_all = _mm256_set1_epi64x(0x4530000080100000); // 2^84 + 2^63 + 2^52 - __m256d magic_d_all = _mm256_castsi256_pd(magic_i_all); - - // Blend the 32 lowest significant bits of v with magic_int_lo - __m256i v_lo = _mm256_blend_epi32(magic_i_lo, v.val, 0x55); - // Extract the 32 most significant bits of v - __m256i v_hi = _mm256_srli_epi64(v.val, 32); - // Flip the msb of v_hi and blend with 0x45300000 - v_hi = _mm256_xor_si256(v_hi, magic_i_hi32); - // Compute in double precision - __m256d v_hi_dbl = _mm256_sub_pd(_mm256_castsi256_pd(v_hi), magic_d_all); - // (v_hi - magic_d_all) + v_lo Do not assume associativity of floating point addition - __m256d result = _mm256_add_pd(v_hi_dbl, _mm256_castsi256_pd(v_lo)); - return v_float64x4(result); -} - -////////////// Lookup table access //////////////////// - -inline v_int8x32 v256_lut(const schar* tab, const int* idx) -{ - return v_int8x32(_mm256_setr_epi8(tab[idx[ 0]], tab[idx[ 1]], tab[idx[ 2]], tab[idx[ 3]], tab[idx[ 4]], tab[idx[ 5]], tab[idx[ 6]], tab[idx[ 7]], - tab[idx[ 8]], tab[idx[ 9]], tab[idx[10]], tab[idx[11]], tab[idx[12]], tab[idx[13]], tab[idx[14]], tab[idx[15]], - tab[idx[16]], tab[idx[17]], tab[idx[18]], tab[idx[19]], tab[idx[20]], tab[idx[21]], tab[idx[22]], tab[idx[23]], - tab[idx[24]], tab[idx[25]], tab[idx[26]], tab[idx[27]], tab[idx[28]], tab[idx[29]], tab[idx[30]], tab[idx[31]])); -} -inline v_int8x32 v256_lut_pairs(const schar* tab, const int* idx) -{ - return v_int8x32(_mm256_setr_epi16(*(const short*)(tab + idx[ 0]), *(const short*)(tab + idx[ 1]), *(const short*)(tab + idx[ 2]), *(const short*)(tab + idx[ 3]), - *(const short*)(tab + idx[ 4]), *(const short*)(tab + idx[ 5]), *(const short*)(tab + idx[ 6]), *(const short*)(tab + idx[ 7]), - *(const short*)(tab + idx[ 8]), *(const short*)(tab + idx[ 9]), *(const short*)(tab + idx[10]), *(const short*)(tab + idx[11]), - *(const short*)(tab + idx[12]), *(const short*)(tab + idx[13]), *(const short*)(tab + idx[14]), *(const short*)(tab + idx[15]))); -} -inline v_int8x32 v256_lut_quads(const schar* tab, const int* idx) -{ - return v_int8x32(_mm256_i32gather_epi32((const int*)tab, _mm256_loadu_si256((const __m256i*)idx), 1)); -} -inline v_uint8x32 v256_lut(const uchar* tab, const int* idx) { return v_reinterpret_as_u8(v256_lut((const schar *)tab, idx)); } -inline v_uint8x32 v256_lut_pairs(const uchar* tab, const int* idx) { return v_reinterpret_as_u8(v256_lut_pairs((const schar *)tab, idx)); } -inline v_uint8x32 v256_lut_quads(const uchar* tab, const int* idx) { return v_reinterpret_as_u8(v256_lut_quads((const schar *)tab, idx)); } - -inline v_int16x16 v256_lut(const short* tab, const int* idx) -{ - return v_int16x16(_mm256_setr_epi16(tab[idx[0]], tab[idx[1]], tab[idx[ 2]], tab[idx[ 3]], tab[idx[ 4]], tab[idx[ 5]], tab[idx[ 6]], tab[idx[ 7]], - tab[idx[8]], tab[idx[9]], tab[idx[10]], tab[idx[11]], tab[idx[12]], tab[idx[13]], tab[idx[14]], tab[idx[15]])); -} -inline v_int16x16 v256_lut_pairs(const short* tab, const int* idx) -{ - return v_int16x16(_mm256_i32gather_epi32((const int*)tab, _mm256_loadu_si256((const __m256i*)idx), 2)); -} -inline v_int16x16 v256_lut_quads(const short* tab, const int* idx) -{ -#if defined(__GNUC__) - return v_int16x16(_mm256_i32gather_epi64((const long long int*)tab, _mm_loadu_si128((const __m128i*)idx), 2));//Looks like intrinsic has wrong definition -#else - return v_int16x16(_mm256_i32gather_epi64((const int64*)tab, _mm_loadu_si128((const __m128i*)idx), 2)); -#endif -} -inline v_uint16x16 v256_lut(const ushort* tab, const int* idx) { return v_reinterpret_as_u16(v256_lut((const short *)tab, idx)); } -inline v_uint16x16 v256_lut_pairs(const ushort* tab, const int* idx) { return v_reinterpret_as_u16(v256_lut_pairs((const short *)tab, idx)); } -inline v_uint16x16 v256_lut_quads(const ushort* tab, const int* idx) { return v_reinterpret_as_u16(v256_lut_quads((const short *)tab, idx)); } - -inline v_int32x8 v256_lut(const int* tab, const int* idx) -{ - return v_int32x8(_mm256_i32gather_epi32(tab, _mm256_loadu_si256((const __m256i*)idx), 4)); -} -inline v_int32x8 v256_lut_pairs(const int* tab, const int* idx) -{ -#if defined(__GNUC__) - return v_int32x8(_mm256_i32gather_epi64((const long long int*)tab, _mm_loadu_si128((const __m128i*)idx), 4)); -#else - return v_int32x8(_mm256_i32gather_epi64((const int64*)tab, _mm_loadu_si128((const __m128i*)idx), 4)); -#endif -} -inline v_int32x8 v256_lut_quads(const int* tab, const int* idx) -{ - return v_int32x8(_v256_combine(_mm_loadu_si128((const __m128i*)(tab + idx[0])), _mm_loadu_si128((const __m128i*)(tab + idx[1])))); -} -inline v_uint32x8 v256_lut(const unsigned* tab, const int* idx) { return v_reinterpret_as_u32(v256_lut((const int *)tab, idx)); } -inline v_uint32x8 v256_lut_pairs(const unsigned* tab, const int* idx) { return v_reinterpret_as_u32(v256_lut_pairs((const int *)tab, idx)); } -inline v_uint32x8 v256_lut_quads(const unsigned* tab, const int* idx) { return v_reinterpret_as_u32(v256_lut_quads((const int *)tab, idx)); } - -inline v_int64x4 v256_lut(const int64* tab, const int* idx) -{ -#if defined(__GNUC__) - return v_int64x4(_mm256_i32gather_epi64((const long long int*)tab, _mm_loadu_si128((const __m128i*)idx), 8)); -#else - return v_int64x4(_mm256_i32gather_epi64(tab, _mm_loadu_si128((const __m128i*)idx), 8)); -#endif -} -inline v_int64x4 v256_lut_pairs(const int64* tab, const int* idx) -{ - return v_int64x4(_v256_combine(_mm_loadu_si128((const __m128i*)(tab + idx[0])), _mm_loadu_si128((const __m128i*)(tab + idx[1])))); -} -inline v_uint64x4 v256_lut(const uint64* tab, const int* idx) { return v_reinterpret_as_u64(v256_lut((const int64 *)tab, idx)); } -inline v_uint64x4 v256_lut_pairs(const uint64* tab, const int* idx) { return v_reinterpret_as_u64(v256_lut_pairs((const int64 *)tab, idx)); } - -inline v_float32x8 v256_lut(const float* tab, const int* idx) -{ - return v_float32x8(_mm256_i32gather_ps(tab, _mm256_loadu_si256((const __m256i*)idx), 4)); -} -inline v_float32x8 v256_lut_pairs(const float* tab, const int* idx) { return v_reinterpret_as_f32(v256_lut_pairs((const int *)tab, idx)); } -inline v_float32x8 v256_lut_quads(const float* tab, const int* idx) { return v_reinterpret_as_f32(v256_lut_quads((const int *)tab, idx)); } - -inline v_float64x4 v256_lut(const double* tab, const int* idx) -{ - return v_float64x4(_mm256_i32gather_pd(tab, _mm_loadu_si128((const __m128i*)idx), 8)); -} -inline v_float64x4 v256_lut_pairs(const double* tab, const int* idx) { return v_float64x4(_v256_combine(_mm_loadu_pd(tab + idx[0]), _mm_loadu_pd(tab + idx[1]))); } - -inline v_int32x8 v_lut(const int* tab, const v_int32x8& idxvec) -{ - return v_int32x8(_mm256_i32gather_epi32(tab, idxvec.val, 4)); -} - -inline v_uint32x8 v_lut(const unsigned* tab, const v_int32x8& idxvec) -{ - return v_reinterpret_as_u32(v_lut((const int *)tab, idxvec)); -} - -inline v_float32x8 v_lut(const float* tab, const v_int32x8& idxvec) -{ - return v_float32x8(_mm256_i32gather_ps(tab, idxvec.val, 4)); -} - -inline v_float64x4 v_lut(const double* tab, const v_int32x8& idxvec) -{ - return v_float64x4(_mm256_i32gather_pd(tab, _mm256_castsi256_si128(idxvec.val), 8)); -} - -inline void v_lut_deinterleave(const float* tab, const v_int32x8& idxvec, v_float32x8& x, v_float32x8& y) -{ - int CV_DECL_ALIGNED(32) idx[8]; - v_store_aligned(idx, idxvec); - __m128 z = _mm_setzero_ps(); - __m128 xy01, xy45, xy23, xy67; - xy01 = _mm_loadl_pi(z, (const __m64*)(tab + idx[0])); - xy01 = _mm_loadh_pi(xy01, (const __m64*)(tab + idx[1])); - xy45 = _mm_loadl_pi(z, (const __m64*)(tab + idx[4])); - xy45 = _mm_loadh_pi(xy45, (const __m64*)(tab + idx[5])); - __m256 xy0145 = _v256_combine(xy01, xy45); - xy23 = _mm_loadl_pi(z, (const __m64*)(tab + idx[2])); - xy23 = _mm_loadh_pi(xy23, (const __m64*)(tab + idx[3])); - xy67 = _mm_loadl_pi(z, (const __m64*)(tab + idx[6])); - xy67 = _mm_loadh_pi(xy67, (const __m64*)(tab + idx[7])); - __m256 xy2367 = _v256_combine(xy23, xy67); - - __m256 xxyy0145 = _mm256_unpacklo_ps(xy0145, xy2367); - __m256 xxyy2367 = _mm256_unpackhi_ps(xy0145, xy2367); - - x = v_float32x8(_mm256_unpacklo_ps(xxyy0145, xxyy2367)); - y = v_float32x8(_mm256_unpackhi_ps(xxyy0145, xxyy2367)); -} - -inline void v_lut_deinterleave(const double* tab, const v_int32x8& idxvec, v_float64x4& x, v_float64x4& y) -{ - int CV_DECL_ALIGNED(32) idx[4]; - v_store_low(idx, idxvec); - __m128d xy0 = _mm_loadu_pd(tab + idx[0]); - __m128d xy2 = _mm_loadu_pd(tab + idx[2]); - __m128d xy1 = _mm_loadu_pd(tab + idx[1]); - __m128d xy3 = _mm_loadu_pd(tab + idx[3]); - __m256d xy02 = _v256_combine(xy0, xy2); - __m256d xy13 = _v256_combine(xy1, xy3); - - x = v_float64x4(_mm256_unpacklo_pd(xy02, xy13)); - y = v_float64x4(_mm256_unpackhi_pd(xy02, xy13)); -} - -inline v_int8x32 v_interleave_pairs(const v_int8x32& vec) -{ - return v_int8x32(_mm256_shuffle_epi8(vec.val, _mm256_set_epi64x(0x0f0d0e0c0b090a08, 0x0705060403010200, 0x0f0d0e0c0b090a08, 0x0705060403010200))); -} -inline v_uint8x32 v_interleave_pairs(const v_uint8x32& vec) { return v_reinterpret_as_u8(v_interleave_pairs(v_reinterpret_as_s8(vec))); } -inline v_int8x32 v_interleave_quads(const v_int8x32& vec) -{ - return v_int8x32(_mm256_shuffle_epi8(vec.val, _mm256_set_epi64x(0x0f0b0e0a0d090c08, 0x0703060205010400, 0x0f0b0e0a0d090c08, 0x0703060205010400))); -} -inline v_uint8x32 v_interleave_quads(const v_uint8x32& vec) { return v_reinterpret_as_u8(v_interleave_quads(v_reinterpret_as_s8(vec))); } - -inline v_int16x16 v_interleave_pairs(const v_int16x16& vec) -{ - return v_int16x16(_mm256_shuffle_epi8(vec.val, _mm256_set_epi64x(0x0f0e0b0a0d0c0908, 0x0706030205040100, 0x0f0e0b0a0d0c0908, 0x0706030205040100))); -} -inline v_uint16x16 v_interleave_pairs(const v_uint16x16& vec) { return v_reinterpret_as_u16(v_interleave_pairs(v_reinterpret_as_s16(vec))); } -inline v_int16x16 v_interleave_quads(const v_int16x16& vec) -{ - return v_int16x16(_mm256_shuffle_epi8(vec.val, _mm256_set_epi64x(0x0f0e07060d0c0504, 0x0b0a030209080100, 0x0f0e07060d0c0504, 0x0b0a030209080100))); -} -inline v_uint16x16 v_interleave_quads(const v_uint16x16& vec) { return v_reinterpret_as_u16(v_interleave_quads(v_reinterpret_as_s16(vec))); } - -inline v_int32x8 v_interleave_pairs(const v_int32x8& vec) -{ - return v_int32x8(_mm256_shuffle_epi32(vec.val, _MM_SHUFFLE(3, 1, 2, 0))); -} -inline v_uint32x8 v_interleave_pairs(const v_uint32x8& vec) { return v_reinterpret_as_u32(v_interleave_pairs(v_reinterpret_as_s32(vec))); } -inline v_float32x8 v_interleave_pairs(const v_float32x8& vec) { return v_reinterpret_as_f32(v_interleave_pairs(v_reinterpret_as_s32(vec))); } - -inline v_int8x32 v_pack_triplets(const v_int8x32& vec) -{ - return v_int8x32(_mm256_permutevar8x32_epi32(_mm256_shuffle_epi8(vec.val, _mm256_broadcastsi128_si256(_mm_set_epi64x(0xffffff0f0e0d0c0a, 0x0908060504020100))), - _mm256_set_epi64x(0x0000000700000007, 0x0000000600000005, 0x0000000400000002, 0x0000000100000000))); -} -inline v_uint8x32 v_pack_triplets(const v_uint8x32& vec) { return v_reinterpret_as_u8(v_pack_triplets(v_reinterpret_as_s8(vec))); } - -inline v_int16x16 v_pack_triplets(const v_int16x16& vec) -{ - return v_int16x16(_mm256_permutevar8x32_epi32(_mm256_shuffle_epi8(vec.val, _mm256_broadcastsi128_si256(_mm_set_epi64x(0xffff0f0e0d0c0b0a, 0x0908050403020100))), - _mm256_set_epi64x(0x0000000700000007, 0x0000000600000005, 0x0000000400000002, 0x0000000100000000))); -} -inline v_uint16x16 v_pack_triplets(const v_uint16x16& vec) { return v_reinterpret_as_u16(v_pack_triplets(v_reinterpret_as_s16(vec))); } - -inline v_int32x8 v_pack_triplets(const v_int32x8& vec) -{ - return v_int32x8(_mm256_permutevar8x32_epi32(vec.val, _mm256_set_epi64x(0x0000000700000007, 0x0000000600000005, 0x0000000400000002, 0x0000000100000000))); -} -inline v_uint32x8 v_pack_triplets(const v_uint32x8& vec) { return v_reinterpret_as_u32(v_pack_triplets(v_reinterpret_as_s32(vec))); } -inline v_float32x8 v_pack_triplets(const v_float32x8& vec) -{ - return v_float32x8(_mm256_permutevar8x32_ps(vec.val, _mm256_set_epi64x(0x0000000700000007, 0x0000000600000005, 0x0000000400000002, 0x0000000100000000))); -} - -////////// Matrix operations ///////// - -//////// Dot Product //////// - -// 16 >> 32 -inline v_int32x8 v_dotprod(const v_int16x16& a, const v_int16x16& b) -{ return v_int32x8(_mm256_madd_epi16(a.val, b.val)); } -inline v_int32x8 v_dotprod(const v_int16x16& a, const v_int16x16& b, const v_int32x8& c) -{ return v_dotprod(a, b) + c; } - -// 32 >> 64 -inline v_int64x4 v_dotprod(const v_int32x8& a, const v_int32x8& b) -{ - __m256i even = _mm256_mul_epi32(a.val, b.val); - __m256i odd = _mm256_mul_epi32(_mm256_srli_epi64(a.val, 32), _mm256_srli_epi64(b.val, 32)); - return v_int64x4(_mm256_add_epi64(even, odd)); -} -inline v_int64x4 v_dotprod(const v_int32x8& a, const v_int32x8& b, const v_int64x4& c) -{ return v_dotprod(a, b) + c; } - -// 8 >> 32 -inline v_uint32x8 v_dotprod_expand(const v_uint8x32& a, const v_uint8x32& b) -{ - __m256i even_m = _mm256_set1_epi32(0xFF00FF00); - __m256i even_a = _mm256_blendv_epi8(a.val, _mm256_setzero_si256(), even_m); - __m256i odd_a = _mm256_srli_epi16(a.val, 8); - - __m256i even_b = _mm256_blendv_epi8(b.val, _mm256_setzero_si256(), even_m); - __m256i odd_b = _mm256_srli_epi16(b.val, 8); - - __m256i prod0 = _mm256_madd_epi16(even_a, even_b); - __m256i prod1 = _mm256_madd_epi16(odd_a, odd_b); - return v_uint32x8(_mm256_add_epi32(prod0, prod1)); -} -inline v_uint32x8 v_dotprod_expand(const v_uint8x32& a, const v_uint8x32& b, const v_uint32x8& c) -{ return v_dotprod_expand(a, b) + c; } - -inline v_int32x8 v_dotprod_expand(const v_int8x32& a, const v_int8x32& b) -{ - __m256i even_a = _mm256_srai_epi16(_mm256_bslli_epi128(a.val, 1), 8); - __m256i odd_a = _mm256_srai_epi16(a.val, 8); - - __m256i even_b = _mm256_srai_epi16(_mm256_bslli_epi128(b.val, 1), 8); - __m256i odd_b = _mm256_srai_epi16(b.val, 8); - - __m256i prod0 = _mm256_madd_epi16(even_a, even_b); - __m256i prod1 = _mm256_madd_epi16(odd_a, odd_b); - return v_int32x8(_mm256_add_epi32(prod0, prod1)); -} -inline v_int32x8 v_dotprod_expand(const v_int8x32& a, const v_int8x32& b, const v_int32x8& c) -{ return v_dotprod_expand(a, b) + c; } - -// 16 >> 64 -inline v_uint64x4 v_dotprod_expand(const v_uint16x16& a, const v_uint16x16& b) -{ - __m256i mullo = _mm256_mullo_epi16(a.val, b.val); - __m256i mulhi = _mm256_mulhi_epu16(a.val, b.val); - __m256i mul0 = _mm256_unpacklo_epi16(mullo, mulhi); - __m256i mul1 = _mm256_unpackhi_epi16(mullo, mulhi); - - __m256i p02 = _mm256_blend_epi32(mul0, _mm256_setzero_si256(), 0xAA); - __m256i p13 = _mm256_srli_epi64(mul0, 32); - __m256i p46 = _mm256_blend_epi32(mul1, _mm256_setzero_si256(), 0xAA); - __m256i p57 = _mm256_srli_epi64(mul1, 32); - - __m256i p15_ = _mm256_add_epi64(p02, p13); - __m256i p9d_ = _mm256_add_epi64(p46, p57); - - return v_uint64x4(_mm256_add_epi64( - _mm256_unpacklo_epi64(p15_, p9d_), - _mm256_unpackhi_epi64(p15_, p9d_) - )); -} -inline v_uint64x4 v_dotprod_expand(const v_uint16x16& a, const v_uint16x16& b, const v_uint64x4& c) -{ return v_dotprod_expand(a, b) + c; } - -inline v_int64x4 v_dotprod_expand(const v_int16x16& a, const v_int16x16& b) -{ - __m256i prod = _mm256_madd_epi16(a.val, b.val); - __m256i sign = _mm256_srai_epi32(prod, 31); - - __m256i lo = _mm256_unpacklo_epi32(prod, sign); - __m256i hi = _mm256_unpackhi_epi32(prod, sign); - - return v_int64x4(_mm256_add_epi64( - _mm256_unpacklo_epi64(lo, hi), - _mm256_unpackhi_epi64(lo, hi) - )); -} -inline v_int64x4 v_dotprod_expand(const v_int16x16& a, const v_int16x16& b, const v_int64x4& c) -{ return v_dotprod_expand(a, b) + c; } - -// 32 >> 64f -inline v_float64x4 v_dotprod_expand(const v_int32x8& a, const v_int32x8& b) -{ return v_cvt_f64(v_dotprod(a, b)); } -inline v_float64x4 v_dotprod_expand(const v_int32x8& a, const v_int32x8& b, const v_float64x4& c) -{ return v_dotprod_expand(a, b) + c; } - -//////// Fast Dot Product //////// - -// 16 >> 32 -inline v_int32x8 v_dotprod_fast(const v_int16x16& a, const v_int16x16& b) -{ return v_dotprod(a, b); } -inline v_int32x8 v_dotprod_fast(const v_int16x16& a, const v_int16x16& b, const v_int32x8& c) -{ return v_dotprod(a, b, c); } - -// 32 >> 64 -inline v_int64x4 v_dotprod_fast(const v_int32x8& a, const v_int32x8& b) -{ return v_dotprod(a, b); } -inline v_int64x4 v_dotprod_fast(const v_int32x8& a, const v_int32x8& b, const v_int64x4& c) -{ return v_dotprod(a, b, c); } - -// 8 >> 32 -inline v_uint32x8 v_dotprod_expand_fast(const v_uint8x32& a, const v_uint8x32& b) -{ return v_dotprod_expand(a, b); } -inline v_uint32x8 v_dotprod_expand_fast(const v_uint8x32& a, const v_uint8x32& b, const v_uint32x8& c) -{ return v_dotprod_expand(a, b, c); } - -inline v_int32x8 v_dotprod_expand_fast(const v_int8x32& a, const v_int8x32& b) -{ return v_dotprod_expand(a, b); } -inline v_int32x8 v_dotprod_expand_fast(const v_int8x32& a, const v_int8x32& b, const v_int32x8& c) -{ return v_dotprod_expand(a, b, c); } - -// 16 >> 64 -inline v_uint64x4 v_dotprod_expand_fast(const v_uint16x16& a, const v_uint16x16& b) -{ - __m256i mullo = _mm256_mullo_epi16(a.val, b.val); - __m256i mulhi = _mm256_mulhi_epu16(a.val, b.val); - __m256i mul0 = _mm256_unpacklo_epi16(mullo, mulhi); - __m256i mul1 = _mm256_unpackhi_epi16(mullo, mulhi); - - __m256i p02 = _mm256_blend_epi32(mul0, _mm256_setzero_si256(), 0xAA); - __m256i p13 = _mm256_srli_epi64(mul0, 32); - __m256i p46 = _mm256_blend_epi32(mul1, _mm256_setzero_si256(), 0xAA); - __m256i p57 = _mm256_srli_epi64(mul1, 32); - - __m256i p15_ = _mm256_add_epi64(p02, p13); - __m256i p9d_ = _mm256_add_epi64(p46, p57); - - return v_uint64x4(_mm256_add_epi64(p15_, p9d_)); -} -inline v_uint64x4 v_dotprod_expand_fast(const v_uint16x16& a, const v_uint16x16& b, const v_uint64x4& c) -{ return v_dotprod_expand_fast(a, b) + c; } - -inline v_int64x4 v_dotprod_expand_fast(const v_int16x16& a, const v_int16x16& b) -{ - __m256i prod = _mm256_madd_epi16(a.val, b.val); - __m256i sign = _mm256_srai_epi32(prod, 31); - __m256i lo = _mm256_unpacklo_epi32(prod, sign); - __m256i hi = _mm256_unpackhi_epi32(prod, sign); - return v_int64x4(_mm256_add_epi64(lo, hi)); -} -inline v_int64x4 v_dotprod_expand_fast(const v_int16x16& a, const v_int16x16& b, const v_int64x4& c) -{ return v_dotprod_expand_fast(a, b) + c; } - -// 32 >> 64f -inline v_float64x4 v_dotprod_expand_fast(const v_int32x8& a, const v_int32x8& b) -{ return v_dotprod_expand(a, b); } -inline v_float64x4 v_dotprod_expand_fast(const v_int32x8& a, const v_int32x8& b, const v_float64x4& c) -{ return v_dotprod_expand(a, b, c); } - -#define OPENCV_HAL_AVX_SPLAT2_PS(a, im) \ - v_float32x8(_mm256_permute_ps(a.val, _MM_SHUFFLE(im, im, im, im))) - -inline v_float32x8 v_matmul(const v_float32x8& v, const v_float32x8& m0, - const v_float32x8& m1, const v_float32x8& m2, - const v_float32x8& m3) -{ - v_float32x8 v04 = OPENCV_HAL_AVX_SPLAT2_PS(v, 0); - v_float32x8 v15 = OPENCV_HAL_AVX_SPLAT2_PS(v, 1); - v_float32x8 v26 = OPENCV_HAL_AVX_SPLAT2_PS(v, 2); - v_float32x8 v37 = OPENCV_HAL_AVX_SPLAT2_PS(v, 3); - return v_fma(v04, m0, v_fma(v15, m1, v_fma(v26, m2, v37 * m3))); -} - -inline v_float32x8 v_matmuladd(const v_float32x8& v, const v_float32x8& m0, - const v_float32x8& m1, const v_float32x8& m2, - const v_float32x8& a) -{ - v_float32x8 v04 = OPENCV_HAL_AVX_SPLAT2_PS(v, 0); - v_float32x8 v15 = OPENCV_HAL_AVX_SPLAT2_PS(v, 1); - v_float32x8 v26 = OPENCV_HAL_AVX_SPLAT2_PS(v, 2); - return v_fma(v04, m0, v_fma(v15, m1, v_fma(v26, m2, a))); -} - -#define OPENCV_HAL_IMPL_AVX_TRANSPOSE4x4(_Tpvec, suffix, cast_from, cast_to) \ - inline void v_transpose4x4(const _Tpvec& a0, const _Tpvec& a1, \ - const _Tpvec& a2, const _Tpvec& a3, \ - _Tpvec& b0, _Tpvec& b1, _Tpvec& b2, _Tpvec& b3) \ - { \ - __m256i t0 = cast_from(_mm256_unpacklo_##suffix(a0.val, a1.val)); \ - __m256i t1 = cast_from(_mm256_unpacklo_##suffix(a2.val, a3.val)); \ - __m256i t2 = cast_from(_mm256_unpackhi_##suffix(a0.val, a1.val)); \ - __m256i t3 = cast_from(_mm256_unpackhi_##suffix(a2.val, a3.val)); \ - b0.val = cast_to(_mm256_unpacklo_epi64(t0, t1)); \ - b1.val = cast_to(_mm256_unpackhi_epi64(t0, t1)); \ - b2.val = cast_to(_mm256_unpacklo_epi64(t2, t3)); \ - b3.val = cast_to(_mm256_unpackhi_epi64(t2, t3)); \ - } - -OPENCV_HAL_IMPL_AVX_TRANSPOSE4x4(v_uint32x8, epi32, OPENCV_HAL_NOP, OPENCV_HAL_NOP) -OPENCV_HAL_IMPL_AVX_TRANSPOSE4x4(v_int32x8, epi32, OPENCV_HAL_NOP, OPENCV_HAL_NOP) -OPENCV_HAL_IMPL_AVX_TRANSPOSE4x4(v_float32x8, ps, _mm256_castps_si256, _mm256_castsi256_ps) - - - -#define OPENCV_HAL_IMPL_AVX_EXPAND_Q(_Tpvec, _Tp, intrin) \ - inline _Tpvec v256_load_expand_q(const _Tp* ptr) \ - { \ - __m128i a = _mm_loadl_epi64((const __m128i*)ptr); \ - return _Tpvec(intrin(a)); \ - } - -OPENCV_HAL_IMPL_AVX_EXPAND_Q(v_uint32x8, uchar, _mm256_cvtepu8_epi32) -OPENCV_HAL_IMPL_AVX_EXPAND_Q(v_int32x8, schar, _mm256_cvtepi8_epi32) - -/* pack */ -// 16 -inline v_int8x32 v_pack(const v_int16x16& a, const v_int16x16& b) -{ return v_int8x32(_v256_shuffle_odd_64(_mm256_packs_epi16(a.val, b.val))); } - -inline v_uint8x32 v_pack(const v_uint16x16& a, const v_uint16x16& b) -{ - __m256i t = _mm256_set1_epi16(255); - __m256i a1 = _mm256_min_epu16(a.val, t); - __m256i b1 = _mm256_min_epu16(b.val, t); - return v_uint8x32(_v256_shuffle_odd_64(_mm256_packus_epi16(a1, b1))); -} - -inline v_uint8x32 v_pack_u(const v_int16x16& a, const v_int16x16& b) -{ - return v_uint8x32(_v256_shuffle_odd_64(_mm256_packus_epi16(a.val, b.val))); -} - -static inline v_uint8x32 v_packus(const v_int16x16& a, const v_int16x16& b) -{ - return v_uint8x32(_mm256_packus_epi16(a.val, b.val)); -} - -template -static inline v_uint8x32 v_blend_shiftleft(const v_uint8x32& a, const v_uint8x32& b) -{ - return v_uint8x32(_mm256_blend_epi16(a.val, _mm256_slli_si256(b.val, shift), mask)); -} - -template -static inline v_uint8x32 v_blend_shiftright(const v_uint8x32& a, const v_uint8x32& b) -{ - return v_uint8x32(_mm256_blend_epi16(_mm256_srli_si256(a.val, shift), b.val, mask)); -} - -template -static inline __m256 v_blend_shiftleft(const v_float32x8& a, const v_float32x8& b) -{ - return _mm256_castsi256_ps(_mm256_blend_epi32(_mm256_castps_si256(a.val), _mm256_slli_si256(_mm256_castps_si256(b.val), shift), mask)); -} - -template -static inline __m256 v_blend_shiftright(const v_float32x8& a, const v_float32x8& b) -{ - return _mm256_castsi256_ps(_mm256_blend_epi32(_mm256_srli_si256(_mm256_castps_si256(a.val), shift), _mm256_castps_si256(b.val), mask)); -} - -static inline v_uint8x32 v_setr_s8(char b0, char b1, char b2, char b3, char b4, - char b5, char b6, char b7, char b8, char b9, - char b10, char b11, char b12, char b13, char b14, - char b15, char b16, char b17, char b18, char b19, - char b20, char b21, char b22, char b23, char b24, - char b25, char b26, char b27, char b28, char b29, - char b30, char b31) -{ - return v_uint8x32(_mm256_setr_epi8(b0, b1, b2, b3, b4, b5, b6, b7, - b8, b9, b10, b11, b12, b13, b14, b15, - b16, b17, b18, b19, b20, b21, b22, b23, - b24, b25, b26, b27, b28, b29, b30, b31)); -} - -static inline v_uint32x8 v_setr_s32(int b0, int b1, int b2, int b3, int b4, int b5, int b6, int b7) -{ - return v_uint32x8(_mm256_setr_epi32(b0, b1, b2, b3, b4, b5, b6, b7)); -} - -inline void v_pack_store(schar* ptr, const v_int16x16& a) -{ v_store_low(ptr, v_pack(a, a)); } - -inline void v_pack_store(uchar* ptr, const v_uint16x16& a) -{ - const __m256i m = _mm256_set1_epi16(255); - __m256i am = _mm256_min_epu16(a.val, m); - am = _v256_shuffle_odd_64(_mm256_packus_epi16(am, am)); - v_store_low(ptr, v_uint8x32(am)); -} - -inline void v_pack_u_store(uchar* ptr, const v_int16x16& a) -{ v_store_low(ptr, v_pack_u(a, a)); } - -template inline -v_uint8x32 v_rshr_pack(const v_uint16x16& a, const v_uint16x16& b) -{ - // we assume that n > 0, and so the shifted 16-bit values can be treated as signed numbers. - v_uint16x16 delta = v256_setall_u16((short)(1 << (n-1))); - return v_pack_u(v_reinterpret_as_s16((a + delta) >> n), - v_reinterpret_as_s16((b + delta) >> n)); -} - -template inline -void v_rshr_pack_store(uchar* ptr, const v_uint16x16& a) -{ - v_uint16x16 delta = v256_setall_u16((short)(1 << (n-1))); - v_pack_u_store(ptr, v_reinterpret_as_s16((a + delta) >> n)); -} - -template inline -v_uint8x32 v_rshr_pack_u(const v_int16x16& a, const v_int16x16& b) -{ - v_int16x16 delta = v256_setall_s16((short)(1 << (n-1))); - return v_pack_u((a + delta) >> n, (b + delta) >> n); -} - -template inline -void v_rshr_pack_u_store(uchar* ptr, const v_int16x16& a) -{ - v_int16x16 delta = v256_setall_s16((short)(1 << (n-1))); - v_pack_u_store(ptr, (a + delta) >> n); -} - -template inline -v_int8x32 v_rshr_pack(const v_int16x16& a, const v_int16x16& b) -{ - v_int16x16 delta = v256_setall_s16((short)(1 << (n-1))); - return v_pack((a + delta) >> n, (b + delta) >> n); -} - -template inline -void v_rshr_pack_store(schar* ptr, const v_int16x16& a) -{ - v_int16x16 delta = v256_setall_s16((short)(1 << (n-1))); - v_pack_store(ptr, (a + delta) >> n); -} - -// 32 -inline v_int16x16 v_pack(const v_int32x8& a, const v_int32x8& b) -{ return v_int16x16(_v256_shuffle_odd_64(_mm256_packs_epi32(a.val, b.val))); } - -inline v_uint16x16 v_pack(const v_uint32x8& a, const v_uint32x8& b) -{ return v_uint16x16(_v256_shuffle_odd_64(_v256_packs_epu32(a.val, b.val))); } - -inline v_uint16x16 v_pack_u(const v_int32x8& a, const v_int32x8& b) -{ return v_uint16x16(_v256_shuffle_odd_64(_mm256_packus_epi32(a.val, b.val))); } - -inline void v_pack_store(short* ptr, const v_int32x8& a) -{ v_store_low(ptr, v_pack(a, a)); } - -inline void v_pack_store(ushort* ptr, const v_uint32x8& a) -{ - const __m256i m = _mm256_set1_epi32(65535); - __m256i am = _mm256_min_epu32(a.val, m); - am = _v256_shuffle_odd_64(_mm256_packus_epi32(am, am)); - v_store_low(ptr, v_uint16x16(am)); -} - -inline void v_pack_u_store(ushort* ptr, const v_int32x8& a) -{ v_store_low(ptr, v_pack_u(a, a)); } - - -template inline -v_uint16x16 v_rshr_pack(const v_uint32x8& a, const v_uint32x8& b) -{ - // we assume that n > 0, and so the shifted 32-bit values can be treated as signed numbers. - v_uint32x8 delta = v256_setall_u32(1 << (n-1)); - return v_pack_u(v_reinterpret_as_s32((a + delta) >> n), - v_reinterpret_as_s32((b + delta) >> n)); -} - -template inline -void v_rshr_pack_store(ushort* ptr, const v_uint32x8& a) -{ - v_uint32x8 delta = v256_setall_u32(1 << (n-1)); - v_pack_u_store(ptr, v_reinterpret_as_s32((a + delta) >> n)); -} - -template inline -v_uint16x16 v_rshr_pack_u(const v_int32x8& a, const v_int32x8& b) -{ - v_int32x8 delta = v256_setall_s32(1 << (n-1)); - return v_pack_u((a + delta) >> n, (b + delta) >> n); -} - -template inline -void v_rshr_pack_u_store(ushort* ptr, const v_int32x8& a) -{ - v_int32x8 delta = v256_setall_s32(1 << (n-1)); - v_pack_u_store(ptr, (a + delta) >> n); -} - -template inline -v_int16x16 v_rshr_pack(const v_int32x8& a, const v_int32x8& b) -{ - v_int32x8 delta = v256_setall_s32(1 << (n-1)); - return v_pack((a + delta) >> n, (b + delta) >> n); -} - -template inline -void v_rshr_pack_store(short* ptr, const v_int32x8& a) -{ - v_int32x8 delta = v256_setall_s32(1 << (n-1)); - v_pack_store(ptr, (a + delta) >> n); -} - -// 64 -// Non-saturating pack -inline v_uint32x8 v_pack(const v_uint64x4& a, const v_uint64x4& b) -{ - __m256i a0 = _mm256_shuffle_epi32(a.val, _MM_SHUFFLE(0, 0, 2, 0)); - __m256i b0 = _mm256_shuffle_epi32(b.val, _MM_SHUFFLE(0, 0, 2, 0)); - __m256i ab = _mm256_unpacklo_epi64(a0, b0); // a0, a1, b0, b1, a2, a3, b2, b3 - return v_uint32x8(_v256_shuffle_odd_64(ab)); -} - -inline v_int32x8 v_pack(const v_int64x4& a, const v_int64x4& b) -{ return v_reinterpret_as_s32(v_pack(v_reinterpret_as_u64(a), v_reinterpret_as_u64(b))); } - -inline void v_pack_store(unsigned* ptr, const v_uint64x4& a) -{ - __m256i a0 = _mm256_shuffle_epi32(a.val, _MM_SHUFFLE(0, 0, 2, 0)); - v_store_low(ptr, v_uint32x8(_v256_shuffle_odd_64(a0))); -} - -inline void v_pack_store(int* ptr, const v_int64x4& b) -{ v_pack_store((unsigned*)ptr, v_reinterpret_as_u64(b)); } - -template inline -v_uint32x8 v_rshr_pack(const v_uint64x4& a, const v_uint64x4& b) -{ - v_uint64x4 delta = v256_setall_u64((uint64)1 << (n-1)); - return v_pack((a + delta) >> n, (b + delta) >> n); -} - -template inline -void v_rshr_pack_store(unsigned* ptr, const v_uint64x4& a) -{ - v_uint64x4 delta = v256_setall_u64((uint64)1 << (n-1)); - v_pack_store(ptr, (a + delta) >> n); -} - -template inline -v_int32x8 v_rshr_pack(const v_int64x4& a, const v_int64x4& b) -{ - v_int64x4 delta = v256_setall_s64((int64)1 << (n-1)); - return v_pack((a + delta) >> n, (b + delta) >> n); -} - -template inline -void v_rshr_pack_store(int* ptr, const v_int64x4& a) -{ - v_int64x4 delta = v256_setall_s64((int64)1 << (n-1)); - v_pack_store(ptr, (a + delta) >> n); -} - -// pack boolean -inline v_uint8x32 v_pack_b(const v_uint16x16& a, const v_uint16x16& b) -{ - __m256i ab = _mm256_packs_epi16(a.val, b.val); - return v_uint8x32(_v256_shuffle_odd_64(ab)); -} - -inline v_uint8x32 v_pack_b(const v_uint32x8& a, const v_uint32x8& b, - const v_uint32x8& c, const v_uint32x8& d) -{ - __m256i ab = _mm256_packs_epi32(a.val, b.val); - __m256i cd = _mm256_packs_epi32(c.val, d.val); - - __m256i abcd = _v256_shuffle_odd_64(_mm256_packs_epi16(ab, cd)); - return v_uint8x32(_mm256_shuffle_epi32(abcd, _MM_SHUFFLE(3, 1, 2, 0))); -} - -inline v_uint8x32 v_pack_b(const v_uint64x4& a, const v_uint64x4& b, const v_uint64x4& c, - const v_uint64x4& d, const v_uint64x4& e, const v_uint64x4& f, - const v_uint64x4& g, const v_uint64x4& h) -{ - __m256i ab = _mm256_packs_epi32(a.val, b.val); - __m256i cd = _mm256_packs_epi32(c.val, d.val); - __m256i ef = _mm256_packs_epi32(e.val, f.val); - __m256i gh = _mm256_packs_epi32(g.val, h.val); - - __m256i abcd = _mm256_packs_epi32(ab, cd); - __m256i efgh = _mm256_packs_epi32(ef, gh); - __m256i pkall = _v256_shuffle_odd_64(_mm256_packs_epi16(abcd, efgh)); - - __m256i rev = _mm256_alignr_epi8(pkall, pkall, 8); - return v_uint8x32(_mm256_unpacklo_epi16(pkall, rev)); -} - -/* Recombine */ -// its up there with load and store operations - -/* Extract */ -#define OPENCV_HAL_IMPL_AVX_EXTRACT(_Tpvec) \ - template \ - inline _Tpvec v_extract(const _Tpvec& a, const _Tpvec& b) \ - { return v_rotate_right(a, b); } - -OPENCV_HAL_IMPL_AVX_EXTRACT(v_uint8x32) -OPENCV_HAL_IMPL_AVX_EXTRACT(v_int8x32) -OPENCV_HAL_IMPL_AVX_EXTRACT(v_uint16x16) -OPENCV_HAL_IMPL_AVX_EXTRACT(v_int16x16) -OPENCV_HAL_IMPL_AVX_EXTRACT(v_uint32x8) -OPENCV_HAL_IMPL_AVX_EXTRACT(v_int32x8) -OPENCV_HAL_IMPL_AVX_EXTRACT(v_uint64x4) -OPENCV_HAL_IMPL_AVX_EXTRACT(v_int64x4) -OPENCV_HAL_IMPL_AVX_EXTRACT(v_float32x8) -OPENCV_HAL_IMPL_AVX_EXTRACT(v_float64x4) - - -///////////////////// load deinterleave ///////////////////////////// - -inline void v_load_deinterleave( const uchar* ptr, v_uint8x32& a, v_uint8x32& b ) -{ - __m256i ab0 = _mm256_loadu_si256((const __m256i*)ptr); - __m256i ab1 = _mm256_loadu_si256((const __m256i*)(ptr + 32)); - - const __m256i sh = _mm256_setr_epi8(0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15, - 0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15); - __m256i p0 = _mm256_shuffle_epi8(ab0, sh); - __m256i p1 = _mm256_shuffle_epi8(ab1, sh); - __m256i pl = _mm256_permute2x128_si256(p0, p1, 0 + 2*16); - __m256i ph = _mm256_permute2x128_si256(p0, p1, 1 + 3*16); - __m256i a0 = _mm256_unpacklo_epi64(pl, ph); - __m256i b0 = _mm256_unpackhi_epi64(pl, ph); - a = v_uint8x32(a0); - b = v_uint8x32(b0); -} - -inline void v_load_deinterleave( const ushort* ptr, v_uint16x16& a, v_uint16x16& b ) -{ - __m256i ab0 = _mm256_loadu_si256((const __m256i*)ptr); - __m256i ab1 = _mm256_loadu_si256((const __m256i*)(ptr + 16)); - - const __m256i sh = _mm256_setr_epi8(0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15, - 0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15); - __m256i p0 = _mm256_shuffle_epi8(ab0, sh); - __m256i p1 = _mm256_shuffle_epi8(ab1, sh); - __m256i pl = _mm256_permute2x128_si256(p0, p1, 0 + 2*16); - __m256i ph = _mm256_permute2x128_si256(p0, p1, 1 + 3*16); - __m256i a0 = _mm256_unpacklo_epi64(pl, ph); - __m256i b0 = _mm256_unpackhi_epi64(pl, ph); - a = v_uint16x16(a0); - b = v_uint16x16(b0); -} - -inline void v_load_deinterleave( const unsigned* ptr, v_uint32x8& a, v_uint32x8& b ) -{ - __m256i ab0 = _mm256_loadu_si256((const __m256i*)ptr); - __m256i ab1 = _mm256_loadu_si256((const __m256i*)(ptr + 8)); - - const int sh = 0+2*4+1*16+3*64; - __m256i p0 = _mm256_shuffle_epi32(ab0, sh); - __m256i p1 = _mm256_shuffle_epi32(ab1, sh); - __m256i pl = _mm256_permute2x128_si256(p0, p1, 0 + 2*16); - __m256i ph = _mm256_permute2x128_si256(p0, p1, 1 + 3*16); - __m256i a0 = _mm256_unpacklo_epi64(pl, ph); - __m256i b0 = _mm256_unpackhi_epi64(pl, ph); - a = v_uint32x8(a0); - b = v_uint32x8(b0); -} - -inline void v_load_deinterleave( const uint64* ptr, v_uint64x4& a, v_uint64x4& b ) -{ - __m256i ab0 = _mm256_loadu_si256((const __m256i*)ptr); - __m256i ab1 = _mm256_loadu_si256((const __m256i*)(ptr + 4)); - - __m256i pl = _mm256_permute2x128_si256(ab0, ab1, 0 + 2*16); - __m256i ph = _mm256_permute2x128_si256(ab0, ab1, 1 + 3*16); - __m256i a0 = _mm256_unpacklo_epi64(pl, ph); - __m256i b0 = _mm256_unpackhi_epi64(pl, ph); - a = v_uint64x4(a0); - b = v_uint64x4(b0); -} - -inline void v_load_deinterleave( const uchar* ptr, v_uint8x32& a, v_uint8x32& b, v_uint8x32& c ) -{ - __m256i bgr0 = _mm256_loadu_si256((const __m256i*)ptr); - __m256i bgr1 = _mm256_loadu_si256((const __m256i*)(ptr + 32)); - __m256i bgr2 = _mm256_loadu_si256((const __m256i*)(ptr + 64)); - - __m256i s02_low = _mm256_permute2x128_si256(bgr0, bgr2, 0 + 2*16); - __m256i s02_high = _mm256_permute2x128_si256(bgr0, bgr2, 1 + 3*16); - - const __m256i m0 = _mm256_setr_epi8(0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, - 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0); - const __m256i m1 = _mm256_setr_epi8(0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, - -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1); - - __m256i b0 = _mm256_blendv_epi8(_mm256_blendv_epi8(s02_low, s02_high, m0), bgr1, m1); - __m256i g0 = _mm256_blendv_epi8(_mm256_blendv_epi8(s02_high, s02_low, m1), bgr1, m0); - __m256i r0 = _mm256_blendv_epi8(_mm256_blendv_epi8(bgr1, s02_low, m0), s02_high, m1); - - const __m256i - sh_b = _mm256_setr_epi8(0, 3, 6, 9, 12, 15, 2, 5, 8, 11, 14, 1, 4, 7, 10, 13, - 0, 3, 6, 9, 12, 15, 2, 5, 8, 11, 14, 1, 4, 7, 10, 13), - sh_g = _mm256_setr_epi8(1, 4, 7, 10, 13, 0, 3, 6, 9, 12, 15, 2, 5, 8, 11, 14, - 1, 4, 7, 10, 13, 0, 3, 6, 9, 12, 15, 2, 5, 8, 11, 14), - sh_r = _mm256_setr_epi8(2, 5, 8, 11, 14, 1, 4, 7, 10, 13, 0, 3, 6, 9, 12, 15, - 2, 5, 8, 11, 14, 1, 4, 7, 10, 13, 0, 3, 6, 9, 12, 15); - b0 = _mm256_shuffle_epi8(b0, sh_b); - g0 = _mm256_shuffle_epi8(g0, sh_g); - r0 = _mm256_shuffle_epi8(r0, sh_r); - - a = v_uint8x32(b0); - b = v_uint8x32(g0); - c = v_uint8x32(r0); -} - -inline void v_load_deinterleave( const ushort* ptr, v_uint16x16& a, v_uint16x16& b, v_uint16x16& c ) -{ - __m256i bgr0 = _mm256_loadu_si256((const __m256i*)ptr); - __m256i bgr1 = _mm256_loadu_si256((const __m256i*)(ptr + 16)); - __m256i bgr2 = _mm256_loadu_si256((const __m256i*)(ptr + 32)); - - __m256i s02_low = _mm256_permute2x128_si256(bgr0, bgr2, 0 + 2*16); - __m256i s02_high = _mm256_permute2x128_si256(bgr0, bgr2, 1 + 3*16); - - const __m256i m0 = _mm256_setr_epi8(0, 0, -1, -1, 0, 0, 0, 0, -1, -1, 0, 0, 0, 0, -1, -1, - 0, 0, 0, 0, -1, -1, 0, 0, 0, 0, -1, -1, 0, 0, 0, 0); - const __m256i m1 = _mm256_setr_epi8(0, 0, 0, 0, -1, -1, 0, 0, 0, 0, -1, -1, 0, 0, 0, 0, - -1, -1, 0, 0, 0, 0, -1, -1, 0, 0, 0, 0, -1, -1, 0, 0); - __m256i b0 = _mm256_blendv_epi8(_mm256_blendv_epi8(s02_low, s02_high, m0), bgr1, m1); - __m256i g0 = _mm256_blendv_epi8(_mm256_blendv_epi8(bgr1, s02_low, m0), s02_high, m1); - __m256i r0 = _mm256_blendv_epi8(_mm256_blendv_epi8(s02_high, s02_low, m1), bgr1, m0); - const __m256i sh_b = _mm256_setr_epi8(0, 1, 6, 7, 12, 13, 2, 3, 8, 9, 14, 15, 4, 5, 10, 11, - 0, 1, 6, 7, 12, 13, 2, 3, 8, 9, 14, 15, 4, 5, 10, 11); - const __m256i sh_g = _mm256_setr_epi8(2, 3, 8, 9, 14, 15, 4, 5, 10, 11, 0, 1, 6, 7, 12, 13, - 2, 3, 8, 9, 14, 15, 4, 5, 10, 11, 0, 1, 6, 7, 12, 13); - const __m256i sh_r = _mm256_setr_epi8(4, 5, 10, 11, 0, 1, 6, 7, 12, 13, 2, 3, 8, 9, 14, 15, - 4, 5, 10, 11, 0, 1, 6, 7, 12, 13, 2, 3, 8, 9, 14, 15); - b0 = _mm256_shuffle_epi8(b0, sh_b); - g0 = _mm256_shuffle_epi8(g0, sh_g); - r0 = _mm256_shuffle_epi8(r0, sh_r); - - a = v_uint16x16(b0); - b = v_uint16x16(g0); - c = v_uint16x16(r0); -} - -inline void v_load_deinterleave( const unsigned* ptr, v_uint32x8& a, v_uint32x8& b, v_uint32x8& c ) -{ - __m256i bgr0 = _mm256_loadu_si256((const __m256i*)ptr); - __m256i bgr1 = _mm256_loadu_si256((const __m256i*)(ptr + 8)); - __m256i bgr2 = _mm256_loadu_si256((const __m256i*)(ptr + 16)); - - __m256i s02_low = _mm256_permute2x128_si256(bgr0, bgr2, 0 + 2*16); - __m256i s02_high = _mm256_permute2x128_si256(bgr0, bgr2, 1 + 3*16); - - __m256i b0 = _mm256_blend_epi32(_mm256_blend_epi32(s02_low, s02_high, 0x24), bgr1, 0x92); - __m256i g0 = _mm256_blend_epi32(_mm256_blend_epi32(s02_high, s02_low, 0x92), bgr1, 0x24); - __m256i r0 = _mm256_blend_epi32(_mm256_blend_epi32(bgr1, s02_low, 0x24), s02_high, 0x92); - - b0 = _mm256_shuffle_epi32(b0, 0x6c); - g0 = _mm256_shuffle_epi32(g0, 0xb1); - r0 = _mm256_shuffle_epi32(r0, 0xc6); - - a = v_uint32x8(b0); - b = v_uint32x8(g0); - c = v_uint32x8(r0); -} - -inline void v_load_deinterleave( const uint64* ptr, v_uint64x4& a, v_uint64x4& b, v_uint64x4& c ) -{ - __m256i bgr0 = _mm256_loadu_si256((const __m256i*)ptr); - __m256i bgr1 = _mm256_loadu_si256((const __m256i*)(ptr + 4)); - __m256i bgr2 = _mm256_loadu_si256((const __m256i*)(ptr + 8)); - - __m256i s01 = _mm256_blend_epi32(bgr0, bgr1, 0xf0); - __m256i s12 = _mm256_blend_epi32(bgr1, bgr2, 0xf0); - __m256i s20r = _mm256_permute4x64_epi64(_mm256_blend_epi32(bgr2, bgr0, 0xf0), 0x1b); - __m256i b0 = _mm256_unpacklo_epi64(s01, s20r); - __m256i g0 = _mm256_alignr_epi8(s12, s01, 8); - __m256i r0 = _mm256_unpackhi_epi64(s20r, s12); - - a = v_uint64x4(b0); - b = v_uint64x4(g0); - c = v_uint64x4(r0); -} - -inline void v_load_deinterleave( const uchar* ptr, v_uint8x32& a, v_uint8x32& b, v_uint8x32& c, v_uint8x32& d ) -{ - __m256i bgr0 = _mm256_loadu_si256((const __m256i*)ptr); - __m256i bgr1 = _mm256_loadu_si256((const __m256i*)(ptr + 32)); - __m256i bgr2 = _mm256_loadu_si256((const __m256i*)(ptr + 64)); - __m256i bgr3 = _mm256_loadu_si256((const __m256i*)(ptr + 96)); - const __m256i sh = _mm256_setr_epi8(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15, - 0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15); - - __m256i p0 = _mm256_shuffle_epi8(bgr0, sh); - __m256i p1 = _mm256_shuffle_epi8(bgr1, sh); - __m256i p2 = _mm256_shuffle_epi8(bgr2, sh); - __m256i p3 = _mm256_shuffle_epi8(bgr3, sh); - - __m256i p01l = _mm256_unpacklo_epi32(p0, p1); - __m256i p01h = _mm256_unpackhi_epi32(p0, p1); - __m256i p23l = _mm256_unpacklo_epi32(p2, p3); - __m256i p23h = _mm256_unpackhi_epi32(p2, p3); - - __m256i pll = _mm256_permute2x128_si256(p01l, p23l, 0 + 2*16); - __m256i plh = _mm256_permute2x128_si256(p01l, p23l, 1 + 3*16); - __m256i phl = _mm256_permute2x128_si256(p01h, p23h, 0 + 2*16); - __m256i phh = _mm256_permute2x128_si256(p01h, p23h, 1 + 3*16); - - __m256i b0 = _mm256_unpacklo_epi32(pll, plh); - __m256i g0 = _mm256_unpackhi_epi32(pll, plh); - __m256i r0 = _mm256_unpacklo_epi32(phl, phh); - __m256i a0 = _mm256_unpackhi_epi32(phl, phh); - - a = v_uint8x32(b0); - b = v_uint8x32(g0); - c = v_uint8x32(r0); - d = v_uint8x32(a0); -} - -inline void v_load_deinterleave( const ushort* ptr, v_uint16x16& a, v_uint16x16& b, v_uint16x16& c, v_uint16x16& d ) -{ - __m256i bgr0 = _mm256_loadu_si256((const __m256i*)ptr); - __m256i bgr1 = _mm256_loadu_si256((const __m256i*)(ptr + 16)); - __m256i bgr2 = _mm256_loadu_si256((const __m256i*)(ptr + 32)); - __m256i bgr3 = _mm256_loadu_si256((const __m256i*)(ptr + 48)); - const __m256i sh = _mm256_setr_epi8(0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15, - 0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15); - __m256i p0 = _mm256_shuffle_epi8(bgr0, sh); - __m256i p1 = _mm256_shuffle_epi8(bgr1, sh); - __m256i p2 = _mm256_shuffle_epi8(bgr2, sh); - __m256i p3 = _mm256_shuffle_epi8(bgr3, sh); - - __m256i p01l = _mm256_unpacklo_epi32(p0, p1); - __m256i p01h = _mm256_unpackhi_epi32(p0, p1); - __m256i p23l = _mm256_unpacklo_epi32(p2, p3); - __m256i p23h = _mm256_unpackhi_epi32(p2, p3); - - __m256i pll = _mm256_permute2x128_si256(p01l, p23l, 0 + 2*16); - __m256i plh = _mm256_permute2x128_si256(p01l, p23l, 1 + 3*16); - __m256i phl = _mm256_permute2x128_si256(p01h, p23h, 0 + 2*16); - __m256i phh = _mm256_permute2x128_si256(p01h, p23h, 1 + 3*16); - - __m256i b0 = _mm256_unpacklo_epi32(pll, plh); - __m256i g0 = _mm256_unpackhi_epi32(pll, plh); - __m256i r0 = _mm256_unpacklo_epi32(phl, phh); - __m256i a0 = _mm256_unpackhi_epi32(phl, phh); - - a = v_uint16x16(b0); - b = v_uint16x16(g0); - c = v_uint16x16(r0); - d = v_uint16x16(a0); -} - -inline void v_load_deinterleave( const unsigned* ptr, v_uint32x8& a, v_uint32x8& b, v_uint32x8& c, v_uint32x8& d ) -{ - __m256i p0 = _mm256_loadu_si256((const __m256i*)ptr); - __m256i p1 = _mm256_loadu_si256((const __m256i*)(ptr + 8)); - __m256i p2 = _mm256_loadu_si256((const __m256i*)(ptr + 16)); - __m256i p3 = _mm256_loadu_si256((const __m256i*)(ptr + 24)); - - __m256i p01l = _mm256_unpacklo_epi32(p0, p1); - __m256i p01h = _mm256_unpackhi_epi32(p0, p1); - __m256i p23l = _mm256_unpacklo_epi32(p2, p3); - __m256i p23h = _mm256_unpackhi_epi32(p2, p3); - - __m256i pll = _mm256_permute2x128_si256(p01l, p23l, 0 + 2*16); - __m256i plh = _mm256_permute2x128_si256(p01l, p23l, 1 + 3*16); - __m256i phl = _mm256_permute2x128_si256(p01h, p23h, 0 + 2*16); - __m256i phh = _mm256_permute2x128_si256(p01h, p23h, 1 + 3*16); - - __m256i b0 = _mm256_unpacklo_epi32(pll, plh); - __m256i g0 = _mm256_unpackhi_epi32(pll, plh); - __m256i r0 = _mm256_unpacklo_epi32(phl, phh); - __m256i a0 = _mm256_unpackhi_epi32(phl, phh); - - a = v_uint32x8(b0); - b = v_uint32x8(g0); - c = v_uint32x8(r0); - d = v_uint32x8(a0); -} - -inline void v_load_deinterleave( const uint64* ptr, v_uint64x4& a, v_uint64x4& b, v_uint64x4& c, v_uint64x4& d ) -{ - __m256i bgra0 = _mm256_loadu_si256((const __m256i*)ptr); - __m256i bgra1 = _mm256_loadu_si256((const __m256i*)(ptr + 4)); - __m256i bgra2 = _mm256_loadu_si256((const __m256i*)(ptr + 8)); - __m256i bgra3 = _mm256_loadu_si256((const __m256i*)(ptr + 12)); - - __m256i l02 = _mm256_permute2x128_si256(bgra0, bgra2, 0 + 2*16); - __m256i h02 = _mm256_permute2x128_si256(bgra0, bgra2, 1 + 3*16); - __m256i l13 = _mm256_permute2x128_si256(bgra1, bgra3, 0 + 2*16); - __m256i h13 = _mm256_permute2x128_si256(bgra1, bgra3, 1 + 3*16); - - __m256i b0 = _mm256_unpacklo_epi64(l02, l13); - __m256i g0 = _mm256_unpackhi_epi64(l02, l13); - __m256i r0 = _mm256_unpacklo_epi64(h02, h13); - __m256i a0 = _mm256_unpackhi_epi64(h02, h13); - - a = v_uint64x4(b0); - b = v_uint64x4(g0); - c = v_uint64x4(r0); - d = v_uint64x4(a0); -} - -///////////////////////////// store interleave ///////////////////////////////////// - -inline void v_store_interleave( uchar* ptr, const v_uint8x32& x, const v_uint8x32& y, - hal::StoreMode mode=hal::STORE_UNALIGNED ) -{ - __m256i xy_l = _mm256_unpacklo_epi8(x.val, y.val); - __m256i xy_h = _mm256_unpackhi_epi8(x.val, y.val); - - __m256i xy0 = _mm256_permute2x128_si256(xy_l, xy_h, 0 + 2*16); - __m256i xy1 = _mm256_permute2x128_si256(xy_l, xy_h, 1 + 3*16); - - if( mode == hal::STORE_ALIGNED_NOCACHE ) - { - _mm256_stream_si256((__m256i*)ptr, xy0); - _mm256_stream_si256((__m256i*)(ptr + 32), xy1); - } - else if( mode == hal::STORE_ALIGNED ) - { - _mm256_store_si256((__m256i*)ptr, xy0); - _mm256_store_si256((__m256i*)(ptr + 32), xy1); - } - else - { - _mm256_storeu_si256((__m256i*)ptr, xy0); - _mm256_storeu_si256((__m256i*)(ptr + 32), xy1); - } -} - -inline void v_store_interleave( ushort* ptr, const v_uint16x16& x, const v_uint16x16& y, - hal::StoreMode mode=hal::STORE_UNALIGNED ) -{ - __m256i xy_l = _mm256_unpacklo_epi16(x.val, y.val); - __m256i xy_h = _mm256_unpackhi_epi16(x.val, y.val); - - __m256i xy0 = _mm256_permute2x128_si256(xy_l, xy_h, 0 + 2*16); - __m256i xy1 = _mm256_permute2x128_si256(xy_l, xy_h, 1 + 3*16); - - if( mode == hal::STORE_ALIGNED_NOCACHE ) - { - _mm256_stream_si256((__m256i*)ptr, xy0); - _mm256_stream_si256((__m256i*)(ptr + 16), xy1); - } - else if( mode == hal::STORE_ALIGNED ) - { - _mm256_store_si256((__m256i*)ptr, xy0); - _mm256_store_si256((__m256i*)(ptr + 16), xy1); - } - else - { - _mm256_storeu_si256((__m256i*)ptr, xy0); - _mm256_storeu_si256((__m256i*)(ptr + 16), xy1); - } -} - -inline void v_store_interleave( unsigned* ptr, const v_uint32x8& x, const v_uint32x8& y, - hal::StoreMode mode=hal::STORE_UNALIGNED ) -{ - __m256i xy_l = _mm256_unpacklo_epi32(x.val, y.val); - __m256i xy_h = _mm256_unpackhi_epi32(x.val, y.val); - - __m256i xy0 = _mm256_permute2x128_si256(xy_l, xy_h, 0 + 2*16); - __m256i xy1 = _mm256_permute2x128_si256(xy_l, xy_h, 1 + 3*16); - - if( mode == hal::STORE_ALIGNED_NOCACHE ) - { - _mm256_stream_si256((__m256i*)ptr, xy0); - _mm256_stream_si256((__m256i*)(ptr + 8), xy1); - } - else if( mode == hal::STORE_ALIGNED ) - { - _mm256_store_si256((__m256i*)ptr, xy0); - _mm256_store_si256((__m256i*)(ptr + 8), xy1); - } - else - { - _mm256_storeu_si256((__m256i*)ptr, xy0); - _mm256_storeu_si256((__m256i*)(ptr + 8), xy1); - } -} - -inline void v_store_interleave( uint64* ptr, const v_uint64x4& x, const v_uint64x4& y, - hal::StoreMode mode=hal::STORE_UNALIGNED ) -{ - __m256i xy_l = _mm256_unpacklo_epi64(x.val, y.val); - __m256i xy_h = _mm256_unpackhi_epi64(x.val, y.val); - - __m256i xy0 = _mm256_permute2x128_si256(xy_l, xy_h, 0 + 2*16); - __m256i xy1 = _mm256_permute2x128_si256(xy_l, xy_h, 1 + 3*16); - - if( mode == hal::STORE_ALIGNED_NOCACHE ) - { - _mm256_stream_si256((__m256i*)ptr, xy0); - _mm256_stream_si256((__m256i*)(ptr + 4), xy1); - } - else if( mode == hal::STORE_ALIGNED ) - { - _mm256_store_si256((__m256i*)ptr, xy0); - _mm256_store_si256((__m256i*)(ptr + 4), xy1); - } - else - { - _mm256_storeu_si256((__m256i*)ptr, xy0); - _mm256_storeu_si256((__m256i*)(ptr + 4), xy1); - } -} - -inline void v_store_interleave( uchar* ptr, const v_uint8x32& a, const v_uint8x32& b, const v_uint8x32& c, - hal::StoreMode mode=hal::STORE_UNALIGNED ) -{ - const __m256i sh_b = _mm256_setr_epi8( - 0, 11, 6, 1, 12, 7, 2, 13, 8, 3, 14, 9, 4, 15, 10, 5, - 0, 11, 6, 1, 12, 7, 2, 13, 8, 3, 14, 9, 4, 15, 10, 5); - const __m256i sh_g = _mm256_setr_epi8( - 5, 0, 11, 6, 1, 12, 7, 2, 13, 8, 3, 14, 9, 4, 15, 10, - 5, 0, 11, 6, 1, 12, 7, 2, 13, 8, 3, 14, 9, 4, 15, 10); - const __m256i sh_r = _mm256_setr_epi8( - 10, 5, 0, 11, 6, 1, 12, 7, 2, 13, 8, 3, 14, 9, 4, 15, - 10, 5, 0, 11, 6, 1, 12, 7, 2, 13, 8, 3, 14, 9, 4, 15); - - __m256i b0 = _mm256_shuffle_epi8(a.val, sh_b); - __m256i g0 = _mm256_shuffle_epi8(b.val, sh_g); - __m256i r0 = _mm256_shuffle_epi8(c.val, sh_r); - - const __m256i m0 = _mm256_setr_epi8(0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, - 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0); - const __m256i m1 = _mm256_setr_epi8(0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, - 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0); - - __m256i p0 = _mm256_blendv_epi8(_mm256_blendv_epi8(b0, g0, m0), r0, m1); - __m256i p1 = _mm256_blendv_epi8(_mm256_blendv_epi8(g0, r0, m0), b0, m1); - __m256i p2 = _mm256_blendv_epi8(_mm256_blendv_epi8(r0, b0, m0), g0, m1); - - __m256i bgr0 = _mm256_permute2x128_si256(p0, p1, 0 + 2*16); - __m256i bgr1 = _mm256_permute2x128_si256(p2, p0, 0 + 3*16); - __m256i bgr2 = _mm256_permute2x128_si256(p1, p2, 1 + 3*16); - - if( mode == hal::STORE_ALIGNED_NOCACHE ) - { - _mm256_stream_si256((__m256i*)ptr, bgr0); - _mm256_stream_si256((__m256i*)(ptr + 32), bgr1); - _mm256_stream_si256((__m256i*)(ptr + 64), bgr2); - } - else if( mode == hal::STORE_ALIGNED ) - { - _mm256_store_si256((__m256i*)ptr, bgr0); - _mm256_store_si256((__m256i*)(ptr + 32), bgr1); - _mm256_store_si256((__m256i*)(ptr + 64), bgr2); - } - else - { - _mm256_storeu_si256((__m256i*)ptr, bgr0); - _mm256_storeu_si256((__m256i*)(ptr + 32), bgr1); - _mm256_storeu_si256((__m256i*)(ptr + 64), bgr2); - } -} - -inline void v_store_interleave( ushort* ptr, const v_uint16x16& a, const v_uint16x16& b, const v_uint16x16& c, - hal::StoreMode mode=hal::STORE_UNALIGNED ) -{ - const __m256i sh_b = _mm256_setr_epi8( - 0, 1, 6, 7, 12, 13, 2, 3, 8, 9, 14, 15, 4, 5, 10, 11, - 0, 1, 6, 7, 12, 13, 2, 3, 8, 9, 14, 15, 4, 5, 10, 11); - const __m256i sh_g = _mm256_setr_epi8( - 10, 11, 0, 1, 6, 7, 12, 13, 2, 3, 8, 9, 14, 15, 4, 5, - 10, 11, 0, 1, 6, 7, 12, 13, 2, 3, 8, 9, 14, 15, 4, 5); - const __m256i sh_r = _mm256_setr_epi8( - 4, 5, 10, 11, 0, 1, 6, 7, 12, 13, 2, 3, 8, 9, 14, 15, - 4, 5, 10, 11, 0, 1, 6, 7, 12, 13, 2, 3, 8, 9, 14, 15); - - __m256i b0 = _mm256_shuffle_epi8(a.val, sh_b); - __m256i g0 = _mm256_shuffle_epi8(b.val, sh_g); - __m256i r0 = _mm256_shuffle_epi8(c.val, sh_r); - - const __m256i m0 = _mm256_setr_epi8(0, 0, -1, -1, 0, 0, 0, 0, -1, -1, 0, 0, 0, 0, -1, -1, - 0, 0, 0, 0, -1, -1, 0, 0, 0, 0, -1, -1, 0, 0, 0, 0); - const __m256i m1 = _mm256_setr_epi8(0, 0, 0, 0, -1, -1, 0, 0, 0, 0, -1, -1, 0, 0, 0, 0, - -1, -1, 0, 0, 0, 0, -1, -1, 0, 0, 0, 0, -1, -1, 0, 0); - - __m256i p0 = _mm256_blendv_epi8(_mm256_blendv_epi8(b0, g0, m0), r0, m1); - __m256i p1 = _mm256_blendv_epi8(_mm256_blendv_epi8(g0, r0, m0), b0, m1); - __m256i p2 = _mm256_blendv_epi8(_mm256_blendv_epi8(r0, b0, m0), g0, m1); - - __m256i bgr0 = _mm256_permute2x128_si256(p0, p2, 0 + 2*16); - //__m256i bgr1 = p1; - __m256i bgr2 = _mm256_permute2x128_si256(p0, p2, 1 + 3*16); - - if( mode == hal::STORE_ALIGNED_NOCACHE ) - { - _mm256_stream_si256((__m256i*)ptr, bgr0); - _mm256_stream_si256((__m256i*)(ptr + 16), p1); - _mm256_stream_si256((__m256i*)(ptr + 32), bgr2); - } - else if( mode == hal::STORE_ALIGNED ) - { - _mm256_store_si256((__m256i*)ptr, bgr0); - _mm256_store_si256((__m256i*)(ptr + 16), p1); - _mm256_store_si256((__m256i*)(ptr + 32), bgr2); - } - else - { - _mm256_storeu_si256((__m256i*)ptr, bgr0); - _mm256_storeu_si256((__m256i*)(ptr + 16), p1); - _mm256_storeu_si256((__m256i*)(ptr + 32), bgr2); - } -} - -inline void v_store_interleave( unsigned* ptr, const v_uint32x8& a, const v_uint32x8& b, const v_uint32x8& c, - hal::StoreMode mode=hal::STORE_UNALIGNED ) -{ - __m256i b0 = _mm256_shuffle_epi32(a.val, 0x6c); - __m256i g0 = _mm256_shuffle_epi32(b.val, 0xb1); - __m256i r0 = _mm256_shuffle_epi32(c.val, 0xc6); - - __m256i p0 = _mm256_blend_epi32(_mm256_blend_epi32(b0, g0, 0x92), r0, 0x24); - __m256i p1 = _mm256_blend_epi32(_mm256_blend_epi32(g0, r0, 0x92), b0, 0x24); - __m256i p2 = _mm256_blend_epi32(_mm256_blend_epi32(r0, b0, 0x92), g0, 0x24); - - __m256i bgr0 = _mm256_permute2x128_si256(p0, p1, 0 + 2*16); - //__m256i bgr1 = p2; - __m256i bgr2 = _mm256_permute2x128_si256(p0, p1, 1 + 3*16); - - if( mode == hal::STORE_ALIGNED_NOCACHE ) - { - _mm256_stream_si256((__m256i*)ptr, bgr0); - _mm256_stream_si256((__m256i*)(ptr + 8), p2); - _mm256_stream_si256((__m256i*)(ptr + 16), bgr2); - } - else if( mode == hal::STORE_ALIGNED ) - { - _mm256_store_si256((__m256i*)ptr, bgr0); - _mm256_store_si256((__m256i*)(ptr + 8), p2); - _mm256_store_si256((__m256i*)(ptr + 16), bgr2); - } - else - { - _mm256_storeu_si256((__m256i*)ptr, bgr0); - _mm256_storeu_si256((__m256i*)(ptr + 8), p2); - _mm256_storeu_si256((__m256i*)(ptr + 16), bgr2); - } -} - -inline void v_store_interleave( uint64* ptr, const v_uint64x4& a, const v_uint64x4& b, const v_uint64x4& c, - hal::StoreMode mode=hal::STORE_UNALIGNED ) -{ - __m256i s01 = _mm256_unpacklo_epi64(a.val, b.val); - __m256i s12 = _mm256_unpackhi_epi64(b.val, c.val); - __m256i s20 = _mm256_blend_epi32(c.val, a.val, 0xcc); - - __m256i bgr0 = _mm256_permute2x128_si256(s01, s20, 0 + 2*16); - __m256i bgr1 = _mm256_blend_epi32(s01, s12, 0x0f); - __m256i bgr2 = _mm256_permute2x128_si256(s20, s12, 1 + 3*16); - - if( mode == hal::STORE_ALIGNED_NOCACHE ) - { - _mm256_stream_si256((__m256i*)ptr, bgr0); - _mm256_stream_si256((__m256i*)(ptr + 4), bgr1); - _mm256_stream_si256((__m256i*)(ptr + 8), bgr2); - } - else if( mode == hal::STORE_ALIGNED ) - { - _mm256_store_si256((__m256i*)ptr, bgr0); - _mm256_store_si256((__m256i*)(ptr + 4), bgr1); - _mm256_store_si256((__m256i*)(ptr + 8), bgr2); - } - else - { - _mm256_storeu_si256((__m256i*)ptr, bgr0); - _mm256_storeu_si256((__m256i*)(ptr + 4), bgr1); - _mm256_storeu_si256((__m256i*)(ptr + 8), bgr2); - } -} - -inline void v_store_interleave( uchar* ptr, const v_uint8x32& a, const v_uint8x32& b, - const v_uint8x32& c, const v_uint8x32& d, - hal::StoreMode mode=hal::STORE_UNALIGNED ) -{ - __m256i bg0 = _mm256_unpacklo_epi8(a.val, b.val); - __m256i bg1 = _mm256_unpackhi_epi8(a.val, b.val); - __m256i ra0 = _mm256_unpacklo_epi8(c.val, d.val); - __m256i ra1 = _mm256_unpackhi_epi8(c.val, d.val); - - __m256i bgra0_ = _mm256_unpacklo_epi16(bg0, ra0); - __m256i bgra1_ = _mm256_unpackhi_epi16(bg0, ra0); - __m256i bgra2_ = _mm256_unpacklo_epi16(bg1, ra1); - __m256i bgra3_ = _mm256_unpackhi_epi16(bg1, ra1); - - __m256i bgra0 = _mm256_permute2x128_si256(bgra0_, bgra1_, 0 + 2*16); - __m256i bgra2 = _mm256_permute2x128_si256(bgra0_, bgra1_, 1 + 3*16); - __m256i bgra1 = _mm256_permute2x128_si256(bgra2_, bgra3_, 0 + 2*16); - __m256i bgra3 = _mm256_permute2x128_si256(bgra2_, bgra3_, 1 + 3*16); - - if( mode == hal::STORE_ALIGNED_NOCACHE ) - { - _mm256_stream_si256((__m256i*)ptr, bgra0); - _mm256_stream_si256((__m256i*)(ptr + 32), bgra1); - _mm256_stream_si256((__m256i*)(ptr + 64), bgra2); - _mm256_stream_si256((__m256i*)(ptr + 96), bgra3); - } - else if( mode == hal::STORE_ALIGNED ) - { - _mm256_store_si256((__m256i*)ptr, bgra0); - _mm256_store_si256((__m256i*)(ptr + 32), bgra1); - _mm256_store_si256((__m256i*)(ptr + 64), bgra2); - _mm256_store_si256((__m256i*)(ptr + 96), bgra3); - } - else - { - _mm256_storeu_si256((__m256i*)ptr, bgra0); - _mm256_storeu_si256((__m256i*)(ptr + 32), bgra1); - _mm256_storeu_si256((__m256i*)(ptr + 64), bgra2); - _mm256_storeu_si256((__m256i*)(ptr + 96), bgra3); - } -} - -inline void v_store_interleave( ushort* ptr, const v_uint16x16& a, const v_uint16x16& b, - const v_uint16x16& c, const v_uint16x16& d, - hal::StoreMode mode=hal::STORE_UNALIGNED ) -{ - __m256i bg0 = _mm256_unpacklo_epi16(a.val, b.val); - __m256i bg1 = _mm256_unpackhi_epi16(a.val, b.val); - __m256i ra0 = _mm256_unpacklo_epi16(c.val, d.val); - __m256i ra1 = _mm256_unpackhi_epi16(c.val, d.val); - - __m256i bgra0_ = _mm256_unpacklo_epi32(bg0, ra0); - __m256i bgra1_ = _mm256_unpackhi_epi32(bg0, ra0); - __m256i bgra2_ = _mm256_unpacklo_epi32(bg1, ra1); - __m256i bgra3_ = _mm256_unpackhi_epi32(bg1, ra1); - - __m256i bgra0 = _mm256_permute2x128_si256(bgra0_, bgra1_, 0 + 2*16); - __m256i bgra2 = _mm256_permute2x128_si256(bgra0_, bgra1_, 1 + 3*16); - __m256i bgra1 = _mm256_permute2x128_si256(bgra2_, bgra3_, 0 + 2*16); - __m256i bgra3 = _mm256_permute2x128_si256(bgra2_, bgra3_, 1 + 3*16); - - if( mode == hal::STORE_ALIGNED_NOCACHE ) - { - _mm256_stream_si256((__m256i*)ptr, bgra0); - _mm256_stream_si256((__m256i*)(ptr + 16), bgra1); - _mm256_stream_si256((__m256i*)(ptr + 32), bgra2); - _mm256_stream_si256((__m256i*)(ptr + 48), bgra3); - } - else if( mode == hal::STORE_ALIGNED ) - { - _mm256_store_si256((__m256i*)ptr, bgra0); - _mm256_store_si256((__m256i*)(ptr + 16), bgra1); - _mm256_store_si256((__m256i*)(ptr + 32), bgra2); - _mm256_store_si256((__m256i*)(ptr + 48), bgra3); - } - else - { - _mm256_storeu_si256((__m256i*)ptr, bgra0); - _mm256_storeu_si256((__m256i*)(ptr + 16), bgra1); - _mm256_storeu_si256((__m256i*)(ptr + 32), bgra2); - _mm256_storeu_si256((__m256i*)(ptr + 48), bgra3); - } -} - -inline void v_store_interleave( unsigned* ptr, const v_uint32x8& a, const v_uint32x8& b, - const v_uint32x8& c, const v_uint32x8& d, - hal::StoreMode mode=hal::STORE_UNALIGNED ) -{ - __m256i bg0 = _mm256_unpacklo_epi32(a.val, b.val); - __m256i bg1 = _mm256_unpackhi_epi32(a.val, b.val); - __m256i ra0 = _mm256_unpacklo_epi32(c.val, d.val); - __m256i ra1 = _mm256_unpackhi_epi32(c.val, d.val); - - __m256i bgra0_ = _mm256_unpacklo_epi64(bg0, ra0); - __m256i bgra1_ = _mm256_unpackhi_epi64(bg0, ra0); - __m256i bgra2_ = _mm256_unpacklo_epi64(bg1, ra1); - __m256i bgra3_ = _mm256_unpackhi_epi64(bg1, ra1); - - __m256i bgra0 = _mm256_permute2x128_si256(bgra0_, bgra1_, 0 + 2*16); - __m256i bgra2 = _mm256_permute2x128_si256(bgra0_, bgra1_, 1 + 3*16); - __m256i bgra1 = _mm256_permute2x128_si256(bgra2_, bgra3_, 0 + 2*16); - __m256i bgra3 = _mm256_permute2x128_si256(bgra2_, bgra3_, 1 + 3*16); - - if( mode == hal::STORE_ALIGNED_NOCACHE ) - { - _mm256_stream_si256((__m256i*)ptr, bgra0); - _mm256_stream_si256((__m256i*)(ptr + 8), bgra1); - _mm256_stream_si256((__m256i*)(ptr + 16), bgra2); - _mm256_stream_si256((__m256i*)(ptr + 24), bgra3); - } - else if( mode == hal::STORE_ALIGNED ) - { - _mm256_store_si256((__m256i*)ptr, bgra0); - _mm256_store_si256((__m256i*)(ptr + 8), bgra1); - _mm256_store_si256((__m256i*)(ptr + 16), bgra2); - _mm256_store_si256((__m256i*)(ptr + 24), bgra3); - } - else - { - _mm256_storeu_si256((__m256i*)ptr, bgra0); - _mm256_storeu_si256((__m256i*)(ptr + 8), bgra1); - _mm256_storeu_si256((__m256i*)(ptr + 16), bgra2); - _mm256_storeu_si256((__m256i*)(ptr + 24), bgra3); - } -} - -inline void v_store_interleave( uint64* ptr, const v_uint64x4& a, const v_uint64x4& b, - const v_uint64x4& c, const v_uint64x4& d, - hal::StoreMode mode=hal::STORE_UNALIGNED ) -{ - __m256i bg0 = _mm256_unpacklo_epi64(a.val, b.val); - __m256i bg1 = _mm256_unpackhi_epi64(a.val, b.val); - __m256i ra0 = _mm256_unpacklo_epi64(c.val, d.val); - __m256i ra1 = _mm256_unpackhi_epi64(c.val, d.val); - - __m256i bgra0 = _mm256_permute2x128_si256(bg0, ra0, 0 + 2*16); - __m256i bgra1 = _mm256_permute2x128_si256(bg1, ra1, 0 + 2*16); - __m256i bgra2 = _mm256_permute2x128_si256(bg0, ra0, 1 + 3*16); - __m256i bgra3 = _mm256_permute2x128_si256(bg1, ra1, 1 + 3*16); - - if( mode == hal::STORE_ALIGNED_NOCACHE ) - { - _mm256_stream_si256((__m256i*)ptr, bgra0); - _mm256_stream_si256((__m256i*)(ptr + 4), bgra1); - _mm256_stream_si256((__m256i*)(ptr + 8), bgra2); - _mm256_stream_si256((__m256i*)(ptr + 12), bgra3); - } - else if( mode == hal::STORE_ALIGNED ) - { - _mm256_store_si256((__m256i*)ptr, bgra0); - _mm256_store_si256((__m256i*)(ptr + 4), bgra1); - _mm256_store_si256((__m256i*)(ptr + 8), bgra2); - _mm256_store_si256((__m256i*)(ptr + 12), bgra3); - } - else - { - _mm256_storeu_si256((__m256i*)ptr, bgra0); - _mm256_storeu_si256((__m256i*)(ptr + 4), bgra1); - _mm256_storeu_si256((__m256i*)(ptr + 8), bgra2); - _mm256_storeu_si256((__m256i*)(ptr + 12), bgra3); - } -} - -#define OPENCV_HAL_IMPL_AVX_LOADSTORE_INTERLEAVE(_Tpvec0, _Tp0, suffix0, _Tpvec1, _Tp1, suffix1) \ -inline void v_load_deinterleave( const _Tp0* ptr, _Tpvec0& a0, _Tpvec0& b0 ) \ -{ \ - _Tpvec1 a1, b1; \ - v_load_deinterleave((const _Tp1*)ptr, a1, b1); \ - a0 = v_reinterpret_as_##suffix0(a1); \ - b0 = v_reinterpret_as_##suffix0(b1); \ -} \ -inline void v_load_deinterleave( const _Tp0* ptr, _Tpvec0& a0, _Tpvec0& b0, _Tpvec0& c0 ) \ -{ \ - _Tpvec1 a1, b1, c1; \ - v_load_deinterleave((const _Tp1*)ptr, a1, b1, c1); \ - a0 = v_reinterpret_as_##suffix0(a1); \ - b0 = v_reinterpret_as_##suffix0(b1); \ - c0 = v_reinterpret_as_##suffix0(c1); \ -} \ -inline void v_load_deinterleave( const _Tp0* ptr, _Tpvec0& a0, _Tpvec0& b0, _Tpvec0& c0, _Tpvec0& d0 ) \ -{ \ - _Tpvec1 a1, b1, c1, d1; \ - v_load_deinterleave((const _Tp1*)ptr, a1, b1, c1, d1); \ - a0 = v_reinterpret_as_##suffix0(a1); \ - b0 = v_reinterpret_as_##suffix0(b1); \ - c0 = v_reinterpret_as_##suffix0(c1); \ - d0 = v_reinterpret_as_##suffix0(d1); \ -} \ -inline void v_store_interleave( _Tp0* ptr, const _Tpvec0& a0, const _Tpvec0& b0, \ - hal::StoreMode mode=hal::STORE_UNALIGNED ) \ -{ \ - _Tpvec1 a1 = v_reinterpret_as_##suffix1(a0); \ - _Tpvec1 b1 = v_reinterpret_as_##suffix1(b0); \ - v_store_interleave((_Tp1*)ptr, a1, b1, mode); \ -} \ -inline void v_store_interleave( _Tp0* ptr, const _Tpvec0& a0, const _Tpvec0& b0, const _Tpvec0& c0, \ - hal::StoreMode mode=hal::STORE_UNALIGNED ) \ -{ \ - _Tpvec1 a1 = v_reinterpret_as_##suffix1(a0); \ - _Tpvec1 b1 = v_reinterpret_as_##suffix1(b0); \ - _Tpvec1 c1 = v_reinterpret_as_##suffix1(c0); \ - v_store_interleave((_Tp1*)ptr, a1, b1, c1, mode); \ -} \ -inline void v_store_interleave( _Tp0* ptr, const _Tpvec0& a0, const _Tpvec0& b0, \ - const _Tpvec0& c0, const _Tpvec0& d0, \ - hal::StoreMode mode=hal::STORE_UNALIGNED ) \ -{ \ - _Tpvec1 a1 = v_reinterpret_as_##suffix1(a0); \ - _Tpvec1 b1 = v_reinterpret_as_##suffix1(b0); \ - _Tpvec1 c1 = v_reinterpret_as_##suffix1(c0); \ - _Tpvec1 d1 = v_reinterpret_as_##suffix1(d0); \ - v_store_interleave((_Tp1*)ptr, a1, b1, c1, d1, mode); \ -} - -OPENCV_HAL_IMPL_AVX_LOADSTORE_INTERLEAVE(v_int8x32, schar, s8, v_uint8x32, uchar, u8) -OPENCV_HAL_IMPL_AVX_LOADSTORE_INTERLEAVE(v_int16x16, short, s16, v_uint16x16, ushort, u16) -OPENCV_HAL_IMPL_AVX_LOADSTORE_INTERLEAVE(v_int32x8, int, s32, v_uint32x8, unsigned, u32) -OPENCV_HAL_IMPL_AVX_LOADSTORE_INTERLEAVE(v_float32x8, float, f32, v_uint32x8, unsigned, u32) -OPENCV_HAL_IMPL_AVX_LOADSTORE_INTERLEAVE(v_int64x4, int64, s64, v_uint64x4, uint64, u64) -OPENCV_HAL_IMPL_AVX_LOADSTORE_INTERLEAVE(v_float64x4, double, f64, v_uint64x4, uint64, u64) - -////////////// FP16 support /////////////////////////// -// Currently disabled (DM) -#if 0 -inline v_float32x8 v256_load_expand(const float16_t* ptr) -{ - return v_float32x8(_mm256_cvtph_ps(_mm_loadu_si128((const __m128i*)ptr))); -} - -inline void v_pack_store(float16_t* ptr, const v_float32x8& a) -{ - __m128i ah = _mm256_cvtps_ph(a.val, 0); - _mm_storeu_si128((__m128i*)ptr, ah); -} -#endif -inline void v256_cleanup() { _mm256_zeroall(); } - -static inline void v_deinterleave(const v_float32x8& low, const v_float32x8& high, - v_float32x8& even, v_float32x8& odd) -{ - __m256 tmp0 = _mm256_unpacklo_ps(low.val, high.val); - __m256 tmp1 = _mm256_unpackhi_ps(low.val, high.val); - __m256 tmp2 = _mm256_unpacklo_ps(tmp0, tmp1); - __m256 tmp3 = _mm256_unpackhi_ps(tmp0, tmp1); - even.val = _mm256_castsi256_ps(_mm256_permute4x64_epi64(_mm256_castps_si256(tmp2), 216 /*11011000*/)); - odd.val = _mm256_castsi256_ps(_mm256_permute4x64_epi64(_mm256_castps_si256(tmp3), 216 /*11011000*/)); -} - -static inline void v_deinterleave(const v_uint8x32& v0, const v_uint8x32& v1, - const v_uint8x32& v2, const v_uint8x32& v3, - v_uint8x32& a, v_uint8x32& b, - v_uint8x32& c, v_uint8x32& d, - v_uint8x32& shuf_mask) -{ - /* a0a1a2a3 b0b1b2b3 c0c1c2c3 d0d1d2d3 a16a17a18a19 b16b17b18b19 c16c17c18c19 d16d17d18d19 */ - __m256i u0 = _mm256_shuffle_epi8(v0.val, shuf_mask.val); - /* a4a5a6a7 b4b5b6b7 c4c5c6c7 d4d5d6d7 a20a21a22a23 b20b21b22b23 c20c21c22c23 d20d21d22d23 */ - __m256i u1 = _mm256_shuffle_epi8(v1.val, shuf_mask.val); - /* a8a9a10a11 b8b9b10b11 c8c9c10c11 d8d9d10d11 */ - __m256i u2 = _mm256_shuffle_epi8(v2.val, shuf_mask.val); - __m256i u3 = _mm256_shuffle_epi8(v3.val, shuf_mask.val); - - __m256i s0 = _mm256_blend_epi16(u0, _mm256_slli_si256(u1, 4), 0xCC /*0b11001100*/); // a0a1a2a3a4a5a6a7 c0c1c2c3c4c5c6c7 a16a17a18a19a29a21a22a23 ... - __m256i s1 = _mm256_blend_epi16(_mm256_srli_si256(u0, 4), u1, 0xCC /*0b11001100*/); - __m256i s2 = _mm256_blend_epi16(u2, _mm256_slli_si256(u3, 4), 0xCC /*0b11001100*/); - __m256i s3 = _mm256_blend_epi16(_mm256_srli_si256(u2, 4), u3, 0xCC /*0b11001100*/); - - a.val = _mm256_blend_epi16(s0, _mm256_slli_si256(s2, 8), 0xF0 /*0b11110000*/); - c.val = _mm256_blend_epi16(_mm256_srli_si256(s0, 8), s2, 0xF0 /*0b11110000*/); - b.val = _mm256_blend_epi16(s1, _mm256_slli_si256(s3, 8), 0xF0 /*0b11110000*/); - d.val = _mm256_blend_epi16(_mm256_srli_si256(s1, 8), s3, 0xF0 /*0b11110000*/); - } - -static inline v_uint8x32 v_interleave_low(const v_uint8x32& a, const v_uint8x32& b) -{ - return v_uint8x32(_mm256_unpacklo_epi8(a.val, b.val)); -} - -static inline v_uint8x32 v_interleave_high(const v_uint8x32& a, const v_uint8x32& b) -{ - return v_uint8x32(_mm256_unpackhi_epi8(a.val, b.val)); -} - -static inline v_int16x16 v_interleave_low(const v_int16x16& a, const v_int16x16& b) -{ - return v_int16x16(_mm256_unpacklo_epi16(a.val, b.val)); -} - -static inline v_int16x16 v_interleave_high(const v_int16x16& a, const v_int16x16& b) -{ - return v_int16x16(_mm256_unpackhi_epi16(a.val, b.val)); -} - -static inline v_uint8x32 v_saturate_u8(const v_int16x16& a) -{ - v_uint8x32 r; - r.val = _mm256_packus_epi16(a.val, _mm256_setzero_si256()); - return r; -} - -static inline v_int16x16 v_saturate_s16(const v_int32x8& a) -{ - v_int16x16 r; - r.val = _mm256_packs_epi32(a.val, _mm256_setzero_si256()); - return r; -} - -static inline v_int32x8 v_madd(const v_int16x16& a, const v_int16x16& b) -{ - v_int32x8 r; - r.val = _mm256_madd_epi16(a.val, b.val); - return r; -} - -static inline void v_deinterleave_expand(const v_uint8x32& src, v_int16x16& even, v_int16x16& odd) -{ - static const __m256i mask_even = - _mm256_setr_epi8(0, -1, 2, -1, 4, -1, 6, -1, 8, -1, 10, - -1, 12, -1, 14, -1, 16, -1, 18, -1, 20, - -1, 22, -1, 24, -1, 26, -1, 28, -1, 30, -1); - static const __m256i mask_odd = - _mm256_setr_epi8(1, -1, 3, -1, 5, -1, 7, -1, 9, -1, 11, - -1, 13, -1, 15, -1, 17, -1, 19, -1, 21, - -1, 23, -1, 25, -1, 27, -1, 29, -1, 31, -1); - - even.val = _mm256_shuffle_epi8(src.val, mask_even); - odd .val = _mm256_shuffle_epi8(src.val, mask_odd); -} - -static inline v_int16x16 v_mulhi(const v_int16x16& a, short b) -{ - return v_int16x16(_mm256_mulhi_epi16(a.val, _mm256_set1_epi16(b))); -} - -static inline v_uint16x16 v_mulhi(const v_uint16x16& a, v_uint16x16& b) -{ - return v_uint16x16(_mm256_mulhi_epu16(a.val, b.val)); -} - -static inline v_uint16x16 v_mulhi(const v_uint16x16& a, uint16_t b) -{ - return v_uint16x16(_mm256_mulhi_epu16(a.val, _mm256_set1_epi16(b))); -} - -static inline v_int16x16 v_mulhrs(const v_int16x16& a, const v_int16x16& b) -{ - return v_int16x16(_mm256_mulhrs_epi16(a.val, b.val)); -} - -static inline v_int16x16 v_mulhrs(const v_int16x16& a, short b) -{ - return v_mulhrs(a, v256_setall_s16(b)); -} - -static inline v_float32x8 v_fma(const v_float32x8& a, float b, const v_float32x8& c) -{ - return v_fma(a, v256_setall_f32(b), c); -} - -static inline v_int16x16 operator+ (const v_int16x16& a, short b) -{ - return a + v256_setall_s16(b); -} - -static inline v_int16x16 operator- (short a, const v_int16x16& b) -{ - return v256_setall_s16(a) - b; -} - -static inline v_float32x8 operator- (float a, const v_float32x8& b) -{ - return v256_setall_f32(a) - b; -} - -static inline v_float32x8 operator* (const v_float32x8& a, float b) -{ - return a * v256_setall_f32(b); -} - -static inline v_uint8x32 v_shuffle_s8(const v_uint8x32& a, const v_uint8x32& mask) -{ - return v_uint8x32(_mm256_shuffle_epi8(a.val, mask.val)); -} - -#if !defined(__GNUC__) || defined(__GNUC__) && defined(__x86_64) -template -static inline __m256i v_insert64(v_uint8x32& a, const int64_t& i) -{ - return _mm256_insert_epi64(a.val, i, index); -} -#endif - -static inline void v_setr64(v_uint8x32& val_0, v_uint8x32& val_1,v_uint8x32& val_2, v_uint8x32& val_3, const short mapsx[], - uint8_t tmp[], const int& x, const int& shift) { - val_0.val = _mm256_setr_epi64x(*reinterpret_cast(&tmp[4 * mapsx[x + 0]]), - *reinterpret_cast(&tmp[4 * mapsx[x + 1]]), - *reinterpret_cast(&tmp[4 * mapsx[x + 2]]), - *reinterpret_cast(&tmp[4 * mapsx[x + 3]])); - - val_1.val = _mm256_setr_epi64x(*reinterpret_cast(&tmp[4 * mapsx[x + shift + 0]]), - *reinterpret_cast(&tmp[4 * mapsx[x + shift + 1]]), - *reinterpret_cast(&tmp[4 * mapsx[x + shift + 2]]), - *reinterpret_cast(&tmp[4 * mapsx[x + shift + 3]])); - - val_2.val = _mm256_setr_epi64x(*reinterpret_cast(&tmp[4 * mapsx[x + 2*shift + 0]]), - *reinterpret_cast(&tmp[4 * mapsx[x + 2*shift + 1]]), - *reinterpret_cast(&tmp[4 * mapsx[x + 2*shift + 2]]), - *reinterpret_cast(&tmp[4 * mapsx[x + 2*shift + 3]])); - - val_3.val = _mm256_setr_epi64x(*reinterpret_cast(&tmp[4 * mapsx[x + 3 * shift + 0]]), - *reinterpret_cast(&tmp[4 * mapsx[x + 3 * shift + 1]]), - *reinterpret_cast(&tmp[4 * mapsx[x + 3 * shift + 2]]), - *reinterpret_cast(&tmp[4 * mapsx[x + 3 * shift + 3]])); -} - -static inline v_uint8x32 v_permute32(v_uint8x32& a, v_uint32x8& idxs) -{ - return v_uint8x32(_mm256_permutevar8x32_epi32(a.val, idxs.val)); -} - -static inline void v_gather_channel(v_uint8x32& vec, const uint8_t tmp[], const short mapsx[], - int chanNum, int c, int x, int shift) -{ - vec.val = _mm256_insert_epi32(vec.val, *reinterpret_cast(&tmp[4 * (chanNum * mapsx[x + shift + 0] + c)]), 0); - vec.val = _mm256_insert_epi32(vec.val, *reinterpret_cast(&tmp[4 * (chanNum * mapsx[x + shift + 1] + c)]), 1); - vec.val = _mm256_insert_epi32(vec.val, *reinterpret_cast(&tmp[4 * (chanNum * mapsx[x + shift + 2] + c)]), 2); - vec.val = _mm256_insert_epi32(vec.val, *reinterpret_cast(&tmp[4 * (chanNum * mapsx[x + shift + 3] + c)]), 3); - - vec.val = _mm256_insert_epi32(vec.val, *reinterpret_cast(&tmp[4 * (chanNum * (mapsx[x + shift + 0] + 1) + c)]), 4); - vec.val = _mm256_insert_epi32(vec.val, *reinterpret_cast(&tmp[4 * (chanNum * (mapsx[x + shift + 1] + 1) + c)]), 5); - vec.val = _mm256_insert_epi32(vec.val, *reinterpret_cast(&tmp[4 * (chanNum * (mapsx[x + shift + 2] + 1) + c)]), 6); - vec.val = _mm256_insert_epi32(vec.val, *reinterpret_cast(&tmp[4 * (chanNum * (mapsx[x + shift + 3] + 1) + c)]), 7); -} - -// for each j=index[k], load two chars src[j] and src[j+1] -static inline v_uint8x32 v_gather_pairs(const uchar src[], const v_int16x16& index) { - v_uint8x32 r; - r.val = _mm256_insert_epi16(r.val, *reinterpret_cast(&src[_mm256_extract_epi16(index.val, 0)]), 0); - r.val = _mm256_insert_epi16(r.val, *reinterpret_cast(&src[_mm256_extract_epi16(index.val, 1)]), 1); - r.val = _mm256_insert_epi16(r.val, *reinterpret_cast(&src[_mm256_extract_epi16(index.val, 2)]), 2); - r.val = _mm256_insert_epi16(r.val, *reinterpret_cast(&src[_mm256_extract_epi16(index.val, 3)]), 3); - r.val = _mm256_insert_epi16(r.val, *reinterpret_cast(&src[_mm256_extract_epi16(index.val, 4)]), 4); - r.val = _mm256_insert_epi16(r.val, *reinterpret_cast(&src[_mm256_extract_epi16(index.val, 5)]), 5); - r.val = _mm256_insert_epi16(r.val, *reinterpret_cast(&src[_mm256_extract_epi16(index.val, 6)]), 6); - r.val = _mm256_insert_epi16(r.val, *reinterpret_cast(&src[_mm256_extract_epi16(index.val, 7)]), 7); - r.val = _mm256_insert_epi16(r.val, *reinterpret_cast(&src[_mm256_extract_epi16(index.val, 8)]), 8); - r.val = _mm256_insert_epi16(r.val, *reinterpret_cast(&src[_mm256_extract_epi16(index.val, 9)]), 9); - r.val = _mm256_insert_epi16(r.val, *reinterpret_cast(&src[_mm256_extract_epi16(index.val, 10)]), 10); - r.val = _mm256_insert_epi16(r.val, *reinterpret_cast(&src[_mm256_extract_epi16(index.val, 11)]), 11); - r.val = _mm256_insert_epi16(r.val, *reinterpret_cast(&src[_mm256_extract_epi16(index.val, 12)]), 12); - r.val = _mm256_insert_epi16(r.val, *reinterpret_cast(&src[_mm256_extract_epi16(index.val, 13)]), 13); - r.val = _mm256_insert_epi16(r.val, *reinterpret_cast(&src[_mm256_extract_epi16(index.val, 14)]), 14); - r.val = _mm256_insert_epi16(r.val, *reinterpret_cast(&src[_mm256_extract_epi16(index.val, 15)]), 15); - - return r; -} - -static inline void v_gather_pairs(const float src[], const int mapsx[], int x, - v_float32x8& low, v_float32x8& high) { - low.val = _mm256_castsi256_ps(_mm256_setr_epi64x(*reinterpret_cast(&src[mapsx[x + 0]]), - *reinterpret_cast(&src[mapsx[x + 1]]), - *reinterpret_cast(&src[mapsx[x + 2]]), - *reinterpret_cast(&src[mapsx[x + 3]]))); - high.val = _mm256_castsi256_ps(_mm256_setr_epi64x(*reinterpret_cast(&src[mapsx[x + 4]]), - *reinterpret_cast(&src[mapsx[x + 5]]), - *reinterpret_cast(&src[mapsx[x + 6]]), - *reinterpret_cast(&src[mapsx[x + 7]]))); -} - -namespace { - template - static inline v_int16x16 v_gather_chan(const uchar src[], const v_int16x16& index, int channel, int pos) { - v_int16x16 r; - r.val = _mm256_insert_epi16(r.val, *reinterpret_cast(&src[chanNum*(_mm256_extract_epi16(index.val, 0) + pos) + channel]), 0); - r.val = _mm256_insert_epi16(r.val, *reinterpret_cast(&src[chanNum*(_mm256_extract_epi16(index.val, 1) + pos) + channel]), 1); - r.val = _mm256_insert_epi16(r.val, *reinterpret_cast(&src[chanNum*(_mm256_extract_epi16(index.val, 2) + pos) + channel]), 2); - r.val = _mm256_insert_epi16(r.val, *reinterpret_cast(&src[chanNum*(_mm256_extract_epi16(index.val, 3) + pos) + channel]), 3); - r.val = _mm256_insert_epi16(r.val, *reinterpret_cast(&src[chanNum*(_mm256_extract_epi16(index.val, 4) + pos) + channel]), 4); - r.val = _mm256_insert_epi16(r.val, *reinterpret_cast(&src[chanNum*(_mm256_extract_epi16(index.val, 5) + pos) + channel]), 5); - r.val = _mm256_insert_epi16(r.val, *reinterpret_cast(&src[chanNum*(_mm256_extract_epi16(index.val, 6) + pos) + channel]), 6); - r.val = _mm256_insert_epi16(r.val, *reinterpret_cast(&src[chanNum*(_mm256_extract_epi16(index.val, 7) + pos) + channel]), 7); - r.val = _mm256_insert_epi16(r.val, *reinterpret_cast(&src[chanNum*(_mm256_extract_epi16(index.val, 8) + pos) + channel]), 8); - r.val = _mm256_insert_epi16(r.val, *reinterpret_cast(&src[chanNum*(_mm256_extract_epi16(index.val, 9) + pos) + channel]), 9); - r.val = _mm256_insert_epi16(r.val, *reinterpret_cast(&src[chanNum*(_mm256_extract_epi16(index.val, 10) + pos) + channel]), 10); - r.val = _mm256_insert_epi16(r.val, *reinterpret_cast(&src[chanNum*(_mm256_extract_epi16(index.val, 11) + pos) + channel]), 11); - r.val = _mm256_insert_epi16(r.val, *reinterpret_cast(&src[chanNum*(_mm256_extract_epi16(index.val, 12) + pos) + channel]), 12); - r.val = _mm256_insert_epi16(r.val, *reinterpret_cast(&src[chanNum*(_mm256_extract_epi16(index.val, 13) + pos) + channel]), 13); - r.val = _mm256_insert_epi16(r.val, *reinterpret_cast(&src[chanNum*(_mm256_extract_epi16(index.val, 14) + pos) + channel]), 14); - r.val = _mm256_insert_epi16(r.val, *reinterpret_cast(&src[chanNum*(_mm256_extract_epi16(index.val, 15) + pos) + channel]), 15); - return r; - } -} // namespace - -CV_CPU_OPTIMIZATION_HAL_NAMESPACE_END - -//! @endcond - -} // cv:: - -#endif // OPENCV_HAL_INTRIN_AVX_HPP diff --git a/thirdparty/ocv/opencv_hal_avx512.hpp b/thirdparty/ocv/opencv_hal_avx512.hpp deleted file mode 100644 index bf9f7a3664b..00000000000 --- a/thirdparty/ocv/opencv_hal_avx512.hpp +++ /dev/null @@ -1,3397 +0,0 @@ -// This file is part of OpenCV project. -// It is subject to the license terms in the LICENSE file found in the top-level directory -// of this distribution and at http://opencv.org/license.html - -#ifndef OPENCV_HAL_INTRIN_AVX512_HPP -#define OPENCV_HAL_INTRIN_AVX512_HPP - -#if defined(_MSC_VER) && (_MSC_VER < 1920/*MSVS2019*/) -# pragma warning(disable:4146) // unary minus operator applied to unsigned type, result still unsigned -# pragma warning(disable:4309) // 'argument': truncation of constant value -# pragma warning(disable:4310) // cast truncates constant value -#endif - -#define CVT_ROUND_MODES_IMPLEMENTED 0 - -#define CV_SIMD512 1 -#define CV_SIMD512_64F 1 -#define CV_SIMD512_FP16 0 // no native operations with FP16 type. Only load/store from float32x8 are available (if CV_FP16 == 1) - -#define _v512_set_epu64(a7, a6, a5, a4, a3, a2, a1, a0) _mm512_set_epi64((int64)(a7),(int64)(a6),(int64)(a5),(int64)(a4),(int64)(a3),(int64)(a2),(int64)(a1),(int64)(a0)) -#define _v512_set_epu32(a15, a14, a13, a12, a11, a10, a9, a8, a7, a6, a5, a4, a3, a2, a1, a0) \ - _mm512_set_epi64(((int64)(a15)<<32)|(int64)(a14), ((int64)(a13)<<32)|(int64)(a12), ((int64)(a11)<<32)|(int64)(a10), ((int64)( a9)<<32)|(int64)( a8), \ - ((int64)( a7)<<32)|(int64)( a6), ((int64)( a5)<<32)|(int64)( a4), ((int64)( a3)<<32)|(int64)( a2), ((int64)( a1)<<32)|(int64)( a0)) -#define _v512_set_epu16(a31, a30, a29, a28, a27, a26, a25, a24, a23, a22, a21, a20, a19, a18, a17, a16, \ - a15, a14, a13, a12, a11, a10, a9, a8, a7, a6, a5, a4, a3, a2, a1, a0) \ - _v512_set_epu32(((unsigned)(a31)<<16)|(unsigned)(a30), ((unsigned)(a29)<<16)|(unsigned)(a28), ((unsigned)(a27)<<16)|(unsigned)(a26), ((unsigned)(a25)<<16)|(unsigned)(a24), \ - ((unsigned)(a23)<<16)|(unsigned)(a22), ((unsigned)(a21)<<16)|(unsigned)(a20), ((unsigned)(a19)<<16)|(unsigned)(a18), ((unsigned)(a17)<<16)|(unsigned)(a16), \ - ((unsigned)(a15)<<16)|(unsigned)(a14), ((unsigned)(a13)<<16)|(unsigned)(a12), ((unsigned)(a11)<<16)|(unsigned)(a10), ((unsigned)( a9)<<16)|(unsigned)( a8), \ - ((unsigned)( a7)<<16)|(unsigned)( a6), ((unsigned)( a5)<<16)|(unsigned)( a4), ((unsigned)( a3)<<16)|(unsigned)( a2), ((unsigned)( a1)<<16)|(unsigned)( a0)) -#define _v512_set_epu8(a63, a62, a61, a60, a59, a58, a57, a56, a55, a54, a53, a52, a51, a50, a49, a48, \ - a47, a46, a45, a44, a43, a42, a41, a40, a39, a38, a37, a36, a35, a34, a33, a32, \ - a31, a30, a29, a28, a27, a26, a25, a24, a23, a22, a21, a20, a19, a18, a17, a16, \ - a15, a14, a13, a12, a11, a10, a9, a8, a7, a6, a5, a4, a3, a2, a1, a0) \ - _v512_set_epu32(((unsigned)(a63)<<24)|((unsigned)(a62)<<16)|((unsigned)(a61)<<8)|(unsigned)(a60),((unsigned)(a59)<<24)|((unsigned)(a58)<<16)|((unsigned)(a57)<<8)|(unsigned)(a56), \ - ((unsigned)(a55)<<24)|((unsigned)(a54)<<16)|((unsigned)(a53)<<8)|(unsigned)(a52),((unsigned)(a51)<<24)|((unsigned)(a50)<<16)|((unsigned)(a49)<<8)|(unsigned)(a48), \ - ((unsigned)(a47)<<24)|((unsigned)(a46)<<16)|((unsigned)(a45)<<8)|(unsigned)(a44),((unsigned)(a43)<<24)|((unsigned)(a42)<<16)|((unsigned)(a41)<<8)|(unsigned)(a40), \ - ((unsigned)(a39)<<24)|((unsigned)(a38)<<16)|((unsigned)(a37)<<8)|(unsigned)(a36),((unsigned)(a35)<<24)|((unsigned)(a34)<<16)|((unsigned)(a33)<<8)|(unsigned)(a32), \ - ((unsigned)(a31)<<24)|((unsigned)(a30)<<16)|((unsigned)(a29)<<8)|(unsigned)(a28),((unsigned)(a27)<<24)|((unsigned)(a26)<<16)|((unsigned)(a25)<<8)|(unsigned)(a24), \ - ((unsigned)(a23)<<24)|((unsigned)(a22)<<16)|((unsigned)(a21)<<8)|(unsigned)(a20),((unsigned)(a19)<<24)|((unsigned)(a18)<<16)|((unsigned)(a17)<<8)|(unsigned)(a16), \ - ((unsigned)(a15)<<24)|((unsigned)(a14)<<16)|((unsigned)(a13)<<8)|(unsigned)(a12),((unsigned)(a11)<<24)|((unsigned)(a10)<<16)|((unsigned)( a9)<<8)|(unsigned)( a8), \ - ((unsigned)( a7)<<24)|((unsigned)( a6)<<16)|((unsigned)( a5)<<8)|(unsigned)( a4),((unsigned)( a3)<<24)|((unsigned)( a2)<<16)|((unsigned)( a1)<<8)|(unsigned)( a0)) -#define _v512_set_epi8(a63, a62, a61, a60, a59, a58, a57, a56, a55, a54, a53, a52, a51, a50, a49, a48, \ - a47, a46, a45, a44, a43, a42, a41, a40, a39, a38, a37, a36, a35, a34, a33, a32, \ - a31, a30, a29, a28, a27, a26, a25, a24, a23, a22, a21, a20, a19, a18, a17, a16, \ - a15, a14, a13, a12, a11, a10, a9, a8, a7, a6, a5, a4, a3, a2, a1, a0) \ - _v512_set_epu8((uchar)(a63), (uchar)(a62), (uchar)(a61), (uchar)(a60), (uchar)(a59), (uchar)(a58), (uchar)(a57), (uchar)(a56), \ - (uchar)(a55), (uchar)(a54), (uchar)(a53), (uchar)(a52), (uchar)(a51), (uchar)(a50), (uchar)(a49), (uchar)(a48), \ - (uchar)(a47), (uchar)(a46), (uchar)(a45), (uchar)(a44), (uchar)(a43), (uchar)(a42), (uchar)(a41), (uchar)(a40), \ - (uchar)(a39), (uchar)(a38), (uchar)(a37), (uchar)(a36), (uchar)(a35), (uchar)(a34), (uchar)(a33), (uchar)(a32), \ - (uchar)(a31), (uchar)(a30), (uchar)(a29), (uchar)(a28), (uchar)(a27), (uchar)(a26), (uchar)(a25), (uchar)(a24), \ - (uchar)(a23), (uchar)(a22), (uchar)(a21), (uchar)(a20), (uchar)(a19), (uchar)(a18), (uchar)(a17), (uchar)(a16), \ - (uchar)(a15), (uchar)(a14), (uchar)(a13), (uchar)(a12), (uchar)(a11), (uchar)(a10), (uchar)( a9), (uchar)( a8), \ - (uchar)( a7), (uchar)( a6), (uchar)( a5), (uchar)( a4), (uchar)( a3), (uchar)( a2), (uchar)( a1), (uchar)( a0)) - -#ifndef _mm512_cvtpd_pslo -#ifdef _mm512_zextsi256_si512 -#define _mm512_cvtpd_pslo(a) _mm512_zextps256_ps512(_mm512_cvtpd_ps(a)) -#else -//if preferred way to extend with zeros is unavailable -#define _mm512_cvtpd_pslo(a) _mm512_castps256_ps512(_mm512_cvtpd_ps(a)) -#endif -#endif - -#include - -using namespace std; - -///////// Utils //////////// - -namespace cv -{ - -inline __m512i _v512_combine(const __m256i& lo, const __m256i& hi) -{ return _mm512_inserti32x8(_mm512_castsi256_si512(lo), hi, 1); } - -inline __m512 _v512_combine(const __m256& lo, const __m256& hi) -{ return _mm512_insertf32x8(_mm512_castps256_ps512(lo), hi, 1); } - -inline __m512d _v512_combine(const __m256d& lo, const __m256d& hi) -{ return _mm512_insertf64x4(_mm512_castpd256_pd512(lo), hi, 1); } - -inline int _v_cvtsi512_si32(const __m512i& a) -{ return _mm_cvtsi128_si32(_mm512_castsi512_si128(a)); } - -inline __m256i _v512_extract_high(const __m512i& v) -{ return _mm512_extracti32x8_epi32(v, 1); } - -inline __m256 _v512_extract_high(const __m512& v) -{ return _mm512_extractf32x8_ps(v, 1); } - -inline __m256d _v512_extract_high(const __m512d& v) -{ return _mm512_mask_extractf64x4_pd(_mm256_setzero_pd(), (__mmask8) -1, v, 1); } - -inline __m256i _v512_extract_low(const __m512i& v) -{ return _mm512_castsi512_si256(v); } - -inline __m256 _v512_extract_low(const __m512& v) -{ return _mm512_castps512_ps256(v); } - -inline __m256d _v512_extract_low(const __m512d& v) -{ return _mm512_castpd512_pd256(v); } - -inline __m512i _v512_insert(const __m512i& a, const __m256i& b) -{ return _mm512_inserti32x8(a, b, 0); } - -inline __m512 _v512_insert(const __m512& a, const __m256& b) -{ return _mm512_insertf32x8(a, b, 0); } - -inline __m512d _v512_insert(const __m512d& a, const __m256d& b) -{ return _mm512_insertf64x4(a, b, 0); } - -} - -namespace cv -{ - -//! @cond IGNORED - -CV_CPU_OPTIMIZATION_HAL_NAMESPACE_BEGIN - -///////// Types //////////// - -struct v_uint8x64 -{ - typedef uchar lane_type; - enum { nlanes = 64 }; - __m512i val; - - explicit v_uint8x64(__m512i v) : val(v) {} - v_uint8x64(uchar v0, uchar v1, uchar v2, uchar v3, - uchar v4, uchar v5, uchar v6, uchar v7, - uchar v8, uchar v9, uchar v10, uchar v11, - uchar v12, uchar v13, uchar v14, uchar v15, - uchar v16, uchar v17, uchar v18, uchar v19, - uchar v20, uchar v21, uchar v22, uchar v23, - uchar v24, uchar v25, uchar v26, uchar v27, - uchar v28, uchar v29, uchar v30, uchar v31, - uchar v32, uchar v33, uchar v34, uchar v35, - uchar v36, uchar v37, uchar v38, uchar v39, - uchar v40, uchar v41, uchar v42, uchar v43, - uchar v44, uchar v45, uchar v46, uchar v47, - uchar v48, uchar v49, uchar v50, uchar v51, - uchar v52, uchar v53, uchar v54, uchar v55, - uchar v56, uchar v57, uchar v58, uchar v59, - uchar v60, uchar v61, uchar v62, uchar v63) - { - val = _v512_set_epu8(v63, v62, v61, v60, v59, v58, v57, v56, v55, v54, v53, v52, v51, v50, v49, v48, - v47, v46, v45, v44, v43, v42, v41, v40, v39, v38, v37, v36, v35, v34, v33, v32, - v31, v30, v29, v28, v27, v26, v25, v24, v23, v22, v21, v20, v19, v18, v17, v16, - v15, v14, v13, v12, v11, v10, v9, v8, v7, v6, v5, v4, v3, v2, v1, v0); - } - v_uint8x64() : val(_mm512_setzero_si512()) {} - uchar get0() const { return (uchar)_v_cvtsi512_si32(val); } -}; - -struct v_int8x64 -{ - typedef schar lane_type; - enum { nlanes = 64 }; - __m512i val; - - explicit v_int8x64(__m512i v) : val(v) {} - v_int8x64(schar v0, schar v1, schar v2, schar v3, - schar v4, schar v5, schar v6, schar v7, - schar v8, schar v9, schar v10, schar v11, - schar v12, schar v13, schar v14, schar v15, - schar v16, schar v17, schar v18, schar v19, - schar v20, schar v21, schar v22, schar v23, - schar v24, schar v25, schar v26, schar v27, - schar v28, schar v29, schar v30, schar v31, - schar v32, schar v33, schar v34, schar v35, - schar v36, schar v37, schar v38, schar v39, - schar v40, schar v41, schar v42, schar v43, - schar v44, schar v45, schar v46, schar v47, - schar v48, schar v49, schar v50, schar v51, - schar v52, schar v53, schar v54, schar v55, - schar v56, schar v57, schar v58, schar v59, - schar v60, schar v61, schar v62, schar v63) - { - val = _v512_set_epi8(v63, v62, v61, v60, v59, v58, v57, v56, v55, v54, v53, v52, v51, v50, v49, v48, - v47, v46, v45, v44, v43, v42, v41, v40, v39, v38, v37, v36, v35, v34, v33, v32, - v31, v30, v29, v28, v27, v26, v25, v24, v23, v22, v21, v20, v19, v18, v17, v16, - v15, v14, v13, v12, v11, v10, v9, v8, v7, v6, v5, v4, v3, v2, v1, v0); - } - v_int8x64() : val(_mm512_setzero_si512()) {} - schar get0() const { return (schar)_v_cvtsi512_si32(val); } -}; - -struct v_uint16x32 -{ - typedef ushort lane_type; - enum { nlanes = 32 }; - __m512i val; - - explicit v_uint16x32(__m512i v) : val(v) {} - v_uint16x32(ushort v0, ushort v1, ushort v2, ushort v3, - ushort v4, ushort v5, ushort v6, ushort v7, - ushort v8, ushort v9, ushort v10, ushort v11, - ushort v12, ushort v13, ushort v14, ushort v15, - ushort v16, ushort v17, ushort v18, ushort v19, - ushort v20, ushort v21, ushort v22, ushort v23, - ushort v24, ushort v25, ushort v26, ushort v27, - ushort v28, ushort v29, ushort v30, ushort v31) - { - val = _v512_set_epu16(v31, v30, v29, v28, v27, v26, v25, v24, v23, v22, v21, v20, v19, v18, v17, v16, - v15, v14, v13, v12, v11, v10, v9, v8, v7, v6, v5, v4, v3, v2, v1, v0); - } - v_uint16x32() : val(_mm512_setzero_si512()) {} - ushort get0() const { return (ushort)_v_cvtsi512_si32(val); } -}; - -struct v_int16x32 -{ - typedef short lane_type; - enum { nlanes = 32 }; - __m512i val; - - explicit v_int16x32(__m512i v) : val(v) {} - v_int16x32(short v0, short v1, short v2, short v3, short v4, short v5, short v6, short v7, - short v8, short v9, short v10, short v11, short v12, short v13, short v14, short v15, - short v16, short v17, short v18, short v19, short v20, short v21, short v22, short v23, - short v24, short v25, short v26, short v27, short v28, short v29, short v30, short v31) - { - val = _v512_set_epu16((ushort)v31, (ushort)v30, (ushort)v29, (ushort)v28, (ushort)v27, (ushort)v26, (ushort)v25, (ushort)v24, - (ushort)v23, (ushort)v22, (ushort)v21, (ushort)v20, (ushort)v19, (ushort)v18, (ushort)v17, (ushort)v16, - (ushort)v15, (ushort)v14, (ushort)v13, (ushort)v12, (ushort)v11, (ushort)v10, (ushort)v9 , (ushort)v8, - (ushort)v7 , (ushort)v6 , (ushort)v5 , (ushort)v4 , (ushort)v3 , (ushort)v2 , (ushort)v1 , (ushort)v0); - } - v_int16x32() : val(_mm512_setzero_si512()) {} - short get0() const { return (short)_v_cvtsi512_si32(val); } -}; - -struct v_uint32x16 -{ - typedef unsigned lane_type; - enum { nlanes = 16 }; - __m512i val; - - explicit v_uint32x16(__m512i v) : val(v) {} - v_uint32x16(unsigned v0, unsigned v1, unsigned v2, unsigned v3, - unsigned v4, unsigned v5, unsigned v6, unsigned v7, - unsigned v8, unsigned v9, unsigned v10, unsigned v11, - unsigned v12, unsigned v13, unsigned v14, unsigned v15) - { - val = _mm512_setr_epi32((int)v0, (int)v1, (int)v2, (int)v3, (int)v4, (int)v5, (int)v6, (int)v7, - (int)v8, (int)v9, (int)v10, (int)v11, (int)v12, (int)v13, (int)v14, (int)v15); - } - v_uint32x16() : val(_mm512_setzero_si512()) {} - unsigned get0() const { return (unsigned)_v_cvtsi512_si32(val); } -}; - -struct v_int32x16 -{ - typedef int lane_type; - enum { nlanes = 16 }; - __m512i val; - - explicit v_int32x16(__m512i v) : val(v) {} - v_int32x16(int v0, int v1, int v2, int v3, int v4, int v5, int v6, int v7, - int v8, int v9, int v10, int v11, int v12, int v13, int v14, int v15) - { - val = _mm512_setr_epi32(v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15); - } - v_int32x16() : val(_mm512_setzero_si512()) {} - int get0() const { return _v_cvtsi512_si32(val); } -}; - -struct v_float32x16 -{ - typedef float lane_type; - enum { nlanes = 16 }; - __m512 val; - - explicit v_float32x16(__m512 v) : val(v) {} - v_float32x16(float v0, float v1, float v2, float v3, float v4, float v5, float v6, float v7, - float v8, float v9, float v10, float v11, float v12, float v13, float v14, float v15) - { - val = _mm512_setr_ps(v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15); - } - v_float32x16() : val(_mm512_setzero_ps()) {} - float get0() const { return _mm_cvtss_f32(_mm512_castps512_ps128(val)); } -}; - -struct v_uint64x8 -{ - typedef uint64 lane_type; - enum { nlanes = 8 }; - __m512i val; - - explicit v_uint64x8(__m512i v) : val(v) {} - v_uint64x8(uint64 v0, uint64 v1, uint64 v2, uint64 v3, uint64 v4, uint64 v5, uint64 v6, uint64 v7) - { val = _mm512_setr_epi64((int64)v0, (int64)v1, (int64)v2, (int64)v3, (int64)v4, (int64)v5, (int64)v6, (int64)v7); } - v_uint64x8() : val(_mm512_setzero_si512()) {} - uint64 get0() const - { - #if defined __x86_64__ || defined _M_X64 - return (uint64)_mm_cvtsi128_si64(_mm512_castsi512_si128(val)); - #else - int a = _mm_cvtsi128_si32(_mm512_castsi512_si128(val)); - int b = _mm_cvtsi128_si32(_mm512_castsi512_si128(_mm512_srli_epi64(val, 32))); - return (unsigned)a | ((uint64)(unsigned)b << 32); - #endif - } -}; - -struct v_int64x8 -{ - typedef int64 lane_type; - enum { nlanes = 8 }; - __m512i val; - - explicit v_int64x8(__m512i v) : val(v) {} - v_int64x8(int64 v0, int64 v1, int64 v2, int64 v3, int64 v4, int64 v5, int64 v6, int64 v7) - { val = _mm512_setr_epi64(v0, v1, v2, v3, v4, v5, v6, v7); } - v_int64x8() : val(_mm512_setzero_si512()) {} - - int64 get0() const - { - #if defined __x86_64__ || defined _M_X64 - return (int64)_mm_cvtsi128_si64(_mm512_castsi512_si128(val)); - #else - int a = _mm_cvtsi128_si32(_mm512_castsi512_si128(val)); - int b = _mm_cvtsi128_si32(_mm512_castsi512_si128(_mm512_srli_epi64(val, 32))); - return (int64)((unsigned)a | ((uint64)(unsigned)b << 32)); - #endif - } -}; - -struct v_float64x8 -{ - typedef double lane_type; - enum { nlanes = 8 }; - __m512d val; - - explicit v_float64x8(__m512d v) : val(v) {} - v_float64x8(double v0, double v1, double v2, double v3, double v4, double v5, double v6, double v7) - { val = _mm512_setr_pd(v0, v1, v2, v3, v4, v5, v6, v7); } - v_float64x8() : val(_mm512_setzero_pd()) {} - double get0() const { return _mm_cvtsd_f64(_mm512_castpd512_pd128(val)); } -}; - -//////////////// Load and store operations /////////////// - -#define OPENCV_HAL_IMPL_AVX512_LOADSTORE(_Tpvec, _Tp) \ - inline _Tpvec v512_load(const _Tp* ptr) \ - { return _Tpvec(_mm512_loadu_si512((const __m512i*)ptr)); } \ - inline _Tpvec v512_load_aligned(const _Tp* ptr) \ - { return _Tpvec(_mm512_load_si512((const __m512i*)ptr)); } \ - inline _Tpvec v512_load_low(const _Tp* ptr) \ - { \ - __m256i v256 = _mm256_loadu_si256((const __m256i*)ptr); \ - return _Tpvec(_mm512_castsi256_si512(v256)); \ - } \ - inline _Tpvec v512_load_halves(const _Tp* ptr0, const _Tp* ptr1) \ - { \ - __m256i vlo = _mm256_loadu_si256((const __m256i*)ptr0); \ - __m256i vhi = _mm256_loadu_si256((const __m256i*)ptr1); \ - return _Tpvec(_v512_combine(vlo, vhi)); \ - } \ - inline void v_store(_Tp* ptr, const _Tpvec& a) \ - { _mm512_storeu_si512((__m512i*)ptr, a.val); } \ - inline void v_store_aligned(_Tp* ptr, const _Tpvec& a) \ - { _mm512_store_si512((__m512i*)ptr, a.val); } \ - inline void v_store_aligned_nocache(_Tp* ptr, const _Tpvec& a) \ - { _mm512_stream_si512((__m512i*)ptr, a.val); } \ - inline void v_store(_Tp* ptr, const _Tpvec& a, hal::StoreMode mode) \ - { \ - if( mode == hal::STORE_UNALIGNED ) \ - _mm512_storeu_si512((__m512i*)ptr, a.val); \ - else if( mode == hal::STORE_ALIGNED_NOCACHE ) \ - _mm512_stream_si512((__m512i*)ptr, a.val); \ - else \ - _mm512_store_si512((__m512i*)ptr, a.val); \ - } \ - inline void v_store_low(_Tp* ptr, const _Tpvec& a) \ - { _mm256_storeu_si256((__m256i*)ptr, _v512_extract_low(a.val)); } \ - inline void v_store_high(_Tp* ptr, const _Tpvec& a) \ - { _mm256_storeu_si256((__m256i*)ptr, _v512_extract_high(a.val)); } - -OPENCV_HAL_IMPL_AVX512_LOADSTORE(v_uint8x64, uchar) -OPENCV_HAL_IMPL_AVX512_LOADSTORE(v_int8x64, schar) -OPENCV_HAL_IMPL_AVX512_LOADSTORE(v_uint16x32, ushort) -OPENCV_HAL_IMPL_AVX512_LOADSTORE(v_int16x32, short) -OPENCV_HAL_IMPL_AVX512_LOADSTORE(v_uint32x16, unsigned) -OPENCV_HAL_IMPL_AVX512_LOADSTORE(v_int32x16, int) -OPENCV_HAL_IMPL_AVX512_LOADSTORE(v_uint64x8, uint64) -OPENCV_HAL_IMPL_AVX512_LOADSTORE(v_int64x8, int64) - -#define OPENCV_HAL_IMPL_AVX512_LOADSTORE_FLT(_Tpvec, _Tp, suffix, halfreg) \ - inline _Tpvec v512_load(const _Tp* ptr) \ - { return _Tpvec(_mm512_loadu_##suffix(ptr)); } \ - inline _Tpvec v512_load_aligned(const _Tp* ptr) \ - { return _Tpvec(_mm512_load_##suffix(ptr)); } \ - inline _Tpvec v512_load_low(const _Tp* ptr) \ - { \ - return _Tpvec(_mm512_cast##suffix##256_##suffix##512 \ - (_mm256_loadu_##suffix(ptr))); \ - } \ - inline _Tpvec v512_load_halves(const _Tp* ptr0, const _Tp* ptr1) \ - { \ - halfreg vlo = _mm256_loadu_##suffix(ptr0); \ - halfreg vhi = _mm256_loadu_##suffix(ptr1); \ - return _Tpvec(_v512_combine(vlo, vhi)); \ - } \ - inline void v_store(_Tp* ptr, const _Tpvec& a) \ - { _mm512_storeu_##suffix(ptr, a.val); } \ - inline void v_store_aligned(_Tp* ptr, const _Tpvec& a) \ - { _mm512_store_##suffix(ptr, a.val); } \ - inline void v_store_aligned_nocache(_Tp* ptr, const _Tpvec& a) \ - { _mm512_stream_##suffix(ptr, a.val); } \ - inline void v_store(_Tp* ptr, const _Tpvec& a, hal::StoreMode mode) \ - { \ - if( mode == hal::STORE_UNALIGNED ) \ - _mm512_storeu_##suffix(ptr, a.val); \ - else if( mode == hal::STORE_ALIGNED_NOCACHE ) \ - _mm512_stream_##suffix(ptr, a.val); \ - else \ - _mm512_store_##suffix(ptr, a.val); \ - } \ - inline void v_store_low(_Tp* ptr, const _Tpvec& a) \ - { _mm256_storeu_##suffix(ptr, _v512_extract_low(a.val)); } \ - inline void v_store_high(_Tp* ptr, const _Tpvec& a) \ - { _mm256_storeu_##suffix(ptr, _v512_extract_high(a.val)); } - -OPENCV_HAL_IMPL_AVX512_LOADSTORE_FLT(v_float32x16, float, ps, __m256) -OPENCV_HAL_IMPL_AVX512_LOADSTORE_FLT(v_float64x8, double, pd, __m256d) - -#define OPENCV_HAL_IMPL_AVX512_CAST(_Tpvec, _Tpvecf, suffix, cast) \ - inline _Tpvec v_reinterpret_as_##suffix(const _Tpvecf& a) \ - { return _Tpvec(cast(a.val)); } - -#define OPENCV_HAL_IMPL_AVX512_INIT(_Tpvec, _Tp, suffix, ssuffix, ctype_s) \ - inline _Tpvec v512_setzero_##suffix() \ - { return _Tpvec(_mm512_setzero_si512()); } \ - inline _Tpvec v512_setall_##suffix(_Tp v) \ - { return _Tpvec(_mm512_set1_##ssuffix((ctype_s)v)); } \ - OPENCV_HAL_IMPL_AVX512_CAST(_Tpvec, v_uint8x64, suffix, OPENCV_HAL_NOP) \ - OPENCV_HAL_IMPL_AVX512_CAST(_Tpvec, v_int8x64, suffix, OPENCV_HAL_NOP) \ - OPENCV_HAL_IMPL_AVX512_CAST(_Tpvec, v_uint16x32, suffix, OPENCV_HAL_NOP) \ - OPENCV_HAL_IMPL_AVX512_CAST(_Tpvec, v_int16x32, suffix, OPENCV_HAL_NOP) \ - OPENCV_HAL_IMPL_AVX512_CAST(_Tpvec, v_uint32x16, suffix, OPENCV_HAL_NOP) \ - OPENCV_HAL_IMPL_AVX512_CAST(_Tpvec, v_int32x16, suffix, OPENCV_HAL_NOP) \ - OPENCV_HAL_IMPL_AVX512_CAST(_Tpvec, v_uint64x8, suffix, OPENCV_HAL_NOP) \ - OPENCV_HAL_IMPL_AVX512_CAST(_Tpvec, v_int64x8, suffix, OPENCV_HAL_NOP) \ - OPENCV_HAL_IMPL_AVX512_CAST(_Tpvec, v_float32x16, suffix, _mm512_castps_si512) \ - OPENCV_HAL_IMPL_AVX512_CAST(_Tpvec, v_float64x8, suffix, _mm512_castpd_si512) - -OPENCV_HAL_IMPL_AVX512_INIT(v_uint8x64, uchar, u8, epi8, char) -OPENCV_HAL_IMPL_AVX512_INIT(v_int8x64, schar, s8, epi8, char) -OPENCV_HAL_IMPL_AVX512_INIT(v_uint16x32, ushort, u16, epi16, short) -OPENCV_HAL_IMPL_AVX512_INIT(v_int16x32, short, s16, epi16, short) -OPENCV_HAL_IMPL_AVX512_INIT(v_uint32x16, unsigned, u32, epi32, int) -OPENCV_HAL_IMPL_AVX512_INIT(v_int32x16, int, s32, epi32, int) -OPENCV_HAL_IMPL_AVX512_INIT(v_uint64x8, uint64, u64, epi64, int64) -OPENCV_HAL_IMPL_AVX512_INIT(v_int64x8, int64, s64, epi64, int64) - -#define OPENCV_HAL_IMPL_AVX512_INIT_FLT(_Tpvec, _Tp, suffix, zsuffix, cast) \ - inline _Tpvec v512_setzero_##suffix() \ - { return _Tpvec(_mm512_setzero_##zsuffix()); } \ - inline _Tpvec v512_setall_##suffix(_Tp v) \ - { return _Tpvec(_mm512_set1_##zsuffix(v)); } \ - OPENCV_HAL_IMPL_AVX512_CAST(_Tpvec, v_uint8x64, suffix, cast) \ - OPENCV_HAL_IMPL_AVX512_CAST(_Tpvec, v_int8x64, suffix, cast) \ - OPENCV_HAL_IMPL_AVX512_CAST(_Tpvec, v_uint16x32, suffix, cast) \ - OPENCV_HAL_IMPL_AVX512_CAST(_Tpvec, v_int16x32, suffix, cast) \ - OPENCV_HAL_IMPL_AVX512_CAST(_Tpvec, v_uint32x16, suffix, cast) \ - OPENCV_HAL_IMPL_AVX512_CAST(_Tpvec, v_int32x16, suffix, cast) \ - OPENCV_HAL_IMPL_AVX512_CAST(_Tpvec, v_uint64x8, suffix, cast) \ - OPENCV_HAL_IMPL_AVX512_CAST(_Tpvec, v_int64x8, suffix, cast) - -OPENCV_HAL_IMPL_AVX512_INIT_FLT(v_float32x16, float, f32, ps, _mm512_castsi512_ps) -OPENCV_HAL_IMPL_AVX512_INIT_FLT(v_float64x8, double, f64, pd, _mm512_castsi512_pd) - -inline v_float32x16 v_reinterpret_as_f32(const v_float32x16& a) -{ return a; } -inline v_float32x16 v_reinterpret_as_f32(const v_float64x8& a) -{ return v_float32x16(_mm512_castpd_ps(a.val)); } - -inline v_float64x8 v_reinterpret_as_f64(const v_float64x8& a) -{ return a; } -inline v_float64x8 v_reinterpret_as_f64(const v_float32x16& a) -{ return v_float64x8(_mm512_castps_pd(a.val)); } - -////////////// FP16 support /////////////////////////// -// Currently disabled (DM) -#if 0 -inline v_float32x16 v512_load_expand(const float16_t* ptr) -{ - return v_float32x16(_mm512_cvtph_ps(_mm256_loadu_si256((const __m256i*)ptr))); -} - -inline void v_pack_store(float16_t* ptr, const v_float32x16& a) -{ - __m256i ah = _mm512_cvtps_ph(a.val, 0); - _mm256_storeu_si256((__m256i*)ptr, ah); -} -#endif - -/* Recombine & ZIP */ -inline void v_zip(const v_int8x64& a, const v_int8x64& b, v_int8x64& ab0, v_int8x64& ab1) -{ -#if CV_AVX_512VBMI - __m512i mask0 = _v512_set_epu8( 95, 31, 94, 30, 93, 29, 92, 28, 91, 27, 90, 26, 89, 25, 88, 24, - 87, 23, 86, 22, 85, 21, 84, 20, 83, 19, 82, 18, 81, 17, 80, 16, - 79, 15, 78, 14, 77, 13, 76, 12, 75, 11, 74, 10, 73, 9, 72, 8, - 71, 7, 70, 6, 69, 5, 68, 4, 67, 3, 66, 2, 65, 1, 64, 0); - ab0 = v_int8x64(_mm512_permutex2var_epi8(a.val, mask0, b.val)); - __m512i mask1 = _v512_set_epu8(127, 63, 126, 62, 125, 61, 124, 60, 123, 59, 122, 58, 121, 57, 120, 56, - 119, 55, 118, 54, 117, 53, 116, 52, 115, 51, 114, 50, 113, 49, 112, 48, - 111, 47, 110, 46, 109, 45, 108, 44, 107, 43, 106, 42, 105, 41, 104, 40, - 103, 39, 102, 38, 101, 37, 100, 36, 99, 35, 98, 34, 97, 33, 96, 32); - ab1 = v_int8x64(_mm512_permutex2var_epi8(a.val, mask1, b.val)); -#else - __m512i low = _mm512_unpacklo_epi8(a.val, b.val); - __m512i high = _mm512_unpackhi_epi8(a.val, b.val); - ab0 = v_int8x64(_mm512_permutex2var_epi64(low, _v512_set_epu64(11, 10, 3, 2, 9, 8, 1, 0), high)); - ab1 = v_int8x64(_mm512_permutex2var_epi64(low, _v512_set_epu64(15, 14, 7, 6, 13, 12, 5, 4), high)); -#endif -} -inline void v_zip(const v_int16x32& a, const v_int16x32& b, v_int16x32& ab0, v_int16x32& ab1) -{ - __m512i mask0 = _v512_set_epu16(47, 15, 46, 14, 45, 13, 44, 12, 43, 11, 42, 10, 41, 9, 40, 8, - 39, 7, 38, 6, 37, 5, 36, 4, 35, 3, 34, 2, 33, 1, 32, 0); - ab0 = v_int16x32(_mm512_permutex2var_epi16(a.val, mask0, b.val)); - __m512i mask1 = _v512_set_epu16(63, 31, 62, 30, 61, 29, 60, 28, 59, 27, 58, 26, 57, 25, 56, 24, - 55, 23, 54, 22, 53, 21, 52, 20, 51, 19, 50, 18, 49, 17, 48, 16); - ab1 = v_int16x32(_mm512_permutex2var_epi16(a.val, mask1, b.val)); -} -inline void v_zip(const v_int32x16& a, const v_int32x16& b, v_int32x16& ab0, v_int32x16& ab1) -{ - __m512i mask0 = _v512_set_epu32(23, 7, 22, 6, 21, 5, 20, 4, 19, 3, 18, 2, 17, 1, 16, 0); - ab0 = v_int32x16(_mm512_permutex2var_epi32(a.val, mask0, b.val)); - __m512i mask1 = _v512_set_epu32(31, 15, 30, 14, 29, 13, 28, 12, 27, 11, 26, 10, 25, 9, 24, 8); - ab1 = v_int32x16(_mm512_permutex2var_epi32(a.val, mask1, b.val)); -} -inline void v_zip(const v_int64x8& a, const v_int64x8& b, v_int64x8& ab0, v_int64x8& ab1) -{ - __m512i mask0 = _v512_set_epu64(11, 3, 10, 2, 9, 1, 8, 0); - ab0 = v_int64x8(_mm512_permutex2var_epi64(a.val, mask0, b.val)); - __m512i mask1 = _v512_set_epu64(15, 7, 14, 6, 13, 5, 12, 4); - ab1 = v_int64x8(_mm512_permutex2var_epi64(a.val, mask1, b.val)); -} - -inline void v_zip(const v_uint8x64& a, const v_uint8x64& b, v_uint8x64& ab0, v_uint8x64& ab1) -{ - v_int8x64 i0, i1; - v_zip(v_reinterpret_as_s8(a), v_reinterpret_as_s8(b), i0, i1); - ab0 = v_reinterpret_as_u8(i0); - ab1 = v_reinterpret_as_u8(i1); -} -inline void v_zip(const v_uint16x32& a, const v_uint16x32& b, v_uint16x32& ab0, v_uint16x32& ab1) -{ - v_int16x32 i0, i1; - v_zip(v_reinterpret_as_s16(a), v_reinterpret_as_s16(b), i0, i1); - ab0 = v_reinterpret_as_u16(i0); - ab1 = v_reinterpret_as_u16(i1); -} -inline void v_zip(const v_uint32x16& a, const v_uint32x16& b, v_uint32x16& ab0, v_uint32x16& ab1) -{ - v_int32x16 i0, i1; - v_zip(v_reinterpret_as_s32(a), v_reinterpret_as_s32(b), i0, i1); - ab0 = v_reinterpret_as_u32(i0); - ab1 = v_reinterpret_as_u32(i1); -} -inline void v_zip(const v_uint64x8& a, const v_uint64x8& b, v_uint64x8& ab0, v_uint64x8& ab1) -{ - v_int64x8 i0, i1; - v_zip(v_reinterpret_as_s64(a), v_reinterpret_as_s64(b), i0, i1); - ab0 = v_reinterpret_as_u64(i0); - ab1 = v_reinterpret_as_u64(i1); -} -inline void v_zip(const v_float32x16& a, const v_float32x16& b, v_float32x16& ab0, v_float32x16& ab1) -{ - v_int32x16 i0, i1; - v_zip(v_reinterpret_as_s32(a), v_reinterpret_as_s32(b), i0, i1); - ab0 = v_reinterpret_as_f32(i0); - ab1 = v_reinterpret_as_f32(i1); -} -inline void v_zip(const v_float64x8& a, const v_float64x8& b, v_float64x8& ab0, v_float64x8& ab1) -{ - v_int64x8 i0, i1; - v_zip(v_reinterpret_as_s64(a), v_reinterpret_as_s64(b), i0, i1); - ab0 = v_reinterpret_as_f64(i0); - ab1 = v_reinterpret_as_f64(i1); -} - -#define OPENCV_HAL_IMPL_AVX512_COMBINE(_Tpvec, suffix) \ - inline _Tpvec v_combine_low(const _Tpvec& a, const _Tpvec& b) \ - { return _Tpvec(_v512_combine(_v512_extract_low(a.val), _v512_extract_low(b.val))); } \ - inline _Tpvec v_combine_high(const _Tpvec& a, const _Tpvec& b) \ - { return _Tpvec(_v512_insert(b.val, _v512_extract_high(a.val))); } \ - inline void v_recombine(const _Tpvec& a, const _Tpvec& b, \ - _Tpvec& c, _Tpvec& d) \ - { \ - c.val = _v512_combine(_v512_extract_low(a.val),_v512_extract_low(b.val)); \ - d.val = _v512_insert(b.val,_v512_extract_high(a.val)); \ - } - - -OPENCV_HAL_IMPL_AVX512_COMBINE(v_uint8x64, epi8) -OPENCV_HAL_IMPL_AVX512_COMBINE(v_int8x64, epi8) -OPENCV_HAL_IMPL_AVX512_COMBINE(v_uint16x32, epi16) -OPENCV_HAL_IMPL_AVX512_COMBINE(v_int16x32, epi16) -OPENCV_HAL_IMPL_AVX512_COMBINE(v_uint32x16, epi32) -OPENCV_HAL_IMPL_AVX512_COMBINE(v_int32x16, epi32) -OPENCV_HAL_IMPL_AVX512_COMBINE(v_uint64x8, epi64) -OPENCV_HAL_IMPL_AVX512_COMBINE(v_int64x8, epi64) -OPENCV_HAL_IMPL_AVX512_COMBINE(v_float32x16, ps) -OPENCV_HAL_IMPL_AVX512_COMBINE(v_float64x8, pd) - -////////// Arithmetic, bitwise and comparison operations ///////// - -/* Element-wise binary and unary operations */ - -/** Non-saturating arithmetics **/ -#define OPENCV_HAL_IMPL_AVX512_BIN_FUNC(func, _Tpvec, intrin) \ - inline _Tpvec func(const _Tpvec& a, const _Tpvec& b) \ - { return _Tpvec(intrin(a.val, b.val)); } - -OPENCV_HAL_IMPL_AVX512_BIN_FUNC(v_add_wrap, v_uint8x64, _mm512_add_epi8) -OPENCV_HAL_IMPL_AVX512_BIN_FUNC(v_add_wrap, v_int8x64, _mm512_add_epi8) -OPENCV_HAL_IMPL_AVX512_BIN_FUNC(v_add_wrap, v_uint16x32, _mm512_add_epi16) -OPENCV_HAL_IMPL_AVX512_BIN_FUNC(v_add_wrap, v_int16x32, _mm512_add_epi16) -OPENCV_HAL_IMPL_AVX512_BIN_FUNC(v_sub_wrap, v_uint8x64, _mm512_sub_epi8) -OPENCV_HAL_IMPL_AVX512_BIN_FUNC(v_sub_wrap, v_int8x64, _mm512_sub_epi8) -OPENCV_HAL_IMPL_AVX512_BIN_FUNC(v_sub_wrap, v_uint16x32, _mm512_sub_epi16) -OPENCV_HAL_IMPL_AVX512_BIN_FUNC(v_sub_wrap, v_int16x32, _mm512_sub_epi16) -OPENCV_HAL_IMPL_AVX512_BIN_FUNC(v_mul_wrap, v_uint16x32, _mm512_mullo_epi16) -OPENCV_HAL_IMPL_AVX512_BIN_FUNC(v_mul_wrap, v_int16x32, _mm512_mullo_epi16) - -inline v_uint8x64 v_mul_wrap(const v_uint8x64& a, const v_uint8x64& b) -{ - __m512i ad = _mm512_srai_epi16(a.val, 8); - __m512i bd = _mm512_srai_epi16(b.val, 8); - __m512i p0 = _mm512_mullo_epi16(a.val, b.val); // even - __m512i p1 = _mm512_slli_epi16(_mm512_mullo_epi16(ad, bd), 8); // odd - return v_uint8x64(_mm512_mask_blend_epi8(0xAAAAAAAAAAAAAAAA, p0, p1)); -} -inline v_int8x64 v_mul_wrap(const v_int8x64& a, const v_int8x64& b) -{ - return v_reinterpret_as_s8(v_mul_wrap(v_reinterpret_as_u8(a), v_reinterpret_as_u8(b))); -} - -#define OPENCV_HAL_IMPL_AVX512_BIN_OP(bin_op, _Tpvec, intrin) \ - inline _Tpvec operator bin_op (const _Tpvec& a, const _Tpvec& b) \ - { return _Tpvec(intrin(a.val, b.val)); } \ - inline _Tpvec& operator bin_op##= (_Tpvec& a, const _Tpvec& b) \ - { a.val = intrin(a.val, b.val); return a; } - -OPENCV_HAL_IMPL_AVX512_BIN_OP(+, v_uint32x16, _mm512_add_epi32) -OPENCV_HAL_IMPL_AVX512_BIN_OP(-, v_uint32x16, _mm512_sub_epi32) -OPENCV_HAL_IMPL_AVX512_BIN_OP(+, v_int32x16, _mm512_add_epi32) -OPENCV_HAL_IMPL_AVX512_BIN_OP(-, v_int32x16, _mm512_sub_epi32) -OPENCV_HAL_IMPL_AVX512_BIN_OP(+, v_uint64x8, _mm512_add_epi64) -OPENCV_HAL_IMPL_AVX512_BIN_OP(-, v_uint64x8, _mm512_sub_epi64) -OPENCV_HAL_IMPL_AVX512_BIN_OP(+, v_int64x8, _mm512_add_epi64) -OPENCV_HAL_IMPL_AVX512_BIN_OP(-, v_int64x8, _mm512_sub_epi64) - -OPENCV_HAL_IMPL_AVX512_BIN_OP(*, v_uint32x16, _mm512_mullo_epi32) -OPENCV_HAL_IMPL_AVX512_BIN_OP(*, v_int32x16, _mm512_mullo_epi32) -OPENCV_HAL_IMPL_AVX512_BIN_OP(*, v_uint64x8, _mm512_mullo_epi64) -OPENCV_HAL_IMPL_AVX512_BIN_OP(*, v_int64x8, _mm512_mullo_epi64) - -/** Saturating arithmetics **/ -OPENCV_HAL_IMPL_AVX512_BIN_OP(+, v_uint8x64, _mm512_adds_epu8) -OPENCV_HAL_IMPL_AVX512_BIN_OP(-, v_uint8x64, _mm512_subs_epu8) -OPENCV_HAL_IMPL_AVX512_BIN_OP(+, v_int8x64, _mm512_adds_epi8) -OPENCV_HAL_IMPL_AVX512_BIN_OP(-, v_int8x64, _mm512_subs_epi8) -OPENCV_HAL_IMPL_AVX512_BIN_OP(+, v_uint16x32, _mm512_adds_epu16) -OPENCV_HAL_IMPL_AVX512_BIN_OP(-, v_uint16x32, _mm512_subs_epu16) -OPENCV_HAL_IMPL_AVX512_BIN_OP(+, v_int16x32, _mm512_adds_epi16) -OPENCV_HAL_IMPL_AVX512_BIN_OP(-, v_int16x32, _mm512_subs_epi16) - -OPENCV_HAL_IMPL_AVX512_BIN_OP(+, v_float32x16, _mm512_add_ps) -OPENCV_HAL_IMPL_AVX512_BIN_OP(-, v_float32x16, _mm512_sub_ps) -OPENCV_HAL_IMPL_AVX512_BIN_OP(*, v_float32x16, _mm512_mul_ps) -OPENCV_HAL_IMPL_AVX512_BIN_OP(/, v_float32x16, _mm512_div_ps) -OPENCV_HAL_IMPL_AVX512_BIN_OP(+, v_float64x8, _mm512_add_pd) -OPENCV_HAL_IMPL_AVX512_BIN_OP(-, v_float64x8, _mm512_sub_pd) -OPENCV_HAL_IMPL_AVX512_BIN_OP(*, v_float64x8, _mm512_mul_pd) -OPENCV_HAL_IMPL_AVX512_BIN_OP(/, v_float64x8, _mm512_div_pd) - -// saturating multiply -inline v_uint8x64 operator * (const v_uint8x64& a, const v_uint8x64& b) -{ - v_uint16x32 c, d; - v_mul_expand(a, b, c, d); - return v_pack(c, d); -} -inline v_int8x64 operator * (const v_int8x64& a, const v_int8x64& b) -{ - v_int16x32 c, d; - v_mul_expand(a, b, c, d); - return v_pack(c, d); -} -inline v_uint16x32 operator * (const v_uint16x32& a, const v_uint16x32& b) -{ - __m512i pl = _mm512_mullo_epi16(a.val, b.val); - __m512i ph = _mm512_mulhi_epu16(a.val, b.val); - __m512i p0 = _mm512_unpacklo_epi16(pl, ph); - __m512i p1 = _mm512_unpackhi_epi16(pl, ph); - - const __m512i m = _mm512_set1_epi32(65535); - return v_uint16x32(_mm512_packus_epi32(_mm512_min_epu32(p0, m), _mm512_min_epu32(p1, m))); -} -inline v_int16x32 operator * (const v_int16x32& a, const v_int16x32& b) -{ - __m512i pl = _mm512_mullo_epi16(a.val, b.val); - __m512i ph = _mm512_mulhi_epi16(a.val, b.val); - __m512i p0 = _mm512_unpacklo_epi16(pl, ph); - __m512i p1 = _mm512_unpackhi_epi16(pl, ph); - return v_int16x32(_mm512_packs_epi32(p0, p1)); -} - -inline v_uint8x64& operator *= (v_uint8x64& a, const v_uint8x64& b) -{ a = a * b; return a; } -inline v_int8x64& operator *= (v_int8x64& a, const v_int8x64& b) -{ a = a * b; return a; } -inline v_uint16x32& operator *= (v_uint16x32& a, const v_uint16x32& b) -{ a = a * b; return a; } -inline v_int16x32& operator *= (v_int16x32& a, const v_int16x32& b) -{ a = a * b; return a; } - -inline v_int16x32 v_mul_hi(const v_int16x32& a, const v_int16x32& b) { return v_int16x32(_mm512_mulhi_epi16(a.val, b.val)); } -inline v_uint16x32 v_mul_hi(const v_uint16x32& a, const v_uint16x32& b) { return v_uint16x32(_mm512_mulhi_epu16(a.val, b.val)); } - -// Multiply and expand -inline void v_mul_expand(const v_uint8x64& a, const v_uint8x64& b, - v_uint16x32& c, v_uint16x32& d) -{ - v_uint16x32 a0, a1, b0, b1; - v_expand(a, a0, a1); - v_expand(b, b0, b1); - c = v_mul_wrap(a0, b0); - d = v_mul_wrap(a1, b1); -} - -inline void v_mul_expand(const v_int8x64& a, const v_int8x64& b, - v_int16x32& c, v_int16x32& d) -{ - v_int16x32 a0, a1, b0, b1; - v_expand(a, a0, a1); - v_expand(b, b0, b1); - c = v_mul_wrap(a0, b0); - d = v_mul_wrap(a1, b1); -} - -inline void v_mul_expand(const v_int16x32& a, const v_int16x32& b, - v_int32x16& c, v_int32x16& d) -{ - v_int16x32 v0, v1; - v_zip(v_mul_wrap(a, b), v_mul_hi(a, b), v0, v1); - - c = v_reinterpret_as_s32(v0); - d = v_reinterpret_as_s32(v1); -} - -inline void v_mul_expand(const v_uint16x32& a, const v_uint16x32& b, - v_uint32x16& c, v_uint32x16& d) -{ - v_uint16x32 v0, v1; - v_zip(v_mul_wrap(a, b), v_mul_hi(a, b), v0, v1); - - c = v_reinterpret_as_u32(v0); - d = v_reinterpret_as_u32(v1); -} - -inline void v_mul_expand(const v_uint32x16& a, const v_uint32x16& b, - v_uint64x8& c, v_uint64x8& d) -{ - v_zip(v_uint64x8(_mm512_mul_epu32(a.val, b.val)), - v_uint64x8(_mm512_mul_epu32(_mm512_srli_epi64(a.val, 32), _mm512_srli_epi64(b.val, 32))), c, d); -} - -inline void v_mul_expand(const v_int32x16& a, const v_int32x16& b, - v_int64x8& c, v_int64x8& d) -{ - v_zip(v_int64x8(_mm512_mul_epi32(a.val, b.val)), - v_int64x8(_mm512_mul_epi32(_mm512_srli_epi64(a.val, 32), _mm512_srli_epi64(b.val, 32))), c, d); -} - -/** Bitwise shifts **/ -#define OPENCV_HAL_IMPL_AVX512_SHIFT_OP(_Tpuvec, _Tpsvec, suffix) \ - inline _Tpuvec operator << (const _Tpuvec& a, int imm) \ - { return _Tpuvec(_mm512_slli_##suffix(a.val, imm)); } \ - inline _Tpsvec operator << (const _Tpsvec& a, int imm) \ - { return _Tpsvec(_mm512_slli_##suffix(a.val, imm)); } \ - inline _Tpuvec operator >> (const _Tpuvec& a, int imm) \ - { return _Tpuvec(_mm512_srli_##suffix(a.val, imm)); } \ - inline _Tpsvec operator >> (const _Tpsvec& a, int imm) \ - { return _Tpsvec(_mm512_srai_##suffix(a.val, imm)); } \ - template \ - inline _Tpuvec v_shl(const _Tpuvec& a) \ - { return _Tpuvec(_mm512_slli_##suffix(a.val, imm)); } \ - template \ - inline _Tpsvec v_shl(const _Tpsvec& a) \ - { return _Tpsvec(_mm512_slli_##suffix(a.val, imm)); } \ - template \ - inline _Tpuvec v_shr(const _Tpuvec& a) \ - { return _Tpuvec(_mm512_srli_##suffix(a.val, imm)); } \ - template \ - inline _Tpsvec v_shr(const _Tpsvec& a) \ - { return _Tpsvec(_mm512_srai_##suffix(a.val, imm)); } - -OPENCV_HAL_IMPL_AVX512_SHIFT_OP(v_uint16x32, v_int16x32, epi16) -OPENCV_HAL_IMPL_AVX512_SHIFT_OP(v_uint32x16, v_int32x16, epi32) -OPENCV_HAL_IMPL_AVX512_SHIFT_OP(v_uint64x8, v_int64x8, epi64) - - -/** Bitwise logic **/ -#define OPENCV_HAL_IMPL_AVX512_LOGIC_OP(_Tpvec, suffix, not_const) \ - OPENCV_HAL_IMPL_AVX512_BIN_OP(&, _Tpvec, _mm512_and_##suffix) \ - OPENCV_HAL_IMPL_AVX512_BIN_OP(|, _Tpvec, _mm512_or_##suffix) \ - OPENCV_HAL_IMPL_AVX512_BIN_OP(^, _Tpvec, _mm512_xor_##suffix) \ - inline _Tpvec operator ~ (const _Tpvec& a) \ - { return _Tpvec(_mm512_xor_##suffix(a.val, not_const)); } - -OPENCV_HAL_IMPL_AVX512_LOGIC_OP(v_uint8x64, si512, _mm512_set1_epi32(-1)) -OPENCV_HAL_IMPL_AVX512_LOGIC_OP(v_int8x64, si512, _mm512_set1_epi32(-1)) -OPENCV_HAL_IMPL_AVX512_LOGIC_OP(v_uint16x32, si512, _mm512_set1_epi32(-1)) -OPENCV_HAL_IMPL_AVX512_LOGIC_OP(v_int16x32, si512, _mm512_set1_epi32(-1)) -OPENCV_HAL_IMPL_AVX512_LOGIC_OP(v_uint32x16, si512, _mm512_set1_epi32(-1)) -OPENCV_HAL_IMPL_AVX512_LOGIC_OP(v_int32x16, si512, _mm512_set1_epi32(-1)) -OPENCV_HAL_IMPL_AVX512_LOGIC_OP(v_uint64x8, si512, _mm512_set1_epi64(-1)) -OPENCV_HAL_IMPL_AVX512_LOGIC_OP(v_int64x8, si512, _mm512_set1_epi64(-1)) -OPENCV_HAL_IMPL_AVX512_LOGIC_OP(v_float32x16, ps, _mm512_castsi512_ps(_mm512_set1_epi32(-1))) -OPENCV_HAL_IMPL_AVX512_LOGIC_OP(v_float64x8, pd, _mm512_castsi512_pd(_mm512_set1_epi32(-1))) - -/** Select **/ -#define OPENCV_HAL_IMPL_AVX512_SELECT(_Tpvec, suffix, zsuf) \ - inline _Tpvec v_select(const _Tpvec& mask, const _Tpvec& a, const _Tpvec& b) \ - { return _Tpvec(_mm512_mask_blend_##suffix(_mm512_cmp_##suffix##_mask(mask.val, _mm512_setzero_##zsuf(), _MM_CMPINT_EQ), a.val, b.val)); } - -OPENCV_HAL_IMPL_AVX512_SELECT(v_uint8x64, epi8, si512) -OPENCV_HAL_IMPL_AVX512_SELECT(v_int8x64, epi8, si512) -OPENCV_HAL_IMPL_AVX512_SELECT(v_uint16x32, epi16, si512) -OPENCV_HAL_IMPL_AVX512_SELECT(v_int16x32, epi16, si512) -OPENCV_HAL_IMPL_AVX512_SELECT(v_uint32x16, epi32, si512) -OPENCV_HAL_IMPL_AVX512_SELECT(v_int32x16, epi32, si512) -OPENCV_HAL_IMPL_AVX512_SELECT(v_uint64x8, epi64, si512) -OPENCV_HAL_IMPL_AVX512_SELECT(v_int64x8, epi64, si512) -OPENCV_HAL_IMPL_AVX512_SELECT(v_float32x16, ps, ps) -OPENCV_HAL_IMPL_AVX512_SELECT(v_float64x8, pd, pd) - -/** Comparison **/ -#define OPENCV_HAL_IMPL_AVX512_CMP_INT(bin_op, imm8, _Tpvec, sufcmp, sufset, tval) \ - inline _Tpvec operator bin_op (const _Tpvec& a, const _Tpvec& b) \ - { return _Tpvec(_mm512_maskz_set1_##sufset(_mm512_cmp_##sufcmp##_mask(a.val, b.val, imm8), tval)); } - -#define OPENCV_HAL_IMPL_AVX512_CMP_OP_INT(_Tpvec, sufcmp, sufset, tval) \ - OPENCV_HAL_IMPL_AVX512_CMP_INT(==, _MM_CMPINT_EQ, _Tpvec, sufcmp, sufset, tval) \ - OPENCV_HAL_IMPL_AVX512_CMP_INT(!=, _MM_CMPINT_NE, _Tpvec, sufcmp, sufset, tval) \ - OPENCV_HAL_IMPL_AVX512_CMP_INT(<, _MM_CMPINT_LT, _Tpvec, sufcmp, sufset, tval) \ - OPENCV_HAL_IMPL_AVX512_CMP_INT(>, _MM_CMPINT_NLE, _Tpvec, sufcmp, sufset, tval) \ - OPENCV_HAL_IMPL_AVX512_CMP_INT(<=, _MM_CMPINT_LE, _Tpvec, sufcmp, sufset, tval) \ - OPENCV_HAL_IMPL_AVX512_CMP_INT(>=, _MM_CMPINT_NLT, _Tpvec, sufcmp, sufset, tval) - -OPENCV_HAL_IMPL_AVX512_CMP_OP_INT(v_uint8x64, epu8, epi8, (char)-1) -OPENCV_HAL_IMPL_AVX512_CMP_OP_INT(v_int8x64, epi8, epi8, (char)-1) -OPENCV_HAL_IMPL_AVX512_CMP_OP_INT(v_uint16x32, epu16, epi16, (short)-1) -OPENCV_HAL_IMPL_AVX512_CMP_OP_INT(v_int16x32, epi16, epi16, (short)-1) -OPENCV_HAL_IMPL_AVX512_CMP_OP_INT(v_uint32x16, epu32, epi32, (int)-1) -OPENCV_HAL_IMPL_AVX512_CMP_OP_INT(v_int32x16, epi32, epi32, (int)-1) -OPENCV_HAL_IMPL_AVX512_CMP_OP_INT(v_uint64x8, epu64, epi64, (int64)-1) -OPENCV_HAL_IMPL_AVX512_CMP_OP_INT(v_int64x8, epi64, epi64, (int64)-1) - -#define OPENCV_HAL_IMPL_AVX512_CMP_FLT(bin_op, imm8, _Tpvec, sufcmp, sufset, tval) \ - inline _Tpvec operator bin_op (const _Tpvec& a, const _Tpvec& b) \ - { return _Tpvec(_mm512_castsi512_##sufcmp(_mm512_maskz_set1_##sufset(_mm512_cmp_##sufcmp##_mask(a.val, b.val, imm8), tval))); } - -#define OPENCV_HAL_IMPL_AVX512_CMP_OP_FLT(_Tpvec, sufcmp, sufset, tval) \ - OPENCV_HAL_IMPL_AVX512_CMP_FLT(==, _CMP_EQ_OQ, _Tpvec, sufcmp, sufset, tval) \ - OPENCV_HAL_IMPL_AVX512_CMP_FLT(!=, _CMP_NEQ_OQ, _Tpvec, sufcmp, sufset, tval) \ - OPENCV_HAL_IMPL_AVX512_CMP_FLT(<, _CMP_LT_OQ, _Tpvec, sufcmp, sufset, tval) \ - OPENCV_HAL_IMPL_AVX512_CMP_FLT(>, _CMP_GT_OQ, _Tpvec, sufcmp, sufset, tval) \ - OPENCV_HAL_IMPL_AVX512_CMP_FLT(<=, _CMP_LE_OQ, _Tpvec, sufcmp, sufset, tval) \ - OPENCV_HAL_IMPL_AVX512_CMP_FLT(>=, _CMP_GE_OQ, _Tpvec, sufcmp, sufset, tval) - -OPENCV_HAL_IMPL_AVX512_CMP_OP_FLT(v_float32x16, ps, epi32, (int)-1) -OPENCV_HAL_IMPL_AVX512_CMP_OP_FLT(v_float64x8, pd, epi64, (int64)-1) - -inline v_float32x16 v_not_nan(const v_float32x16& a) -{ return v_float32x16(_mm512_castsi512_ps(_mm512_maskz_set1_epi32(_mm512_cmp_ps_mask(a.val, a.val, _CMP_ORD_Q), (int)-1))); } -inline v_float64x8 v_not_nan(const v_float64x8& a) -{ return v_float64x8(_mm512_castsi512_pd(_mm512_maskz_set1_epi64(_mm512_cmp_pd_mask(a.val, a.val, _CMP_ORD_Q), (int64)-1))); } - -/** min/max **/ -OPENCV_HAL_IMPL_AVX512_BIN_FUNC(v_min, v_uint8x64, _mm512_min_epu8) -OPENCV_HAL_IMPL_AVX512_BIN_FUNC(v_max, v_uint8x64, _mm512_max_epu8) -OPENCV_HAL_IMPL_AVX512_BIN_FUNC(v_min, v_int8x64, _mm512_min_epi8) -OPENCV_HAL_IMPL_AVX512_BIN_FUNC(v_max, v_int8x64, _mm512_max_epi8) -OPENCV_HAL_IMPL_AVX512_BIN_FUNC(v_min, v_uint16x32, _mm512_min_epu16) -OPENCV_HAL_IMPL_AVX512_BIN_FUNC(v_max, v_uint16x32, _mm512_max_epu16) -OPENCV_HAL_IMPL_AVX512_BIN_FUNC(v_min, v_int16x32, _mm512_min_epi16) -OPENCV_HAL_IMPL_AVX512_BIN_FUNC(v_max, v_int16x32, _mm512_max_epi16) -OPENCV_HAL_IMPL_AVX512_BIN_FUNC(v_min, v_uint32x16, _mm512_min_epu32) -OPENCV_HAL_IMPL_AVX512_BIN_FUNC(v_max, v_uint32x16, _mm512_max_epu32) -OPENCV_HAL_IMPL_AVX512_BIN_FUNC(v_min, v_int32x16, _mm512_min_epi32) -OPENCV_HAL_IMPL_AVX512_BIN_FUNC(v_max, v_int32x16, _mm512_max_epi32) -OPENCV_HAL_IMPL_AVX512_BIN_FUNC(v_min, v_uint64x8, _mm512_min_epu64) -OPENCV_HAL_IMPL_AVX512_BIN_FUNC(v_max, v_uint64x8, _mm512_max_epu64) -OPENCV_HAL_IMPL_AVX512_BIN_FUNC(v_min, v_int64x8, _mm512_min_epi64) -OPENCV_HAL_IMPL_AVX512_BIN_FUNC(v_max, v_int64x8, _mm512_max_epi64) -OPENCV_HAL_IMPL_AVX512_BIN_FUNC(v_min, v_float32x16, _mm512_min_ps) -OPENCV_HAL_IMPL_AVX512_BIN_FUNC(v_max, v_float32x16, _mm512_max_ps) -OPENCV_HAL_IMPL_AVX512_BIN_FUNC(v_min, v_float64x8, _mm512_min_pd) -OPENCV_HAL_IMPL_AVX512_BIN_FUNC(v_max, v_float64x8, _mm512_max_pd) - -/** Rotate **/ -namespace { - template - struct _v_rotate_right { static inline v_int8x64 eval(const v_int8x64&, const v_int8x64&) { return v_int8x64(); }}; - template - struct _v_rotate_right { static inline v_int8x64 eval(const v_int8x64& a, const v_int8x64& b) - { - return v_int8x64(_mm512_or_si512(_mm512_srli_epi32(_mm512_alignr_epi32(b.val, a.val, imm32 ), imm4 *8), - _mm512_slli_epi32(_mm512_alignr_epi32(b.val, a.val, imm32 + 1), (4-imm4)*8))); - }}; - template - struct _v_rotate_right { static inline v_int8x64 eval(const v_int8x64& a, const v_int8x64& b) - { - return v_int8x64(_mm512_or_si512(_mm512_srli_epi32(_mm512_alignr_epi32(b.val, a.val, 15), imm4 *8), - _mm512_slli_epi32( b.val, (4-imm4)*8))); - }}; - template - struct _v_rotate_right { static inline v_int8x64 eval(const v_int8x64&, const v_int8x64& b) - { - return v_int8x64(_mm512_or_si512(_mm512_srli_epi32(_mm512_alignr_epi32(_mm512_setzero_si512(), b.val, imm32 - 16), imm4 *8), - _mm512_slli_epi32(_mm512_alignr_epi32(_mm512_setzero_si512(), b.val, imm32 - 15), (4-imm4)*8))); - }}; - template - struct _v_rotate_right { static inline v_int8x64 eval(const v_int8x64&, const v_int8x64& b) - { return v_int8x64(_mm512_srli_epi32(_mm512_alignr_epi32(_mm512_setzero_si512(), b.val, 15), imm4*8)); }}; - template - struct _v_rotate_right { static inline v_int8x64 eval(const v_int8x64& a, const v_int8x64& b) - { return v_int8x64(_mm512_alignr_epi32(b.val, a.val, imm32)); }}; - template<> - struct _v_rotate_right { static inline v_int8x64 eval(const v_int8x64& a, const v_int8x64&) { return a; }}; - template - struct _v_rotate_right { static inline v_int8x64 eval(const v_int8x64&, const v_int8x64& b) - { return v_int8x64(_mm512_alignr_epi32(_mm512_setzero_si512(), b.val, imm32 - 16)); }}; - template<> - struct _v_rotate_right { static inline v_int8x64 eval(const v_int8x64&, const v_int8x64& b) { return b; }}; - template<> - struct _v_rotate_right { static inline v_int8x64 eval(const v_int8x64&, const v_int8x64&) { return v_int8x64(); }}; -} -template inline v_int8x64 v_rotate_right(const v_int8x64& a, const v_int8x64& b) -{ - return imm >= 128 ? v_int8x64() : -#if CV_AVX_512VBMI - v_int8x64(_mm512_permutex2var_epi8(a.val, - _v512_set_epu8(0x3f + imm, 0x3e + imm, 0x3d + imm, 0x3c + imm, 0x3b + imm, 0x3a + imm, 0x39 + imm, 0x38 + imm, - 0x37 + imm, 0x36 + imm, 0x35 + imm, 0x34 + imm, 0x33 + imm, 0x32 + imm, 0x31 + imm, 0x30 + imm, - 0x2f + imm, 0x2e + imm, 0x2d + imm, 0x2c + imm, 0x2b + imm, 0x2a + imm, 0x29 + imm, 0x28 + imm, - 0x27 + imm, 0x26 + imm, 0x25 + imm, 0x24 + imm, 0x23 + imm, 0x22 + imm, 0x21 + imm, 0x20 + imm, - 0x1f + imm, 0x1e + imm, 0x1d + imm, 0x1c + imm, 0x1b + imm, 0x1a + imm, 0x19 + imm, 0x18 + imm, - 0x17 + imm, 0x16 + imm, 0x15 + imm, 0x14 + imm, 0x13 + imm, 0x12 + imm, 0x11 + imm, 0x10 + imm, - 0x0f + imm, 0x0e + imm, 0x0d + imm, 0x0c + imm, 0x0b + imm, 0x0a + imm, 0x09 + imm, 0x08 + imm, - 0x07 + imm, 0x06 + imm, 0x05 + imm, 0x04 + imm, 0x03 + imm, 0x02 + imm, 0x01 + imm, 0x00 + imm), b.val)); -#else - _v_rotate_right 15), imm/4>::eval(a, b); -#endif -} -template -inline v_int8x64 v_rotate_left(const v_int8x64& a, const v_int8x64& b) -{ - if (imm == 0) return a; - if (imm == 64) return b; - if (imm >= 128) return v_int8x64(); -#if CV_AVX_512VBMI - return v_int8x64(_mm512_permutex2var_epi8(b.val, - _v512_set_epi8(0x7f - imm,0x7e - imm,0x7d - imm,0x7c - imm,0x7b - imm,0x7a - imm,0x79 - imm,0x78 - imm, - 0x77 - imm,0x76 - imm,0x75 - imm,0x74 - imm,0x73 - imm,0x72 - imm,0x71 - imm,0x70 - imm, - 0x6f - imm,0x6e - imm,0x6d - imm,0x6c - imm,0x6b - imm,0x6a - imm,0x69 - imm,0x68 - imm, - 0x67 - imm,0x66 - imm,0x65 - imm,0x64 - imm,0x63 - imm,0x62 - imm,0x61 - imm,0x60 - imm, - 0x5f - imm,0x5e - imm,0x5d - imm,0x5c - imm,0x5b - imm,0x5a - imm,0x59 - imm,0x58 - imm, - 0x57 - imm,0x56 - imm,0x55 - imm,0x54 - imm,0x53 - imm,0x52 - imm,0x51 - imm,0x50 - imm, - 0x4f - imm,0x4e - imm,0x4d - imm,0x4c - imm,0x4b - imm,0x4a - imm,0x49 - imm,0x48 - imm, - 0x47 - imm,0x46 - imm,0x45 - imm,0x44 - imm,0x43 - imm,0x42 - imm,0x41 - imm,0x40 - imm), a.val)); -#else - return imm < 64 ? v_rotate_right<64 - imm>(b, a) : v_rotate_right<128 - imm>(v512_setzero_s8(), b); -#endif -} -template -inline v_int8x64 v_rotate_right(const v_int8x64& a) -{ - if (imm == 0) return a; - if (imm >= 64) return v_int8x64(); -#if CV_AVX_512VBMI - return v_int8x64(_mm512_maskz_permutexvar_epi8(0xFFFFFFFFFFFFFFFF >> imm, - _v512_set_epu8(0x3f + imm,0x3e + imm,0x3d + imm,0x3c + imm,0x3b + imm,0x3a + imm,0x39 + imm,0x38 + imm, - 0x37 + imm,0x36 + imm,0x35 + imm,0x34 + imm,0x33 + imm,0x32 + imm,0x31 + imm,0x30 + imm, - 0x2f + imm,0x2e + imm,0x2d + imm,0x2c + imm,0x2b + imm,0x2a + imm,0x29 + imm,0x28 + imm, - 0x27 + imm,0x26 + imm,0x25 + imm,0x24 + imm,0x23 + imm,0x22 + imm,0x21 + imm,0x20 + imm, - 0x1f + imm,0x1e + imm,0x1d + imm,0x1c + imm,0x1b + imm,0x1a + imm,0x19 + imm,0x18 + imm, - 0x17 + imm,0x16 + imm,0x15 + imm,0x14 + imm,0x13 + imm,0x12 + imm,0x11 + imm,0x10 + imm, - 0x0f + imm,0x0e + imm,0x0d + imm,0x0c + imm,0x0b + imm,0x0a + imm,0x09 + imm,0x08 + imm, - 0x07 + imm,0x06 + imm,0x05 + imm,0x04 + imm,0x03 + imm,0x02 + imm,0x01 + imm,0x00 + imm), a.val)); -#else - return v_rotate_right(a, v512_setzero_s8()); -#endif -} -template -inline v_int8x64 v_rotate_left(const v_int8x64& a) -{ - if (imm == 0) return a; - if (imm >= 64) return v_int8x64(); -#if CV_AVX_512VBMI - return v_int8x64(_mm512_maskz_permutexvar_epi8(0xFFFFFFFFFFFFFFFF << imm, - _v512_set_epi8(0x3f - imm,0x3e - imm,0x3d - imm,0x3c - imm,0x3b - imm,0x3a - imm,0x39 - imm,0x38 - imm, - 0x37 - imm,0x36 - imm,0x35 - imm,0x34 - imm,0x33 - imm,0x32 - imm,0x31 - imm,0x30 - imm, - 0x2f - imm,0x2e - imm,0x2d - imm,0x2c - imm,0x2b - imm,0x2a - imm,0x29 - imm,0x28 - imm, - 0x27 - imm,0x26 - imm,0x25 - imm,0x24 - imm,0x23 - imm,0x22 - imm,0x21 - imm,0x20 - imm, - 0x1f - imm,0x1e - imm,0x1d - imm,0x1c - imm,0x1b - imm,0x1a - imm,0x19 - imm,0x18 - imm, - 0x17 - imm,0x16 - imm,0x15 - imm,0x14 - imm,0x13 - imm,0x12 - imm,0x11 - imm,0x10 - imm, - 0x0f - imm,0x0e - imm,0x0d - imm,0x0c - imm,0x0b - imm,0x0a - imm,0x09 - imm,0x08 - imm, - 0x07 - imm,0x06 - imm,0x05 - imm,0x04 - imm,0x03 - imm,0x02 - imm,0x01 - imm,0x00 - imm), a.val)); -#else - return v_rotate_right<64 - imm>(v512_setzero_s8(), a); -#endif -} - -#define OPENCV_HAL_IMPL_AVX512_ROTATE_PM(_Tpvec, suffix) \ -template inline _Tpvec v_rotate_left(const _Tpvec& a, const _Tpvec& b) \ -{ return v_reinterpret_as_##suffix(v_rotate_left(v_reinterpret_as_s8(a), v_reinterpret_as_s8(b))); } \ -template inline _Tpvec v_rotate_right(const _Tpvec& a, const _Tpvec& b) \ -{ return v_reinterpret_as_##suffix(v_rotate_right(v_reinterpret_as_s8(a), v_reinterpret_as_s8(b))); } \ -template inline _Tpvec v_rotate_left(const _Tpvec& a) \ -{ return v_reinterpret_as_##suffix(v_rotate_left(v_reinterpret_as_s8(a))); } \ -template inline _Tpvec v_rotate_right(const _Tpvec& a) \ -{ return v_reinterpret_as_##suffix(v_rotate_right(v_reinterpret_as_s8(a))); } - -#define OPENCV_HAL_IMPL_AVX512_ROTATE_EC(_Tpvec, suffix) \ -template \ -inline _Tpvec v_rotate_left(const _Tpvec& a, const _Tpvec& b) \ -{ \ - enum { SHIFT2 = (_Tpvec::nlanes - imm) }; \ - enum { MASK = ((1 << _Tpvec::nlanes) - 1) }; \ - if (imm == 0) return a; \ - if (imm == _Tpvec::nlanes) return b; \ - if (imm >= 2*_Tpvec::nlanes) return _Tpvec(); \ - return _Tpvec(_mm512_mask_expand_##suffix(_mm512_maskz_compress_##suffix((MASK << SHIFT2)&MASK, b.val), (MASK << (imm))&MASK, a.val)); \ -} \ -template \ -inline _Tpvec v_rotate_right(const _Tpvec& a, const _Tpvec& b) \ -{ \ - enum { SHIFT2 = (_Tpvec::nlanes - imm) }; \ - enum { MASK = ((1 << _Tpvec::nlanes) - 1) }; \ - if (imm == 0) return a; \ - if (imm == _Tpvec::nlanes) return b; \ - if (imm >= 2*_Tpvec::nlanes) return _Tpvec(); \ - return _Tpvec(_mm512_mask_expand_##suffix(_mm512_maskz_compress_##suffix((MASK << (imm))&MASK, a.val), (MASK << SHIFT2)&MASK, b.val)); \ -} \ -template \ -inline _Tpvec v_rotate_left(const _Tpvec& a) \ -{ \ - if (imm == 0) return a; \ - if (imm >= _Tpvec::nlanes) return _Tpvec(); \ - return _Tpvec(_mm512_maskz_expand_##suffix((1 << _Tpvec::nlanes) - (1 << (imm)), a.val)); \ -} \ -template \ -inline _Tpvec v_rotate_right(const _Tpvec& a) \ -{ \ - if (imm == 0) return a; \ - if (imm >= _Tpvec::nlanes) return _Tpvec(); \ - return _Tpvec(_mm512_maskz_compress_##suffix((1 << _Tpvec::nlanes) - (1 << (imm)), a.val)); \ -} - -OPENCV_HAL_IMPL_AVX512_ROTATE_PM(v_uint8x64, u8) -OPENCV_HAL_IMPL_AVX512_ROTATE_PM(v_uint16x32, u16) -OPENCV_HAL_IMPL_AVX512_ROTATE_PM(v_int16x32, s16) -OPENCV_HAL_IMPL_AVX512_ROTATE_EC(v_uint32x16, epi32) -OPENCV_HAL_IMPL_AVX512_ROTATE_EC(v_int32x16, epi32) -OPENCV_HAL_IMPL_AVX512_ROTATE_EC(v_uint64x8, epi64) -OPENCV_HAL_IMPL_AVX512_ROTATE_EC(v_int64x8, epi64) -OPENCV_HAL_IMPL_AVX512_ROTATE_EC(v_float32x16, ps) -OPENCV_HAL_IMPL_AVX512_ROTATE_EC(v_float64x8, pd) - -////////// Reduce ///////// - -/** Reduce **/ -#define OPENCV_HAL_IMPL_AVX512_REDUCE_ADD64(a, b) a + b -#define OPENCV_HAL_IMPL_AVX512_REDUCE_8(sctype, func, _Tpvec, ifunc, scop) \ - inline sctype v_reduce_##func(const _Tpvec& a) \ - { __m256i half = _mm256_##ifunc(_v512_extract_low(a.val), _v512_extract_high(a.val)); \ - sctype CV_DECL_ALIGNED(64) idx[2]; \ - _mm_store_si128((__m128i*)idx, _mm_##ifunc(_mm256_castsi256_si128(half), _mm256_extracti128_si256(half, 1))); \ - return scop(idx[0], idx[1]); } - -OPENCV_HAL_IMPL_AVX512_REDUCE_8(uint64, min, v_uint64x8, min_epu64, min) -OPENCV_HAL_IMPL_AVX512_REDUCE_8(uint64, max, v_uint64x8, max_epu64, max) -OPENCV_HAL_IMPL_AVX512_REDUCE_8(uint64, sum, v_uint64x8, add_epi64, OPENCV_HAL_IMPL_AVX512_REDUCE_ADD64) -OPENCV_HAL_IMPL_AVX512_REDUCE_8(int64, min, v_int64x8, min_epi64, min) -OPENCV_HAL_IMPL_AVX512_REDUCE_8(int64, max, v_int64x8, max_epi64, max) -OPENCV_HAL_IMPL_AVX512_REDUCE_8(int64, sum, v_int64x8, add_epi64, OPENCV_HAL_IMPL_AVX512_REDUCE_ADD64) - -#define OPENCV_HAL_IMPL_AVX512_REDUCE_8F(func, ifunc, scop) \ - inline double v_reduce_##func(const v_float64x8& a) \ - { __m256d half = _mm256_##ifunc(_v512_extract_low(a.val), _v512_extract_high(a.val)); \ - double CV_DECL_ALIGNED(64) idx[2]; \ - _mm_store_pd(idx, _mm_##ifunc(_mm256_castpd256_pd128(half), _mm256_extractf128_pd(half, 1))); \ - return scop(idx[0], idx[1]); } - -OPENCV_HAL_IMPL_AVX512_REDUCE_8F(min, min_pd, min) -OPENCV_HAL_IMPL_AVX512_REDUCE_8F(max, max_pd, max) -OPENCV_HAL_IMPL_AVX512_REDUCE_8F(sum, add_pd, OPENCV_HAL_IMPL_AVX512_REDUCE_ADD64) - -#define OPENCV_HAL_IMPL_AVX512_REDUCE_16(sctype, func, _Tpvec, ifunc) \ - inline sctype v_reduce_##func(const _Tpvec& a) \ - { __m256i half = _mm256_##ifunc(_v512_extract_low(a.val), _v512_extract_high(a.val)); \ - __m128i quarter = _mm_##ifunc(_mm256_castsi256_si128(half), _mm256_extracti128_si256(half, 1)); \ - quarter = _mm_##ifunc(quarter, _mm_srli_si128(quarter, 8)); \ - quarter = _mm_##ifunc(quarter, _mm_srli_si128(quarter, 4)); \ - return (sctype)_mm_cvtsi128_si32(quarter); } -OPENCV_HAL_IMPL_AVX512_REDUCE_16(uint, min, v_uint32x16, min_epu32) -OPENCV_HAL_IMPL_AVX512_REDUCE_16(uint, max, v_uint32x16, max_epu32) -OPENCV_HAL_IMPL_AVX512_REDUCE_16(int, min, v_int32x16, min_epi32) -OPENCV_HAL_IMPL_AVX512_REDUCE_16(int, max, v_int32x16, max_epi32) - -#define OPENCV_HAL_IMPL_AVX512_REDUCE_16F(func, ifunc) \ - inline float v_reduce_##func(const v_float32x16& a) \ - { __m256 half = _mm256_##ifunc(_v512_extract_low(a.val), _v512_extract_high(a.val)); \ - __m128 quarter = _mm_##ifunc(_mm256_castps256_ps128(half), _mm256_extractf128_ps(half, 1)); \ - quarter = _mm_##ifunc(quarter, _mm_permute_ps(quarter, _MM_SHUFFLE(0, 0, 3, 2))); \ - quarter = _mm_##ifunc(quarter, _mm_permute_ps(quarter, _MM_SHUFFLE(0, 0, 0, 1))); \ - return _mm_cvtss_f32(quarter); } -OPENCV_HAL_IMPL_AVX512_REDUCE_16F(min, min_ps) -OPENCV_HAL_IMPL_AVX512_REDUCE_16F(max, max_ps) - -inline float v_reduce_sum(const v_float32x16& a) -{ - __m256 half = _mm256_add_ps(_v512_extract_low(a.val), _v512_extract_high(a.val)); - __m128 quarter = _mm_add_ps(_mm256_castps256_ps128(half), _mm256_extractf128_ps(half, 1)); - quarter = _mm_hadd_ps(quarter, quarter); - return _mm_cvtss_f32(_mm_hadd_ps(quarter, quarter)); -} -inline int v_reduce_sum(const v_int32x16& a) -{ - __m256i half = _mm256_add_epi32(_v512_extract_low(a.val), _v512_extract_high(a.val)); - __m128i quarter = _mm_add_epi32(_mm256_castsi256_si128(half), _mm256_extracti128_si256(half, 1)); - quarter = _mm_hadd_epi32(quarter, quarter); - return _mm_cvtsi128_si32(_mm_hadd_epi32(quarter, quarter)); -} -inline uint v_reduce_sum(const v_uint32x16& a) -{ return (uint)v_reduce_sum(v_reinterpret_as_s32(a)); } - -#define OPENCV_HAL_IMPL_AVX512_REDUCE_32(sctype, func, _Tpvec, ifunc) \ - inline sctype v_reduce_##func(const _Tpvec& a) \ - { __m256i half = _mm256_##ifunc(_v512_extract_low(a.val), _v512_extract_high(a.val)); \ - __m128i quarter = _mm_##ifunc(_mm256_castsi256_si128(half), _mm256_extracti128_si256(half, 1)); \ - quarter = _mm_##ifunc(quarter, _mm_srli_si128(quarter, 8)); \ - quarter = _mm_##ifunc(quarter, _mm_srli_si128(quarter, 4)); \ - quarter = _mm_##ifunc(quarter, _mm_srli_si128(quarter, 2)); \ - return (sctype)_mm_cvtsi128_si32(quarter); } -OPENCV_HAL_IMPL_AVX512_REDUCE_32(ushort, min, v_uint16x32, min_epu16) -OPENCV_HAL_IMPL_AVX512_REDUCE_32(ushort, max, v_uint16x32, max_epu16) -OPENCV_HAL_IMPL_AVX512_REDUCE_32(short, min, v_int16x32, min_epi16) -OPENCV_HAL_IMPL_AVX512_REDUCE_32(short, max, v_int16x32, max_epi16) - -inline int v_reduce_sum(const v_int16x32& a) -{ return v_reduce_sum(v_expand_low(a) + v_expand_high(a)); } -inline uint v_reduce_sum(const v_uint16x32& a) -{ return v_reduce_sum(v_expand_low(a) + v_expand_high(a)); } - -#define OPENCV_HAL_IMPL_AVX512_REDUCE_64(sctype, func, _Tpvec, ifunc) \ - inline sctype v_reduce_##func(const _Tpvec& a) \ - { __m256i half = _mm256_##ifunc(_v512_extract_low(a.val), _v512_extract_high(a.val)); \ - __m128i quarter = _mm_##ifunc(_mm256_castsi256_si128(half), _mm256_extracti128_si256(half, 1)); \ - quarter = _mm_##ifunc(quarter, _mm_srli_si128(quarter, 8)); \ - quarter = _mm_##ifunc(quarter, _mm_srli_si128(quarter, 4)); \ - quarter = _mm_##ifunc(quarter, _mm_srli_si128(quarter, 2)); \ - quarter = _mm_##ifunc(quarter, _mm_srli_si128(quarter, 1)); \ - return (sctype)_mm_cvtsi128_si32(quarter); } -OPENCV_HAL_IMPL_AVX512_REDUCE_64(uchar, min, v_uint8x64, min_epu8) -OPENCV_HAL_IMPL_AVX512_REDUCE_64(uchar, max, v_uint8x64, max_epu8) -OPENCV_HAL_IMPL_AVX512_REDUCE_64(schar, min, v_int8x64, min_epi8) -OPENCV_HAL_IMPL_AVX512_REDUCE_64(schar, max, v_int8x64, max_epi8) - -#define OPENCV_HAL_IMPL_AVX512_REDUCE_64_SUM(sctype, _Tpvec, suffix) \ - inline sctype v_reduce_sum(const _Tpvec& a) \ - { __m512i a16 = _mm512_add_epi16(_mm512_cvt##suffix##_epi16(_v512_extract_low(a.val)), \ - _mm512_cvt##suffix##_epi16(_v512_extract_high(a.val))); \ - a16 = _mm512_cvtepi16_epi32(_mm256_add_epi16(_v512_extract_low(a16), _v512_extract_high(a16))); \ - __m256i a8 = _mm256_add_epi32(_v512_extract_low(a16), _v512_extract_high(a16)); \ - __m128i a4 = _mm_add_epi32(_mm256_castsi256_si128(a8), _mm256_extracti128_si256(a8, 1)); \ - a4 = _mm_hadd_epi32(a4, a4); \ - return (sctype)_mm_cvtsi128_si32(_mm_hadd_epi32(a4, a4)); } -OPENCV_HAL_IMPL_AVX512_REDUCE_64_SUM(uint, v_uint8x64, epu8) -OPENCV_HAL_IMPL_AVX512_REDUCE_64_SUM(int, v_int8x64, epi8) - -inline v_float32x16 v_reduce_sum4(const v_float32x16& a, const v_float32x16& b, - const v_float32x16& c, const v_float32x16& d) -{ - __m256 abl = _mm256_hadd_ps(_v512_extract_low(a.val), _v512_extract_low(b.val)); - __m256 abh = _mm256_hadd_ps(_v512_extract_high(a.val), _v512_extract_high(b.val)); - __m256 cdl = _mm256_hadd_ps(_v512_extract_low(c.val), _v512_extract_low(d.val)); - __m256 cdh = _mm256_hadd_ps(_v512_extract_high(c.val), _v512_extract_high(d.val)); - return v_float32x16(_v512_combine(_mm256_hadd_ps(abl, cdl), _mm256_hadd_ps(abh, cdh))); -} - -inline unsigned v_reduce_sad(const v_uint8x64& a, const v_uint8x64& b) -{ - __m512i val = _mm512_sad_epu8(a.val, b.val); - __m256i half = _mm256_add_epi32(_v512_extract_low(val), _v512_extract_high(val)); - __m128i quarter = _mm_add_epi32(_mm256_castsi256_si128(half), _mm256_extracti128_si256(half, 1)); - return (unsigned)_mm_cvtsi128_si32(_mm_add_epi32(quarter, _mm_unpackhi_epi64(quarter, quarter))); -} -inline unsigned v_reduce_sad(const v_int8x64& a, const v_int8x64& b) -{ - __m512i val = _mm512_set1_epi8(-128); - val = _mm512_sad_epu8(_mm512_add_epi8(a.val, val), _mm512_add_epi8(b.val, val)); - __m256i half = _mm256_add_epi32(_v512_extract_low(val), _v512_extract_high(val)); - __m128i quarter = _mm_add_epi32(_mm256_castsi256_si128(half), _mm256_extracti128_si256(half, 1)); - return (unsigned)_mm_cvtsi128_si32(_mm_add_epi32(quarter, _mm_unpackhi_epi64(quarter, quarter))); -} -inline unsigned v_reduce_sad(const v_uint16x32& a, const v_uint16x32& b) -{ return v_reduce_sum(v_add_wrap(a - b, b - a)); } -inline unsigned v_reduce_sad(const v_int16x32& a, const v_int16x32& b) -{ return v_reduce_sum(v_reinterpret_as_u16(v_sub_wrap(v_max(a, b), v_min(a, b)))); } -inline unsigned v_reduce_sad(const v_uint32x16& a, const v_uint32x16& b) -{ return v_reduce_sum(v_max(a, b) - v_min(a, b)); } -inline unsigned v_reduce_sad(const v_int32x16& a, const v_int32x16& b) -{ return v_reduce_sum(v_reinterpret_as_u32(v_max(a, b) - v_min(a, b))); } -inline float v_reduce_sad(const v_float32x16& a, const v_float32x16& b) -{ return v_reduce_sum((a - b) & v_float32x16(_mm512_castsi512_ps(_mm512_set1_epi32(0x7fffffff)))); } -inline double v_reduce_sad(const v_float64x8& a, const v_float64x8& b) -{ return v_reduce_sum((a - b) & v_float64x8(_mm512_castsi512_pd(_mm512_set1_epi64(0x7fffffffffffffff)))); } - -/** Popcount **/ -inline v_uint8x64 v_popcount(const v_int8x64& a) -{ -#if CV_AVX_512BITALG - return v_uint8x64(_mm512_popcnt_epi8(a.val)); -#elif CV_AVX_512VBMI - __m512i _popcnt_table0 = _v512_set_epu8(7, 6, 6, 5, 6, 5, 5, 4, 6, 5, 5, 4, 5, 4, 4, 3, - 5, 4, 4, 3, 4, 3, 3, 2, 4, 3, 3, 2, 3, 2, 2, 1, - 5, 4, 4, 3, 4, 3, 3, 2, 4, 3, 3, 2, 3, 2, 2, 1, - 4, 3, 3, 2, 3, 2, 2, 1, 3, 2, 2, 1, 2, 1, 1, 0); - __m512i _popcnt_table1 = _v512_set_epu8(7, 6, 6, 5, 6, 5, 5, 4, 6, 5, 5, 4, 5, 4, 4, 3, - 6, 5, 5, 4, 5, 4, 4, 3, 5, 4, 4, 3, 4, 3, 3, 2, - 6, 5, 5, 4, 5, 4, 4, 3, 5, 4, 4, 3, 4, 3, 3, 2, - 5, 4, 4, 3, 4, 3, 3, 2, 4, 3, 3, 2, 3, 2, 2, 1); - return v_uint8x64(_mm512_sub_epi8(_mm512_permutex2var_epi8(_popcnt_table0, a.val, _popcnt_table1), _mm512_movm_epi8(_mm512_movepi8_mask(a.val)))); -#else - __m512i _popcnt_table = _mm512_set4_epi32(0x04030302, 0x03020201, 0x03020201, 0x02010100); - __m512i _popcnt_mask = _mm512_set1_epi8(0x0F); - - return v_uint8x64(_mm512_add_epi8(_mm512_shuffle_epi8(_popcnt_table, _mm512_and_si512( a.val, _popcnt_mask)), - _mm512_shuffle_epi8(_popcnt_table, _mm512_and_si512(_mm512_srli_epi16(a.val, 4), _popcnt_mask)))); -#endif -} -inline v_uint16x32 v_popcount(const v_int16x32& a) -{ -#if CV_AVX_512BITALG - return v_uint16x32(_mm512_popcnt_epi16(a.val)); -#elif CV_AVX_512VPOPCNTDQ - __m512i zero = _mm512_setzero_si512(); - return v_uint16x32(_mm512_packs_epi32(_mm512_popcnt_epi32(_mm512_unpacklo_epi16(a.val, zero)), - _mm512_popcnt_epi32(_mm512_unpackhi_epi16(a.val, zero)))); -#else - v_uint8x64 p = v_popcount(v_reinterpret_as_s8(a)); - p += v_rotate_right<1>(p); - return v_reinterpret_as_u16(p) & v512_setall_u16(0x00ff); -#endif -} -inline v_uint32x16 v_popcount(const v_int32x16& a) -{ -#if CV_AVX_512VPOPCNTDQ - return v_uint32x16(_mm512_popcnt_epi32(a.val)); -#else - v_uint8x64 p = v_popcount(v_reinterpret_as_s8(a)); - p += v_rotate_right<1>(p); - p += v_rotate_right<2>(p); - return v_reinterpret_as_u32(p) & v512_setall_u32(0x000000ff); -#endif -} -inline v_uint64x8 v_popcount(const v_int64x8& a) -{ -#if CV_AVX_512VPOPCNTDQ - return v_uint64x8(_mm512_popcnt_epi64(a.val)); -#else - return v_uint64x8(_mm512_sad_epu8(v_popcount(v_reinterpret_as_s8(a)).val, _mm512_setzero_si512())); -#endif -} - - -inline v_uint8x64 v_popcount(const v_uint8x64& a) { return v_popcount(v_reinterpret_as_s8 (a)); } -inline v_uint16x32 v_popcount(const v_uint16x32& a) { return v_popcount(v_reinterpret_as_s16(a)); } -inline v_uint32x16 v_popcount(const v_uint32x16& a) { return v_popcount(v_reinterpret_as_s32(a)); } -inline v_uint64x8 v_popcount(const v_uint64x8& a) { return v_popcount(v_reinterpret_as_s64(a)); } - - -////////// Other math ///////// - -/** Some frequent operations **/ -#define OPENCV_HAL_IMPL_AVX512_MULADD(_Tpvec, suffix) \ - inline _Tpvec v_fma(const _Tpvec& a, const _Tpvec& b, const _Tpvec& c) \ - { return _Tpvec(_mm512_fmadd_##suffix(a.val, b.val, c.val)); } \ - inline _Tpvec v_muladd(const _Tpvec& a, const _Tpvec& b, const _Tpvec& c) \ - { return _Tpvec(_mm512_fmadd_##suffix(a.val, b.val, c.val)); } \ - inline _Tpvec v_sqrt(const _Tpvec& x) \ - { return _Tpvec(_mm512_sqrt_##suffix(x.val)); } \ - inline _Tpvec v_sqr_magnitude(const _Tpvec& a, const _Tpvec& b) \ - { return v_fma(a, a, b * b); } \ - inline _Tpvec v_magnitude(const _Tpvec& a, const _Tpvec& b) \ - { return v_sqrt(v_fma(a, a, b * b)); } - -OPENCV_HAL_IMPL_AVX512_MULADD(v_float32x16, ps) -OPENCV_HAL_IMPL_AVX512_MULADD(v_float64x8, pd) - -inline v_int32x16 v_fma(const v_int32x16& a, const v_int32x16& b, const v_int32x16& c) -{ return a * b + c; } -inline v_int32x16 v_muladd(const v_int32x16& a, const v_int32x16& b, const v_int32x16& c) -{ return v_fma(a, b, c); } - -inline v_float32x16 v_invsqrt(const v_float32x16& x) -{ -#if CV_AVX_512ER - return v_float32x16(_mm512_rsqrt28_ps(x.val)); -#else - v_float32x16 half = x * v512_setall_f32(0.5); - v_float32x16 t = v_float32x16(_mm512_rsqrt14_ps(x.val)); - t *= v512_setall_f32(1.5) - ((t * t) * half); - return t; -#endif -} - -inline v_float64x8 v_invsqrt(const v_float64x8& x) -{ -#if CV_AVX_512ER - return v_float64x8(_mm512_rsqrt28_pd(x.val)); -#else - return v512_setall_f64(1.) / v_sqrt(x); -// v_float64x8 half = x * v512_setall_f64(0.5); -// v_float64x8 t = v_float64x8(_mm512_rsqrt14_pd(x.val)); -// t *= v512_setall_f64(1.5) - ((t * t) * half); -// t *= v512_setall_f64(1.5) - ((t * t) * half); -// return t; -#endif -} - -/** Absolute values **/ -#define OPENCV_HAL_IMPL_AVX512_ABS(_Tpvec, _Tpuvec, suffix) \ - inline _Tpuvec v_abs(const _Tpvec& x) \ - { return _Tpuvec(_mm512_abs_##suffix(x.val)); } - -OPENCV_HAL_IMPL_AVX512_ABS(v_int8x64, v_uint8x64, epi8) -OPENCV_HAL_IMPL_AVX512_ABS(v_int16x32, v_uint16x32, epi16) -OPENCV_HAL_IMPL_AVX512_ABS(v_int32x16, v_uint32x16, epi32) -OPENCV_HAL_IMPL_AVX512_ABS(v_int64x8, v_uint64x8, epi64) - -inline v_float32x16 v_abs(const v_float32x16& x) -{ -#ifdef _mm512_abs_pd - return v_float32x16(_mm512_abs_ps(x.val)); -#else - return v_float32x16(_mm512_castsi512_ps(_mm512_and_si512(_mm512_castps_si512(x.val), - _v512_set_epu64(0x7FFFFFFF7FFFFFFF, 0x7FFFFFFF7FFFFFFF, 0x7FFFFFFF7FFFFFFF, 0x7FFFFFFF7FFFFFFF, - 0x7FFFFFFF7FFFFFFF, 0x7FFFFFFF7FFFFFFF, 0x7FFFFFFF7FFFFFFF, 0x7FFFFFFF7FFFFFFF)))); -#endif -} - -inline v_float64x8 v_abs(const v_float64x8& x) -{ -#ifdef _mm512_abs_pd - #if defined __GNUC__ && (__GNUC__ < 7 || (__GNUC__ == 7 && __GNUC_MINOR__ <= 3) || (__GNUC__ == 8 && __GNUC_MINOR__ <= 2)) - // Workaround for https://gcc.gnu.org/bugzilla/show_bug.cgi?id=87476 - return v_float64x8(_mm512_abs_pd(_mm512_castpd_ps(x.val))); - #else - return v_float64x8(_mm512_abs_pd(x.val)); - #endif -#else - return v_float64x8(_mm512_castsi512_pd(_mm512_and_si512(_mm512_castpd_si512(x.val), - _v512_set_epu64(0x7FFFFFFFFFFFFFFF, 0x7FFFFFFFFFFFFFFF, 0x7FFFFFFFFFFFFFFF, 0x7FFFFFFFFFFFFFFF, - 0x7FFFFFFFFFFFFFFF, 0x7FFFFFFFFFFFFFFF, 0x7FFFFFFFFFFFFFFF, 0x7FFFFFFFFFFFFFFF)))); -#endif -} - -/** Absolute difference **/ -inline v_uint8x64 v_absdiff(const v_uint8x64& a, const v_uint8x64& b) -{ return v_add_wrap(a - b, b - a); } -inline v_uint16x32 v_absdiff(const v_uint16x32& a, const v_uint16x32& b) -{ return v_add_wrap(a - b, b - a); } -inline v_uint32x16 v_absdiff(const v_uint32x16& a, const v_uint32x16& b) -{ return v_max(a, b) - v_min(a, b); } - -inline v_uint8x64 v_absdiff(const v_int8x64& a, const v_int8x64& b) -{ - v_int8x64 d = v_sub_wrap(a, b); - v_int8x64 m = a < b; - return v_reinterpret_as_u8(v_sub_wrap(d ^ m, m)); -} - -inline v_uint16x32 v_absdiff(const v_int16x32& a, const v_int16x32& b) -{ return v_reinterpret_as_u16(v_sub_wrap(v_max(a, b), v_min(a, b))); } - -inline v_uint32x16 v_absdiff(const v_int32x16& a, const v_int32x16& b) -{ - v_int32x16 d = a - b; - v_int32x16 m = a < b; - return v_reinterpret_as_u32((d ^ m) - m); -} - -inline v_float32x16 v_absdiff(const v_float32x16& a, const v_float32x16& b) -{ return v_abs(a - b); } - -inline v_float64x8 v_absdiff(const v_float64x8& a, const v_float64x8& b) -{ return v_abs(a - b); } - -/** Saturating absolute difference **/ -inline v_int8x64 v_absdiffs(const v_int8x64& a, const v_int8x64& b) -{ - v_int8x64 d = a - b; - v_int8x64 m = a < b; - return (d ^ m) - m; -} -inline v_int16x32 v_absdiffs(const v_int16x32& a, const v_int16x32& b) -{ return v_max(a, b) - v_min(a, b); } - -////////// Conversions ///////// - -/** Rounding **/ -inline v_int32x16 v_round(const v_float32x16& a) -{ return v_int32x16(_mm512_cvtps_epi32(a.val)); } - -inline v_int32x16 v_round(const v_float64x8& a) -{ return v_int32x16(_mm512_castsi256_si512(_mm512_cvtpd_epi32(a.val))); } - -inline v_int32x16 v_round(const v_float64x8& a, const v_float64x8& b) -{ return v_int32x16(_v512_combine(_mm512_cvtpd_epi32(a.val), _mm512_cvtpd_epi32(b.val))); } - -inline v_int32x16 v_trunc(const v_float32x16& a) -{ return v_int32x16(_mm512_cvttps_epi32(a.val)); } - -inline v_int32x16 v_trunc(const v_float64x8& a) -{ return v_int32x16(_mm512_castsi256_si512(_mm512_cvttpd_epi32(a.val))); } - -#if CVT_ROUND_MODES_IMPLEMENTED -inline v_int32x16 v_floor(const v_float32x16& a) -{ return v_int32x16(_mm512_cvt_roundps_epi32(a.val, _MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC)); } - -inline v_int32x16 v_floor(const v_float64x8& a) -{ return v_int32x16(_mm512_castsi256_si512(_mm512_cvt_roundpd_epi32(a.val, _MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC))); } - -inline v_int32x16 v_ceil(const v_float32x16& a) -{ return v_int32x16(_mm512_cvt_roundps_epi32(a.val, _MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC)); } - -inline v_int32x16 v_ceil(const v_float64x8& a) -{ return v_int32x16(_mm512_castsi256_si512(_mm512_cvt_roundpd_epi32(a.val, _MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC))); } -#else -inline v_int32x16 v_floor(const v_float32x16& a) -{ return v_int32x16(_mm512_cvtps_epi32(_mm512_roundscale_ps(a.val, 1))); } - -inline v_int32x16 v_floor(const v_float64x8& a) -{ return v_int32x16(_mm512_castsi256_si512(_mm512_cvtpd_epi32(_mm512_roundscale_pd(a.val, 1)))); } - -inline v_int32x16 v_ceil(const v_float32x16& a) -{ return v_int32x16(_mm512_cvtps_epi32(_mm512_roundscale_ps(a.val, 2))); } - -inline v_int32x16 v_ceil(const v_float64x8& a) -{ return v_int32x16(_mm512_castsi256_si512(_mm512_cvtpd_epi32(_mm512_roundscale_pd(a.val, 2)))); } -#endif - -/** To float **/ -inline v_float32x16 v_cvt_f32(const v_int32x16& a) -{ return v_float32x16(_mm512_cvtepi32_ps(a.val)); } - -inline v_float32x16 v_cvt_f32(const v_float64x8& a) -{ return v_float32x16(_mm512_cvtpd_pslo(a.val)); } - -inline v_float32x16 v_cvt_f32(const v_float64x8& a, const v_float64x8& b) -{ return v_float32x16(_v512_combine(_mm512_cvtpd_ps(a.val), _mm512_cvtpd_ps(b.val))); } - -inline v_float64x8 v_cvt_f64(const v_int32x16& a) -{ return v_float64x8(_mm512_cvtepi32_pd(_v512_extract_low(a.val))); } - -inline v_float64x8 v_cvt_f64_high(const v_int32x16& a) -{ return v_float64x8(_mm512_cvtepi32_pd(_v512_extract_high(a.val))); } - -inline v_float64x8 v_cvt_f64(const v_float32x16& a) -{ return v_float64x8(_mm512_cvtps_pd(_v512_extract_low(a.val))); } - -inline v_float64x8 v_cvt_f64_high(const v_float32x16& a) -{ return v_float64x8(_mm512_cvtps_pd(_v512_extract_high(a.val))); } - -// from (Mysticial and wim) https://stackoverflow.com/q/41144668 -inline v_float64x8 v_cvt_f64(const v_int64x8& v) -{ -#if CV_AVX_512DQ - return v_float64x8(_mm512_cvtepi64_pd(v.val)); -#else - // constants encoded as floating-point - __m512i magic_i_lo = _mm512_set1_epi64(0x4330000000000000); // 2^52 - __m512i magic_i_hi32 = _mm512_set1_epi64(0x4530000080000000); // 2^84 + 2^63 - __m512i magic_i_all = _mm512_set1_epi64(0x4530000080100000); // 2^84 + 2^63 + 2^52 - __m512d magic_d_all = _mm512_castsi512_pd(magic_i_all); - - // Blend the 32 lowest significant bits of v with magic_int_lo - __m512i v_lo = _mm512_mask_blend_epi32 (0x55, magic_i_lo, v.val); - // Extract the 32 most significant bits of v - __m512i v_hi = _mm512_srli_epi64(v.val, 32); - // Flip the msb of v_hi and blend with 0x45300000 - v_hi = _mm512_xor_si512(v_hi, magic_i_hi32); - // Compute in double precision - __m512d v_hi_dbl = _mm512_sub_pd(_mm512_castsi512_pd(v_hi), magic_d_all); - // (v_hi - magic_d_all) + v_lo Do not assume associativity of floating point addition - __m512d result = _mm512_add_pd(v_hi_dbl, _mm512_castsi512_pd(v_lo)); - return v_float64x8(result); -#endif -} - -////////////// Lookup table access //////////////////// - -inline v_int8x64 v512_lut(const schar* tab, const int* idx) -{ - __m128i p0 = _mm512_cvtepi32_epi8(_mm512_i32gather_epi32(_mm512_loadu_si512((const __m512i*)idx ), (const int *)tab, 1)); - __m128i p1 = _mm512_cvtepi32_epi8(_mm512_i32gather_epi32(_mm512_loadu_si512((const __m512i*)idx + 1), (const int *)tab, 1)); - __m128i p2 = _mm512_cvtepi32_epi8(_mm512_i32gather_epi32(_mm512_loadu_si512((const __m512i*)idx + 2), (const int *)tab, 1)); - __m128i p3 = _mm512_cvtepi32_epi8(_mm512_i32gather_epi32(_mm512_loadu_si512((const __m512i*)idx + 3), (const int *)tab, 1)); - return v_int8x64(_mm512_inserti32x4(_mm512_inserti32x4(_mm512_inserti32x4(_mm512_castsi128_si512(p0), p1, 1), p2, 2), p3, 3)); -} -inline v_int8x64 v512_lut_pairs(const schar* tab, const int* idx) -{ - __m256i p0 = _mm512_cvtepi32_epi16(_mm512_i32gather_epi32(_mm512_loadu_si512((const __m512i*)idx ), (const int *)tab, 1)); - __m256i p1 = _mm512_cvtepi32_epi16(_mm512_i32gather_epi32(_mm512_loadu_si512((const __m512i*)idx + 1), (const int *)tab, 1)); - return v_int8x64(_v512_combine(p0, p1)); -} -inline v_int8x64 v512_lut_quads(const schar* tab, const int* idx) -{ - return v_int8x64(_mm512_i32gather_epi32(_mm512_loadu_si512((const __m512i*)idx), (const int *)tab, 1)); -} -inline v_uint8x64 v512_lut(const uchar* tab, const int* idx) { return v_reinterpret_as_u8(v512_lut((const schar *)tab, idx)); } -inline v_uint8x64 v512_lut_pairs(const uchar* tab, const int* idx) { return v_reinterpret_as_u8(v512_lut_pairs((const schar *)tab, idx)); } -inline v_uint8x64 v512_lut_quads(const uchar* tab, const int* idx) { return v_reinterpret_as_u8(v512_lut_quads((const schar *)tab, idx)); } - -inline v_int16x32 v512_lut(const short* tab, const int* idx) -{ - __m256i p0 = _mm512_cvtepi32_epi16(_mm512_i32gather_epi32(_mm512_loadu_si512((const __m512i*)idx ), (const int *)tab, 2)); - __m256i p1 = _mm512_cvtepi32_epi16(_mm512_i32gather_epi32(_mm512_loadu_si512((const __m512i*)idx + 1), (const int *)tab, 2)); - return v_int16x32(_v512_combine(p0, p1)); -} -inline v_int16x32 v512_lut_pairs(const short* tab, const int* idx) -{ - return v_int16x32(_mm512_i32gather_epi32(_mm512_loadu_si512((const __m512i*)idx), (const int *)tab, 2)); -} -inline v_int16x32 v512_lut_quads(const short* tab, const int* idx) -{ -#if defined(__GNUC__) - return v_int16x32(_mm512_i32gather_epi64(_mm256_loadu_si256((const __m256i*)idx), (const long long int*)tab, 2)); -#else - return v_int16x32(_mm512_i32gather_epi64(_mm256_loadu_si256((const __m256i*)idx), (const int64*)tab, 2)); -#endif -} -inline v_uint16x32 v512_lut(const ushort* tab, const int* idx) { return v_reinterpret_as_u16(v512_lut((const short *)tab, idx)); } -inline v_uint16x32 v512_lut_pairs(const ushort* tab, const int* idx) { return v_reinterpret_as_u16(v512_lut_pairs((const short *)tab, idx)); } -inline v_uint16x32 v512_lut_quads(const ushort* tab, const int* idx) { return v_reinterpret_as_u16(v512_lut_quads((const short *)tab, idx)); } - -inline v_int32x16 v512_lut(const int* tab, const int* idx) -{ - return v_int32x16(_mm512_i32gather_epi32(_mm512_loadu_si512((const __m512i*)idx), tab, 4)); -} -inline v_int32x16 v512_lut_pairs(const int* tab, const int* idx) -{ -#if defined(__GNUC__) - return v_int32x16(_mm512_i32gather_epi64(_mm256_loadu_si256((const __m256i*)idx), (const long long int*)tab, 4)); -#else - return v_int32x16(_mm512_i32gather_epi64(_mm256_loadu_si256((const __m256i*)idx), (const int64*)tab, 4)); -#endif -} -inline v_int32x16 v512_lut_quads(const int* tab, const int* idx) -{ - return v_int32x16(_mm512_inserti32x4(_mm512_inserti32x4(_mm512_inserti32x4(_mm512_castsi128_si512( - _mm_loadu_si128((const __m128i*)(tab + idx[0]))), - _mm_loadu_si128((const __m128i*)(tab + idx[1])), 1), - _mm_loadu_si128((const __m128i*)(tab + idx[2])), 2), - _mm_loadu_si128((const __m128i*)(tab + idx[3])), 3)); -} -inline v_uint32x16 v512_lut(const unsigned* tab, const int* idx) { return v_reinterpret_as_u32(v512_lut((const int *)tab, idx)); } -inline v_uint32x16 v512_lut_pairs(const unsigned* tab, const int* idx) { return v_reinterpret_as_u32(v512_lut_pairs((const int *)tab, idx)); } -inline v_uint32x16 v512_lut_quads(const unsigned* tab, const int* idx) { return v_reinterpret_as_u32(v512_lut_quads((const int *)tab, idx)); } - -inline v_int64x8 v512_lut(const int64* tab, const int* idx) -{ -#if defined(__GNUC__) - return v_int64x8(_mm512_i32gather_epi64(_mm256_loadu_si256((const __m256i*)idx), (const long long int*)tab, 8)); -#else - return v_int64x8(_mm512_i32gather_epi64(_mm256_loadu_si256((const __m256i*)idx), tab , 8)); -#endif -} -inline v_int64x8 v512_lut_pairs(const int64* tab, const int* idx) -{ - return v_int64x8(_mm512_inserti32x4(_mm512_inserti32x4(_mm512_inserti32x4(_mm512_castsi128_si512( - _mm_loadu_si128((const __m128i*)(tab + idx[0]))), - _mm_loadu_si128((const __m128i*)(tab + idx[1])), 1), - _mm_loadu_si128((const __m128i*)(tab + idx[2])), 2), - _mm_loadu_si128((const __m128i*)(tab + idx[3])), 3)); -} -inline v_uint64x8 v512_lut(const uint64* tab, const int* idx) { return v_reinterpret_as_u64(v512_lut((const int64 *)tab, idx)); } -inline v_uint64x8 v512_lut_pairs(const uint64* tab, const int* idx) { return v_reinterpret_as_u64(v512_lut_pairs((const int64 *)tab, idx)); } - -inline v_float32x16 v512_lut(const float* tab, const int* idx) -{ - return v_float32x16(_mm512_i32gather_ps(_mm512_loadu_si512((const __m512i*)idx), tab, 4)); -} -inline v_float32x16 v512_lut_pairs(const float* tab, const int* idx) { return v_reinterpret_as_f32(v512_lut_pairs((const int *)tab, idx)); } -inline v_float32x16 v512_lut_quads(const float* tab, const int* idx) { return v_reinterpret_as_f32(v512_lut_quads((const int *)tab, idx)); } - -inline v_float64x8 v512_lut(const double* tab, const int* idx) -{ - return v_float64x8(_mm512_i32gather_pd(_mm256_loadu_si256((const __m256i*)idx), tab, 8)); -} -inline v_float64x8 v512_lut_pairs(const double* tab, const int* idx) -{ - return v_float64x8(_mm512_insertf64x2(_mm512_insertf64x2(_mm512_insertf64x2(_mm512_castpd128_pd512( - _mm_loadu_pd(tab + idx[0])), - _mm_loadu_pd(tab + idx[1]), 1), - _mm_loadu_pd(tab + idx[2]), 2), - _mm_loadu_pd(tab + idx[3]), 3)); -} - -inline v_int32x16 v_lut(const int* tab, const v_int32x16& idxvec) -{ - return v_int32x16(_mm512_i32gather_epi32(idxvec.val, tab, 4)); -} - -inline v_uint32x16 v_lut(const unsigned* tab, const v_int32x16& idxvec) -{ - return v_reinterpret_as_u32(v_lut((const int *)tab, idxvec)); -} - -inline v_float32x16 v_lut(const float* tab, const v_int32x16& idxvec) -{ - return v_float32x16(_mm512_i32gather_ps(idxvec.val, tab, 4)); -} - -inline v_float64x8 v_lut(const double* tab, const v_int32x16& idxvec) -{ - return v_float64x8(_mm512_i32gather_pd(_v512_extract_low(idxvec.val), tab, 8)); -} - -inline void v_lut_deinterleave(const float* tab, const v_int32x16& idxvec, v_float32x16& x, v_float32x16& y) -{ - x.val = _mm512_i32gather_ps(idxvec.val, tab, 4); - y.val = _mm512_i32gather_ps(idxvec.val, &tab[1], 4); -} - -inline void v_lut_deinterleave(const double* tab, const v_int32x16& idxvec, v_float64x8& x, v_float64x8& y) -{ - x.val = _mm512_i32gather_pd(_v512_extract_low(idxvec.val), tab, 8); - y.val = _mm512_i32gather_pd(_v512_extract_low(idxvec.val), &tab[1], 8); -} - -inline v_int8x64 v_interleave_pairs(const v_int8x64& vec) -{ - return v_int8x64(_mm512_shuffle_epi8(vec.val, _mm512_set4_epi32(0x0f0d0e0c, 0x0b090a08, 0x07050604, 0x03010200))); -} -inline v_uint8x64 v_interleave_pairs(const v_uint8x64& vec) { return v_reinterpret_as_u8(v_interleave_pairs(v_reinterpret_as_s8(vec))); } -inline v_int8x64 v_interleave_quads(const v_int8x64& vec) -{ - return v_int8x64(_mm512_shuffle_epi8(vec.val, _mm512_set4_epi32(0x0f0b0e0a, 0x0d090c08, 0x07030602, 0x05010400))); -} -inline v_uint8x64 v_interleave_quads(const v_uint8x64& vec) { return v_reinterpret_as_u8(v_interleave_quads(v_reinterpret_as_s8(vec))); } - -inline v_int16x32 v_interleave_pairs(const v_int16x32& vec) -{ - return v_int16x32(_mm512_shuffle_epi8(vec.val, _mm512_set4_epi32(0x0f0e0b0a, 0x0d0c0908, 0x07060302, 0x05040100))); -} -inline v_uint16x32 v_interleave_pairs(const v_uint16x32& vec) { return v_reinterpret_as_u16(v_interleave_pairs(v_reinterpret_as_s16(vec))); } -inline v_int16x32 v_interleave_quads(const v_int16x32& vec) -{ - return v_int16x32(_mm512_shuffle_epi8(vec.val, _mm512_set4_epi32(0x0f0e0706, 0x0d0c0504, 0x0b0a0302, 0x09080100))); -} -inline v_uint16x32 v_interleave_quads(const v_uint16x32& vec) { return v_reinterpret_as_u16(v_interleave_quads(v_reinterpret_as_s16(vec))); } - -inline v_int32x16 v_interleave_pairs(const v_int32x16& vec) -{ - return v_int32x16(_mm512_shuffle_epi32(vec.val, _MM_PERM_ACBD)); -} -inline v_uint32x16 v_interleave_pairs(const v_uint32x16& vec) { return v_reinterpret_as_u32(v_interleave_pairs(v_reinterpret_as_s32(vec))); } -inline v_float32x16 v_interleave_pairs(const v_float32x16& vec) { return v_reinterpret_as_f32(v_interleave_pairs(v_reinterpret_as_s32(vec))); } - -inline v_int8x64 v_pack_triplets(const v_int8x64& vec) -{ - return v_int8x64(_mm512_permutexvar_epi32(_v512_set_epu64(0x0000000f0000000f, 0x0000000f0000000f, 0x0000000e0000000d, 0x0000000c0000000a, - 0x0000000900000008, 0x0000000600000005, 0x0000000400000002, 0x0000000100000000), - _mm512_shuffle_epi8(vec.val, _mm512_set4_epi32(0xffffff0f, 0x0e0d0c0a, 0x09080605, 0x04020100)))); -} -inline v_uint8x64 v_pack_triplets(const v_uint8x64& vec) { return v_reinterpret_as_u8(v_pack_triplets(v_reinterpret_as_s8(vec))); } - -inline v_int16x32 v_pack_triplets(const v_int16x32& vec) -{ - return v_int16x32(_mm512_permutexvar_epi16(_v512_set_epu64(0x001f001f001f001f, 0x001f001f001f001f, 0x001e001d001c001a, 0x0019001800160015, - 0x0014001200110010, 0x000e000d000c000a, 0x0009000800060005, 0x0004000200010000), vec.val)); -} -inline v_uint16x32 v_pack_triplets(const v_uint16x32& vec) { return v_reinterpret_as_u16(v_pack_triplets(v_reinterpret_as_s16(vec))); } - -inline v_int32x16 v_pack_triplets(const v_int32x16& vec) -{ - return v_int32x16(_mm512_permutexvar_epi32(_v512_set_epu64(0x0000000f0000000f, 0x0000000f0000000f, 0x0000000e0000000d, 0x0000000c0000000a, - 0x0000000900000008, 0x0000000600000005, 0x0000000400000002, 0x0000000100000000), vec.val)); -} -inline v_uint32x16 v_pack_triplets(const v_uint32x16& vec) { return v_reinterpret_as_u32(v_pack_triplets(v_reinterpret_as_s32(vec))); } -inline v_float32x16 v_pack_triplets(const v_float32x16& vec) -{ - return v_float32x16(_mm512_permutexvar_ps(_v512_set_epu64(0x0000000f0000000f, 0x0000000f0000000f, 0x0000000e0000000d, 0x0000000c0000000a, - 0x0000000900000008, 0x0000000600000005, 0x0000000400000002, 0x0000000100000000), vec.val)); -} - -////////// Matrix operations ///////// - -//////// Dot Product //////// - -// 16 >> 32 -inline v_int32x16 v_dotprod(const v_int16x32& a, const v_int16x32& b) -{ return v_int32x16(_mm512_madd_epi16(a.val, b.val)); } -inline v_int32x16 v_dotprod(const v_int16x32& a, const v_int16x32& b, const v_int32x16& c) -{ return v_dotprod(a, b) + c; } - -// 32 >> 64 -inline v_int64x8 v_dotprod(const v_int32x16& a, const v_int32x16& b) -{ - __m512i even = _mm512_mul_epi32(a.val, b.val); - __m512i odd = _mm512_mul_epi32(_mm512_srli_epi64(a.val, 32), _mm512_srli_epi64(b.val, 32)); - return v_int64x8(_mm512_add_epi64(even, odd)); -} -inline v_int64x8 v_dotprod(const v_int32x16& a, const v_int32x16& b, const v_int64x8& c) -{ return v_dotprod(a, b) + c; } - -// 8 >> 32 -inline v_uint32x16 v_dotprod_expand(const v_uint8x64& a, const v_uint8x64& b) -{ - __m512i even_a = _mm512_mask_blend_epi8(0xAAAAAAAAAAAAAAAA, a.val, _mm512_setzero_si512()); - __m512i odd_a = _mm512_srli_epi16(a.val, 8); - - __m512i even_b = _mm512_mask_blend_epi8(0xAAAAAAAAAAAAAAAA, b.val, _mm512_setzero_si512()); - __m512i odd_b = _mm512_srli_epi16(b.val, 8); - - __m512i prod0 = _mm512_madd_epi16(even_a, even_b); - __m512i prod1 = _mm512_madd_epi16(odd_a, odd_b); - return v_uint32x16(_mm512_add_epi32(prod0, prod1)); -} -inline v_uint32x16 v_dotprod_expand(const v_uint8x64& a, const v_uint8x64& b, const v_uint32x16& c) -{ return v_dotprod_expand(a, b) + c; } - -inline v_int32x16 v_dotprod_expand(const v_int8x64& a, const v_int8x64& b) -{ - __m512i even_a = _mm512_srai_epi16(_mm512_bslli_epi128(a.val, 1), 8); - __m512i odd_a = _mm512_srai_epi16(a.val, 8); - - __m512i even_b = _mm512_srai_epi16(_mm512_bslli_epi128(b.val, 1), 8); - __m512i odd_b = _mm512_srai_epi16(b.val, 8); - - __m512i prod0 = _mm512_madd_epi16(even_a, even_b); - __m512i prod1 = _mm512_madd_epi16(odd_a, odd_b); - return v_int32x16(_mm512_add_epi32(prod0, prod1)); -} -inline v_int32x16 v_dotprod_expand(const v_int8x64& a, const v_int8x64& b, const v_int32x16& c) -{ return v_dotprod_expand(a, b) + c; } - -// 16 >> 64 -inline v_uint64x8 v_dotprod_expand(const v_uint16x32& a, const v_uint16x32& b) -{ - __m512i mullo = _mm512_mullo_epi16(a.val, b.val); - __m512i mulhi = _mm512_mulhi_epu16(a.val, b.val); - __m512i mul0 = _mm512_unpacklo_epi16(mullo, mulhi); - __m512i mul1 = _mm512_unpackhi_epi16(mullo, mulhi); - - __m512i p02 = _mm512_mask_blend_epi32(0xAAAA, mul0, _mm512_setzero_si512()); - __m512i p13 = _mm512_srli_epi64(mul0, 32); - __m512i p46 = _mm512_mask_blend_epi32(0xAAAA, mul1, _mm512_setzero_si512()); - __m512i p57 = _mm512_srli_epi64(mul1, 32); - - __m512i p15_ = _mm512_add_epi64(p02, p13); - __m512i p9d_ = _mm512_add_epi64(p46, p57); - - return v_uint64x8(_mm512_add_epi64( - _mm512_unpacklo_epi64(p15_, p9d_), - _mm512_unpackhi_epi64(p15_, p9d_) - )); -} -inline v_uint64x8 v_dotprod_expand(const v_uint16x32& a, const v_uint16x32& b, const v_uint64x8& c) -{ return v_dotprod_expand(a, b) + c; } - -inline v_int64x8 v_dotprod_expand(const v_int16x32& a, const v_int16x32& b) -{ - __m512i prod = _mm512_madd_epi16(a.val, b.val); - __m512i even = _mm512_srai_epi64(_mm512_bslli_epi128(prod, 4), 32); - __m512i odd = _mm512_srai_epi64(prod, 32); - return v_int64x8(_mm512_add_epi64(even, odd)); -} -inline v_int64x8 v_dotprod_expand(const v_int16x32& a, const v_int16x32& b, const v_int64x8& c) -{ return v_dotprod_expand(a, b) + c; } - -// 32 >> 64f -inline v_float64x8 v_dotprod_expand(const v_int32x16& a, const v_int32x16& b) -{ return v_cvt_f64(v_dotprod(a, b)); } -inline v_float64x8 v_dotprod_expand(const v_int32x16& a, const v_int32x16& b, const v_float64x8& c) -{ return v_dotprod_expand(a, b) + c; } - -//////// Fast Dot Product //////// - -// 16 >> 32 -inline v_int32x16 v_dotprod_fast(const v_int16x32& a, const v_int16x32& b) -{ return v_dotprod(a, b); } -inline v_int32x16 v_dotprod_fast(const v_int16x32& a, const v_int16x32& b, const v_int32x16& c) -{ return v_dotprod(a, b, c); } - -// 32 >> 64 -inline v_int64x8 v_dotprod_fast(const v_int32x16& a, const v_int32x16& b) -{ return v_dotprod(a, b); } -inline v_int64x8 v_dotprod_fast(const v_int32x16& a, const v_int32x16& b, const v_int64x8& c) -{ return v_dotprod(a, b, c); } - -// 8 >> 32 -inline v_uint32x16 v_dotprod_expand_fast(const v_uint8x64& a, const v_uint8x64& b) -{ return v_dotprod_expand(a, b); } -inline v_uint32x16 v_dotprod_expand_fast(const v_uint8x64& a, const v_uint8x64& b, const v_uint32x16& c) -{ return v_dotprod_expand(a, b, c); } - -inline v_int32x16 v_dotprod_expand_fast(const v_int8x64& a, const v_int8x64& b) -{ return v_dotprod_expand(a, b); } -inline v_int32x16 v_dotprod_expand_fast(const v_int8x64& a, const v_int8x64& b, const v_int32x16& c) -{ return v_dotprod_expand(a, b, c); } - -// 16 >> 64 -inline v_uint64x8 v_dotprod_expand_fast(const v_uint16x32& a, const v_uint16x32& b) -{ - __m512i mullo = _mm512_mullo_epi16(a.val, b.val); - __m512i mulhi = _mm512_mulhi_epu16(a.val, b.val); - __m512i mul0 = _mm512_unpacklo_epi16(mullo, mulhi); - __m512i mul1 = _mm512_unpackhi_epi16(mullo, mulhi); - - __m512i p02 = _mm512_mask_blend_epi32(0xAAAA, mul0, _mm512_setzero_si512()); - __m512i p13 = _mm512_srli_epi64(mul0, 32); - __m512i p46 = _mm512_mask_blend_epi32(0xAAAA, mul1, _mm512_setzero_si512()); - __m512i p57 = _mm512_srli_epi64(mul1, 32); - - __m512i p15_ = _mm512_add_epi64(p02, p13); - __m512i p9d_ = _mm512_add_epi64(p46, p57); - return v_uint64x8(_mm512_add_epi64(p15_, p9d_)); -} -inline v_uint64x8 v_dotprod_expand_fast(const v_uint16x32& a, const v_uint16x32& b, const v_uint64x8& c) -{ return v_dotprod_expand_fast(a, b) + c; } - -inline v_int64x8 v_dotprod_expand_fast(const v_int16x32& a, const v_int16x32& b) -{ return v_dotprod_expand(a, b); } -inline v_int64x8 v_dotprod_expand_fast(const v_int16x32& a, const v_int16x32& b, const v_int64x8& c) -{ return v_dotprod_expand(a, b, c); } - -// 32 >> 64f -inline v_float64x8 v_dotprod_expand_fast(const v_int32x16& a, const v_int32x16& b) -{ return v_dotprod_expand(a, b); } -inline v_float64x8 v_dotprod_expand_fast(const v_int32x16& a, const v_int32x16& b, const v_float64x8& c) -{ return v_dotprod_expand(a, b) + c; } - - -#define OPENCV_HAL_AVX512_SPLAT2_PS(a, im) \ - v_float32x16(_mm512_permute_ps(a.val, _MM_SHUFFLE(im, im, im, im))) - -inline v_float32x16 v_matmul(const v_float32x16& v, - const v_float32x16& m0, const v_float32x16& m1, - const v_float32x16& m2, const v_float32x16& m3) -{ - v_float32x16 v04 = OPENCV_HAL_AVX512_SPLAT2_PS(v, 0); - v_float32x16 v15 = OPENCV_HAL_AVX512_SPLAT2_PS(v, 1); - v_float32x16 v26 = OPENCV_HAL_AVX512_SPLAT2_PS(v, 2); - v_float32x16 v37 = OPENCV_HAL_AVX512_SPLAT2_PS(v, 3); - return v_fma(v04, m0, v_fma(v15, m1, v_fma(v26, m2, v37 * m3))); -} - -inline v_float32x16 v_matmuladd(const v_float32x16& v, - const v_float32x16& m0, const v_float32x16& m1, - const v_float32x16& m2, const v_float32x16& a) -{ - v_float32x16 v04 = OPENCV_HAL_AVX512_SPLAT2_PS(v, 0); - v_float32x16 v15 = OPENCV_HAL_AVX512_SPLAT2_PS(v, 1); - v_float32x16 v26 = OPENCV_HAL_AVX512_SPLAT2_PS(v, 2); - return v_fma(v04, m0, v_fma(v15, m1, v_fma(v26, m2, a))); -} - -#define OPENCV_HAL_IMPL_AVX512_TRANSPOSE4x4(_Tpvec, suffix, cast_from, cast_to) \ - inline void v_transpose4x4(const _Tpvec& a0, const _Tpvec& a1, \ - const _Tpvec& a2, const _Tpvec& a3, \ - _Tpvec& b0, _Tpvec& b1, _Tpvec& b2, _Tpvec& b3) \ - { \ - __m512i t0 = cast_from(_mm512_unpacklo_##suffix(a0.val, a1.val)); \ - __m512i t1 = cast_from(_mm512_unpacklo_##suffix(a2.val, a3.val)); \ - __m512i t2 = cast_from(_mm512_unpackhi_##suffix(a0.val, a1.val)); \ - __m512i t3 = cast_from(_mm512_unpackhi_##suffix(a2.val, a3.val)); \ - b0.val = cast_to(_mm512_unpacklo_epi64(t0, t1)); \ - b1.val = cast_to(_mm512_unpackhi_epi64(t0, t1)); \ - b2.val = cast_to(_mm512_unpacklo_epi64(t2, t3)); \ - b3.val = cast_to(_mm512_unpackhi_epi64(t2, t3)); \ - } - -OPENCV_HAL_IMPL_AVX512_TRANSPOSE4x4(v_uint32x16, epi32, OPENCV_HAL_NOP, OPENCV_HAL_NOP) -OPENCV_HAL_IMPL_AVX512_TRANSPOSE4x4(v_int32x16, epi32, OPENCV_HAL_NOP, OPENCV_HAL_NOP) -OPENCV_HAL_IMPL_AVX512_TRANSPOSE4x4(v_float32x16, ps, _mm512_castps_si512, _mm512_castsi512_ps) - -//////////////// Value reordering /////////////// - -/* Expand */ -#define OPENCV_HAL_IMPL_AVX512_EXPAND(_Tpvec, _Tpwvec, _Tp, intrin) \ - inline void v_expand(const _Tpvec& a, _Tpwvec& b0, _Tpwvec& b1) \ - { \ - b0.val = intrin(_v512_extract_low(a.val)); \ - b1.val = intrin(_v512_extract_high(a.val)); \ - } \ - inline _Tpwvec v_expand_low(const _Tpvec& a) \ - { return _Tpwvec(intrin(_v512_extract_low(a.val))); } \ - inline _Tpwvec v_expand_high(const _Tpvec& a) \ - { return _Tpwvec(intrin(_v512_extract_high(a.val))); } \ - inline _Tpwvec v512_load_expand(const _Tp* ptr) \ - { \ - __m256i a = _mm256_loadu_si256((const __m256i*)ptr); \ - return _Tpwvec(intrin(a)); \ - } - -OPENCV_HAL_IMPL_AVX512_EXPAND(v_uint8x64, v_uint16x32, uchar, _mm512_cvtepu8_epi16) -OPENCV_HAL_IMPL_AVX512_EXPAND(v_int8x64, v_int16x32, schar, _mm512_cvtepi8_epi16) -OPENCV_HAL_IMPL_AVX512_EXPAND(v_uint16x32, v_uint32x16, ushort, _mm512_cvtepu16_epi32) -OPENCV_HAL_IMPL_AVX512_EXPAND(v_int16x32, v_int32x16, short, _mm512_cvtepi16_epi32) -OPENCV_HAL_IMPL_AVX512_EXPAND(v_uint32x16, v_uint64x8, unsigned, _mm512_cvtepu32_epi64) -OPENCV_HAL_IMPL_AVX512_EXPAND(v_int32x16, v_int64x8, int, _mm512_cvtepi32_epi64) - -#define OPENCV_HAL_IMPL_AVX512_EXPAND_Q(_Tpvec, _Tp, intrin) \ - inline _Tpvec v512_load_expand_q(const _Tp* ptr) \ - { \ - __m128i a = _mm_loadu_si128((const __m128i*)ptr); \ - return _Tpvec(intrin(a)); \ - } - -OPENCV_HAL_IMPL_AVX512_EXPAND_Q(v_uint32x16, uchar, _mm512_cvtepu8_epi32) -OPENCV_HAL_IMPL_AVX512_EXPAND_Q(v_int32x16, schar, _mm512_cvtepi8_epi32) - -/* pack */ -// 16 -inline v_int8x64 v_pack(const v_int16x32& a, const v_int16x32& b) -{ return v_int8x64(_mm512_mask_permutexvar_epi64(_mm512_setzero_si512(), (__mmask8)-1, _v512_set_epu64(7, 5, 3, 1, 6, 4, 2, 0), _mm512_packs_epi16(a.val, b.val))); } - -inline v_uint8x64 v_pack(const v_uint16x32& a, const v_uint16x32& b) -{ - const __m512i t = _mm512_set1_epi16(255); - return v_uint8x64(_v512_combine(_mm512_cvtepi16_epi8(_mm512_min_epu16(a.val, t)), _mm512_cvtepi16_epi8(_mm512_min_epu16(b.val, t)))); -} - -inline v_uint8x64 v_pack_u(const v_int16x32& a, const v_int16x32& b) -{ - return v_uint8x64(_mm512_mask_permutexvar_epi64(_mm512_setzero_si512(), (__mmask8)-1, _v512_set_epu64(7, 5, 3, 1, 6, 4, 2, 0), _mm512_packus_epi16(a.val, b.val))); -} - -inline void v_pack_store(schar* ptr, const v_int16x32& a) -{ v_store_low(ptr, v_pack(a, a)); } - -inline void v_pack_store(uchar* ptr, const v_uint16x32& a) -{ - const __m512i m = _mm512_set1_epi16(255); - _mm256_storeu_si256((__m256i*)ptr, _mm512_cvtepi16_epi8(_mm512_min_epu16(a.val, m))); -} - -inline void v_pack_u_store(uchar* ptr, const v_int16x32& a) -{ v_store_low(ptr, v_pack_u(a, a)); } - -template inline -v_uint8x64 v_rshr_pack(const v_uint16x32& a, const v_uint16x32& b) -{ - // we assume that n > 0, and so the shifted 16-bit values can be treated as signed numbers. - v_uint16x32 delta = v512_setall_u16((short)(1 << (n-1))); - return v_pack_u(v_reinterpret_as_s16((a + delta) >> n), - v_reinterpret_as_s16((b + delta) >> n)); -} - -template inline -void v_rshr_pack_store(uchar* ptr, const v_uint16x32& a) -{ - v_uint16x32 delta = v512_setall_u16((short)(1 << (n-1))); - v_pack_u_store(ptr, v_reinterpret_as_s16((a + delta) >> n)); -} - -template inline -v_uint8x64 v_rshr_pack_u(const v_int16x32& a, const v_int16x32& b) -{ - v_int16x32 delta = v512_setall_s16((short)(1 << (n-1))); - return v_pack_u((a + delta) >> n, (b + delta) >> n); -} - -template inline -void v_rshr_pack_u_store(uchar* ptr, const v_int16x32& a) -{ - v_int16x32 delta = v512_setall_s16((short)(1 << (n-1))); - v_pack_u_store(ptr, (a + delta) >> n); -} - -template inline -v_int8x64 v_rshr_pack(const v_int16x32& a, const v_int16x32& b) -{ - v_int16x32 delta = v512_setall_s16((short)(1 << (n-1))); - return v_pack((a + delta) >> n, (b + delta) >> n); -} - -template inline -void v_rshr_pack_store(schar* ptr, const v_int16x32& a) -{ - v_int16x32 delta = v512_setall_s16((short)(1 << (n-1))); - v_pack_store(ptr, (a + delta) >> n); -} - -// 32 -inline v_int16x32 v_pack(const v_int32x16& a, const v_int32x16& b) -{ return v_int16x32(_mm512_mask_permutexvar_epi64(_mm512_setzero_si512(), (__mmask8) -1, - _v512_set_epu64(7, 5, 3, 1, 6, 4, 2, 0), - _mm512_packs_epi32(a.val, b.val))); } - -inline v_uint16x32 v_pack(const v_uint32x16& a, const v_uint32x16& b) -{ - const __m512i m = _mm512_set1_epi32(65535); - return v_uint16x32(_v512_combine(_mm512_cvtepi32_epi16(_mm512_min_epu32(a.val, m)), _mm512_cvtepi32_epi16(_mm512_min_epu32(b.val, m)))); -} - -inline v_uint16x32 v_pack_u(const v_int32x16& a, const v_int32x16& b) -{ return v_uint16x32(_mm512_mask_permutexvar_epi64(_mm512_setzero_si512(), (__mmask8) -1, - _v512_set_epu64(7, 5, 3, 1, 6, 4, 2, 0), - _mm512_packus_epi32(a.val, b.val))); } - -inline void v_pack_store(short* ptr, const v_int32x16& a) -{ v_store_low(ptr, v_pack(a, a)); } - -inline void v_pack_store(ushort* ptr, const v_uint32x16& a) -{ - const __m512i m = _mm512_set1_epi32(65535); - _mm256_storeu_si256((__m256i*)ptr, _mm512_cvtepi32_epi16(_mm512_min_epu32(a.val, m))); -} - -inline void v_pack_u_store(ushort* ptr, const v_int32x16& a) -{ v_store_low(ptr, v_pack_u(a, a)); } - - -template inline -v_uint16x32 v_rshr_pack(const v_uint32x16& a, const v_uint32x16& b) -{ - v_uint32x16 delta = v512_setall_u32(1 << (n-1)); - return v_pack_u(v_reinterpret_as_s32((a + delta) >> n), - v_reinterpret_as_s32((b + delta) >> n)); -} - -template inline -void v_rshr_pack_store(ushort* ptr, const v_uint32x16& a) -{ - v_uint32x16 delta = v512_setall_u32(1 << (n-1)); - v_pack_u_store(ptr, v_reinterpret_as_s32((a + delta) >> n)); -} - -template inline -v_uint16x32 v_rshr_pack_u(const v_int32x16& a, const v_int32x16& b) -{ - v_int32x16 delta = v512_setall_s32(1 << (n-1)); - return v_pack_u((a + delta) >> n, (b + delta) >> n); -} - -template inline -void v_rshr_pack_u_store(ushort* ptr, const v_int32x16& a) -{ - v_int32x16 delta = v512_setall_s32(1 << (n-1)); - v_pack_u_store(ptr, (a + delta) >> n); -} - -template inline -v_int16x32 v_rshr_pack(const v_int32x16& a, const v_int32x16& b) -{ - v_int32x16 delta = v512_setall_s32(1 << (n-1)); - return v_pack((a + delta) >> n, (b + delta) >> n); -} - -template inline -void v_rshr_pack_store(short* ptr, const v_int32x16& a) -{ - v_int32x16 delta = v512_setall_s32(1 << (n-1)); - v_pack_store(ptr, (a + delta) >> n); -} - -// 64 -// Non-saturating pack -inline v_uint32x16 v_pack(const v_uint64x8& a, const v_uint64x8& b) -{ return v_uint32x16(_v512_combine(_mm512_cvtepi64_epi32(a.val), _mm512_cvtepi64_epi32(b.val))); } - -inline v_int32x16 v_pack(const v_int64x8& a, const v_int64x8& b) -{ return v_reinterpret_as_s32(v_pack(v_reinterpret_as_u64(a), v_reinterpret_as_u64(b))); } - -inline void v_pack_store(unsigned* ptr, const v_uint64x8& a) -{ _mm256_storeu_si256((__m256i*)ptr, _mm512_cvtepi64_epi32(a.val)); } - -inline void v_pack_store(int* ptr, const v_int64x8& b) -{ v_pack_store((unsigned*)ptr, v_reinterpret_as_u64(b)); } - -template inline -v_uint32x16 v_rshr_pack(const v_uint64x8& a, const v_uint64x8& b) -{ - v_uint64x8 delta = v512_setall_u64((uint64)1 << (n-1)); - return v_pack((a + delta) >> n, (b + delta) >> n); -} - -template inline -void v_rshr_pack_store(unsigned* ptr, const v_uint64x8& a) -{ - v_uint64x8 delta = v512_setall_u64((uint64)1 << (n-1)); - v_pack_store(ptr, (a + delta) >> n); -} - -template inline -v_int32x16 v_rshr_pack(const v_int64x8& a, const v_int64x8& b) -{ - v_int64x8 delta = v512_setall_s64((int64)1 << (n-1)); - return v_pack((a + delta) >> n, (b + delta) >> n); -} - -template inline -void v_rshr_pack_store(int* ptr, const v_int64x8& a) -{ - v_int64x8 delta = v512_setall_s64((int64)1 << (n-1)); - v_pack_store(ptr, (a + delta) >> n); -} - -// pack boolean -inline v_uint8x64 v_pack_b(const v_uint16x32& a, const v_uint16x32& b) -{ return v_uint8x64(_mm512_mask_permutexvar_epi64(_mm512_setzero_si512(), (__mmask8) -1, _v512_set_epu64(7, 5, 3, 1, 6, 4, 2, 0), _mm512_packs_epi16(a.val, b.val))); } - -inline v_uint8x64 v_pack_b(const v_uint32x16& a, const v_uint32x16& b, - const v_uint32x16& c, const v_uint32x16& d) -{ - __m512i ab = _mm512_packs_epi32(a.val, b.val); - __m512i cd = _mm512_packs_epi32(c.val, d.val); - - return v_uint8x64(_mm512_permutexvar_epi32(_v512_set_epu32(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0), _mm512_packs_epi16(ab, cd))); -} - -inline v_uint8x64 v_pack_b(const v_uint64x8& a, const v_uint64x8& b, const v_uint64x8& c, - const v_uint64x8& d, const v_uint64x8& e, const v_uint64x8& f, - const v_uint64x8& g, const v_uint64x8& h) -{ - __m512i ab = _mm512_packs_epi32(a.val, b.val); - __m512i cd = _mm512_packs_epi32(c.val, d.val); - __m512i ef = _mm512_packs_epi32(e.val, f.val); - __m512i gh = _mm512_packs_epi32(g.val, h.val); - - __m512i abcd = _mm512_packs_epi32(ab, cd); - __m512i efgh = _mm512_packs_epi32(ef, gh); - - return v_uint8x64(_mm512_permutexvar_epi16(_v512_set_epu16(31, 23, 15, 7, 30, 22, 14, 6, 29, 21, 13, 5, 28, 20, 12, 4, - 27, 19, 11, 3, 26, 18, 10, 2, 25, 17, 9, 1, 24, 16, 8, 0), _mm512_packs_epi16(abcd, efgh))); -} - -/* Recombine */ -// its up there with load and store operations - -/* Extract */ -#define OPENCV_HAL_IMPL_AVX512_EXTRACT(_Tpvec) \ - template \ - inline _Tpvec v_extract(const _Tpvec& a, const _Tpvec& b) \ - { return v_rotate_right(a, b); } - -OPENCV_HAL_IMPL_AVX512_EXTRACT(v_uint8x64) -OPENCV_HAL_IMPL_AVX512_EXTRACT(v_int8x64) -OPENCV_HAL_IMPL_AVX512_EXTRACT(v_uint16x32) -OPENCV_HAL_IMPL_AVX512_EXTRACT(v_int16x32) -OPENCV_HAL_IMPL_AVX512_EXTRACT(v_uint32x16) -OPENCV_HAL_IMPL_AVX512_EXTRACT(v_int32x16) -OPENCV_HAL_IMPL_AVX512_EXTRACT(v_uint64x8) -OPENCV_HAL_IMPL_AVX512_EXTRACT(v_int64x8) -OPENCV_HAL_IMPL_AVX512_EXTRACT(v_float32x16) -OPENCV_HAL_IMPL_AVX512_EXTRACT(v_float64x8) - - -///////////////////// load deinterleave ///////////////////////////// - -inline void v_load_deinterleave( const uchar* ptr, v_uint8x64& a, v_uint8x64& b ) -{ - __m512i ab0 = _mm512_loadu_si512((const __m512i*)ptr); - __m512i ab1 = _mm512_loadu_si512((const __m512i*)(ptr + 64)); -#if CV_AVX_512VBMI - __m512i mask0 = _v512_set_epu8(126, 124, 122, 120, 118, 116, 114, 112, 110, 108, 106, 104, 102, 100, 98, 96, - 94, 92, 90, 88, 86, 84, 82, 80, 78, 76, 74, 72, 70, 68, 66, 64, - 62, 60, 58, 56, 54, 52, 50, 48, 46, 44, 42, 40, 38, 36, 34, 32, - 30, 28, 26, 24, 22, 20, 18, 16, 14, 12, 10, 8, 6, 4, 2, 0); - __m512i mask1 = _v512_set_epu8(127, 125, 123, 121, 119, 117, 115, 113, 111, 109, 107, 105, 103, 101, 99, 97, - 95, 93, 91, 89, 87, 85, 83, 81, 79, 77, 75, 73, 71, 69, 67, 65, - 63, 61, 59, 57, 55, 53, 51, 49, 47, 45, 43, 41, 39, 37, 35, 33, - 31, 29, 27, 25, 23, 21, 19, 17, 15, 13, 11, 9, 7, 5, 3, 1); - a = v_uint8x64(_mm512_permutex2var_epi8(ab0, mask0, ab1)); - b = v_uint8x64(_mm512_permutex2var_epi8(ab0, mask1, ab1)); -#else - __m512i mask0 = _mm512_set4_epi32(0x0f0d0b09, 0x07050301, 0x0e0c0a08, 0x06040200); - __m512i a0b0 = _mm512_shuffle_epi8(ab0, mask0); - __m512i a1b1 = _mm512_shuffle_epi8(ab1, mask0); - __m512i mask1 = _v512_set_epu64(14, 12, 10, 8, 6, 4, 2, 0); - __m512i mask2 = _v512_set_epu64(15, 13, 11, 9, 7, 5, 3, 1); - a = v_uint8x64(_mm512_permutex2var_epi64(a0b0, mask1, a1b1)); - b = v_uint8x64(_mm512_permutex2var_epi64(a0b0, mask2, a1b1)); -#endif -} - -inline void v_load_deinterleave( const ushort* ptr, v_uint16x32& a, v_uint16x32& b ) -{ - __m512i ab0 = _mm512_loadu_si512((const __m512i*)ptr); - __m512i ab1 = _mm512_loadu_si512((const __m512i*)(ptr + 32)); - __m512i mask0 = _v512_set_epu16(62, 60, 58, 56, 54, 52, 50, 48, 46, 44, 42, 40, 38, 36, 34, 32, - 30, 28, 26, 24, 22, 20, 18, 16, 14, 12, 10, 8, 6, 4, 2, 0); - __m512i mask1 = _v512_set_epu16(63, 61, 59, 57, 55, 53, 51, 49, 47, 45, 43, 41, 39, 37, 35, 33, - 31, 29, 27, 25, 23, 21, 19, 17, 15, 13, 11, 9, 7, 5, 3, 1); - a = v_uint16x32(_mm512_permutex2var_epi16(ab0, mask0, ab1)); - b = v_uint16x32(_mm512_permutex2var_epi16(ab0, mask1, ab1)); -} - -inline void v_load_deinterleave( const unsigned* ptr, v_uint32x16& a, v_uint32x16& b ) -{ - __m512i ab0 = _mm512_loadu_si512((const __m512i*)ptr); - __m512i ab1 = _mm512_loadu_si512((const __m512i*)(ptr + 16)); - __m512i mask0 = _v512_set_epu32(30, 28, 26, 24, 22, 20, 18, 16, 14, 12, 10, 8, 6, 4, 2, 0); - __m512i mask1 = _v512_set_epu32(31, 29, 27, 25, 23, 21, 19, 17, 15, 13, 11, 9, 7, 5, 3, 1); - a = v_uint32x16(_mm512_permutex2var_epi32(ab0, mask0, ab1)); - b = v_uint32x16(_mm512_permutex2var_epi32(ab0, mask1, ab1)); -} - -inline void v_load_deinterleave( const uint64* ptr, v_uint64x8& a, v_uint64x8& b ) -{ - __m512i ab0 = _mm512_loadu_si512((const __m512i*)ptr); - __m512i ab1 = _mm512_loadu_si512((const __m512i*)(ptr + 8)); - __m512i mask0 = _v512_set_epu64(14, 12, 10, 8, 6, 4, 2, 0); - __m512i mask1 = _v512_set_epu64(15, 13, 11, 9, 7, 5, 3, 1); - a = v_uint64x8(_mm512_permutex2var_epi64(ab0, mask0, ab1)); - b = v_uint64x8(_mm512_permutex2var_epi64(ab0, mask1, ab1)); -} - -inline void v_load_deinterleave( const uchar* ptr, v_uint8x64& a, v_uint8x64& b, v_uint8x64& c ) -{ - __m512i bgr0 = _mm512_loadu_si512((const __m512i*)ptr); - __m512i bgr1 = _mm512_loadu_si512((const __m512i*)(ptr + 64)); - __m512i bgr2 = _mm512_loadu_si512((const __m512i*)(ptr + 128)); - -#if CV_AVX_512VBMI2 - __m512i mask0 = _v512_set_epu8(126, 123, 120, 117, 114, 111, 108, 105, 102, 99, 96, 93, 90, 87, 84, 81, - 78, 75, 72, 69, 66, 63, 60, 57, 54, 51, 48, 45, 42, 39, 36, 33, - 30, 27, 24, 21, 18, 15, 12, 9, 6, 3, 0, 62, 59, 56, 53, 50, - 47, 44, 41, 38, 35, 32, 29, 26, 23, 20, 17, 14, 11, 8, 5, 2); - __m512i r0b01 = _mm512_permutex2var_epi8(bgr0, mask0, bgr1); - __m512i b1g12 = _mm512_permutex2var_epi8(bgr1, mask0, bgr2); - __m512i r12b2 = _mm512_permutex2var_epi8(bgr1, - _v512_set_epu8(125, 122, 119, 116, 113, 110, 107, 104, 101, 98, 95, 92, 89, 86, 83, 80, - 77, 74, 71, 68, 65, 127, 124, 121, 118, 115, 112, 109, 106, 103, 100, 97, - 94, 91, 88, 85, 82, 79, 76, 73, 70, 67, 64, 61, 58, 55, 52, 49, - 46, 43, 40, 37, 34, 31, 28, 25, 22, 19, 16, 13, 10, 7, 4, 1), bgr2); - a = v_uint8x64(_mm512_mask_compress_epi8(r12b2, 0xffffffffffe00000, r0b01)); - b = v_uint8x64(_mm512_mask_compress_epi8(b1g12, 0x2492492492492492, bgr0)); - c = v_uint8x64(_mm512_mask_expand_epi8(r0b01, 0xffffffffffe00000, r12b2)); -#elif CV_AVX_512VBMI - __m512i b0g0b1 = _mm512_mask_blend_epi8(0xb6db6db6db6db6db, bgr1, bgr0); - __m512i g1r1g2 = _mm512_mask_blend_epi8(0xb6db6db6db6db6db, bgr2, bgr1); - __m512i r2b2r0 = _mm512_mask_blend_epi8(0xb6db6db6db6db6db, bgr0, bgr2); - a = v_uint8x64(_mm512_permutex2var_epi8(b0g0b1, _v512_set_epu8(125, 122, 119, 116, 113, 110, 107, 104, 101, 98, 95, 92, 89, 86, 83, 80, - 77, 74, 71, 68, 65, 63, 61, 60, 58, 57, 55, 54, 52, 51, 49, 48, - 46, 45, 43, 42, 40, 39, 37, 36, 34, 33, 31, 30, 28, 27, 25, 24, - 23, 21, 20, 18, 17, 15, 14, 12, 11, 9, 8, 6, 5, 3, 2, 0), bgr2)); - b = v_uint8x64(_mm512_permutex2var_epi8(g1r1g2, _v512_set_epu8( 63, 61, 60, 58, 57, 55, 54, 52, 51, 49, 48, 46, 45, 43, 42, 40, - 39, 37, 36, 34, 33, 31, 30, 28, 27, 25, 24, 23, 21, 20, 18, 17, - 15, 14, 12, 11, 9, 8, 6, 5, 3, 2, 0, 126, 123, 120, 117, 114, - 111, 108, 105, 102, 99, 96, 93, 90, 87, 84, 81, 78, 75, 72, 69, 66), bgr0)); - c = v_uint8x64(_mm512_permutex2var_epi8(r2b2r0, _v512_set_epu8( 63, 60, 57, 54, 51, 48, 45, 42, 39, 36, 33, 30, 27, 24, 21, 18, - 15, 12, 9, 6, 3, 0, 125, 122, 119, 116, 113, 110, 107, 104, 101, 98, - 95, 92, 89, 86, 83, 80, 77, 74, 71, 68, 65, 62, 59, 56, 53, 50, - 47, 44, 41, 38, 35, 32, 29, 26, 23, 20, 17, 14, 11, 8, 5, 2), bgr1)); -#else - __m512i mask0 = _v512_set_epu16(61, 58, 55, 52, 49, 46, 43, 40, 37, 34, 63, 60, 57, 54, 51, 48, - 45, 42, 39, 36, 33, 30, 27, 24, 21, 18, 15, 12, 9, 6, 3, 0); - __m512i b01g1 = _mm512_permutex2var_epi16(bgr0, mask0, bgr1); - __m512i r12b2 = _mm512_permutex2var_epi16(bgr1, mask0, bgr2); - __m512i g20r0 = _mm512_permutex2var_epi16(bgr2, mask0, bgr0); - - __m512i b0g0 = _mm512_mask_blend_epi32(0xf800, b01g1, r12b2); - __m512i r0b1 = _mm512_permutex2var_epi16(bgr1, _v512_set_epu16(42, 41, 40, 39, 38, 37, 36, 35, 34, 33, 32, 29, 26, 23, 20, 17, - 14, 11, 8, 5, 2, 53, 52, 51, 50, 49, 48, 47, 46, 45, 44, 43), g20r0); - __m512i g1r1 = _mm512_alignr_epi32(r12b2, g20r0, 11); - a = v_uint8x64(_mm512_mask_blend_epi8(0xAAAAAAAAAAAAAAAA, b0g0, r0b1)); - c = v_uint8x64(_mm512_mask_blend_epi8(0xAAAAAAAAAAAAAAAA, r0b1, g1r1)); - b = v_uint8x64(_mm512_shuffle_epi8(_mm512_mask_blend_epi8(0xAAAAAAAAAAAAAAAA, g1r1, b0g0), _mm512_set4_epi32(0x0e0f0c0d, 0x0a0b0809, 0x06070405, 0x02030001))); -#endif -} - -inline void v_load_deinterleave( const ushort* ptr, v_uint16x32& a, v_uint16x32& b, v_uint16x32& c ) -{ - __m512i bgr0 = _mm512_loadu_si512((const __m512i*)ptr); - __m512i bgr1 = _mm512_loadu_si512((const __m512i*)(ptr + 32)); - __m512i bgr2 = _mm512_loadu_si512((const __m512i*)(ptr + 64)); - - __m512i mask0 = _v512_set_epu16(61, 58, 55, 52, 49, 46, 43, 40, 37, 34, 63, 60, 57, 54, 51, 48, - 45, 42, 39, 36, 33, 30, 27, 24, 21, 18, 15, 12, 9, 6, 3, 0); - __m512i b01g1 = _mm512_permutex2var_epi16(bgr0, mask0, bgr1); - __m512i r12b2 = _mm512_permutex2var_epi16(bgr1, mask0, bgr2); - __m512i g20r0 = _mm512_permutex2var_epi16(bgr2, mask0, bgr0); - - a = v_uint16x32(_mm512_mask_blend_epi32(0xf800, b01g1, r12b2)); - b = v_uint16x32(_mm512_permutex2var_epi16(bgr1, _v512_set_epu16(42, 41, 40, 39, 38, 37, 36, 35, 34, 33, 32, 29, 26, 23, 20, 17, - 14, 11, 8, 5, 2, 53, 52, 51, 50, 49, 48, 47, 46, 45, 44, 43), g20r0)); - c = v_uint16x32(_mm512_alignr_epi32(r12b2, g20r0, 11)); -} - -inline void v_load_deinterleave( const unsigned* ptr, v_uint32x16& a, v_uint32x16& b, v_uint32x16& c ) -{ - __m512i bgr0 = _mm512_loadu_si512((const __m512i*)ptr); - __m512i bgr1 = _mm512_loadu_si512((const __m512i*)(ptr + 16)); - __m512i bgr2 = _mm512_loadu_si512((const __m512i*)(ptr + 32)); - - __m512i mask0 = _v512_set_epu32(29, 26, 23, 20, 17, 30, 27, 24, 21, 18, 15, 12, 9, 6, 3, 0); - __m512i b01r1 = _mm512_permutex2var_epi32(bgr0, mask0, bgr1); - __m512i g12b2 = _mm512_permutex2var_epi32(bgr1, mask0, bgr2); - __m512i r20g0 = _mm512_permutex2var_epi32(bgr2, mask0, bgr0); - - a = v_uint32x16(_mm512_mask_blend_epi32(0xf800, b01r1, g12b2)); - b = v_uint32x16(_mm512_alignr_epi32(g12b2, r20g0, 11)); - c = v_uint32x16(_mm512_permutex2var_epi32(bgr1, _v512_set_epu32(21, 20, 19, 18, 17, 16, 13, 10, 7, 4, 1, 26, 25, 24, 23, 22), r20g0)); -} - -inline void v_load_deinterleave( const uint64* ptr, v_uint64x8& a, v_uint64x8& b, v_uint64x8& c ) -{ - __m512i bgr0 = _mm512_loadu_si512((const __m512i*)ptr); - __m512i bgr1 = _mm512_loadu_si512((const __m512i*)(ptr + 8)); - __m512i bgr2 = _mm512_loadu_si512((const __m512i*)(ptr + 16)); - - __m512i mask0 = _v512_set_epu64(13, 10, 15, 12, 9, 6, 3, 0); - __m512i b01g1 = _mm512_permutex2var_epi64(bgr0, mask0, bgr1); - __m512i r12b2 = _mm512_permutex2var_epi64(bgr1, mask0, bgr2); - __m512i g20r0 = _mm512_permutex2var_epi64(bgr2, mask0, bgr0); - - a = v_uint64x8(_mm512_mask_blend_epi64(0xc0, b01g1, r12b2)); - c = v_uint64x8(_mm512_alignr_epi64(r12b2, g20r0, 6)); - b = v_uint64x8(_mm512_permutex2var_epi64(bgr1, _v512_set_epu64(10, 9, 8, 5, 2, 13, 12, 11), g20r0)); -} - -inline void v_load_deinterleave( const uchar* ptr, v_uint8x64& a, v_uint8x64& b, v_uint8x64& c, v_uint8x64& d ) -{ - __m512i bgra0 = _mm512_loadu_si512((const __m512i*)ptr); - __m512i bgra1 = _mm512_loadu_si512((const __m512i*)(ptr + 64)); - __m512i bgra2 = _mm512_loadu_si512((const __m512i*)(ptr + 128)); - __m512i bgra3 = _mm512_loadu_si512((const __m512i*)(ptr + 192)); - -#if CV_AVX_512VBMI - __m512i mask0 = _v512_set_epu8(126, 124, 122, 120, 118, 116, 114, 112, 110, 108, 106, 104, 102, 100, 98, 96, - 94, 92, 90, 88, 86, 84, 82, 80, 78, 76, 74, 72, 70, 68, 66, 64, - 62, 60, 58, 56, 54, 52, 50, 48, 46, 44, 42, 40, 38, 36, 34, 32, - 30, 28, 26, 24, 22, 20, 18, 16, 14, 12, 10, 8, 6, 4, 2, 0); - __m512i mask1 = _v512_set_epu8(127, 125, 123, 121, 119, 117, 115, 113, 111, 109, 107, 105, 103, 101, 99, 97, - 95, 93, 91, 89, 87, 85, 83, 81, 79, 77, 75, 73, 71, 69, 67, 65, - 63, 61, 59, 57, 55, 53, 51, 49, 47, 45, 43, 41, 39, 37, 35, 33, - 31, 29, 27, 25, 23, 21, 19, 17, 15, 13, 11, 9, 7, 5, 3, 1); - - __m512i br01 = _mm512_permutex2var_epi8(bgra0, mask0, bgra1); - __m512i ga01 = _mm512_permutex2var_epi8(bgra0, mask1, bgra1); - __m512i br23 = _mm512_permutex2var_epi8(bgra2, mask0, bgra3); - __m512i ga23 = _mm512_permutex2var_epi8(bgra2, mask1, bgra3); - - a = v_uint8x64(_mm512_permutex2var_epi8(br01, mask0, br23)); - c = v_uint8x64(_mm512_permutex2var_epi8(br01, mask1, br23)); - b = v_uint8x64(_mm512_permutex2var_epi8(ga01, mask0, ga23)); - d = v_uint8x64(_mm512_permutex2var_epi8(ga01, mask1, ga23)); -#else - __m512i mask = _mm512_set4_epi32(0x0f0b0703, 0x0e0a0602, 0x0d090501, 0x0c080400); - __m512i b0g0r0a0 = _mm512_shuffle_epi8(bgra0, mask); - __m512i b1g1r1a1 = _mm512_shuffle_epi8(bgra1, mask); - __m512i b2g2r2a2 = _mm512_shuffle_epi8(bgra2, mask); - __m512i b3g3r3a3 = _mm512_shuffle_epi8(bgra3, mask); - - __m512i mask0 = _v512_set_epu32(30, 28, 26, 24, 22, 20, 18, 16, 14, 12, 10, 8, 6, 4, 2, 0); - __m512i mask1 = _v512_set_epu32(31, 29, 27, 25, 23, 21, 19, 17, 15, 13, 11, 9, 7, 5, 3, 1); - - __m512i br01 = _mm512_permutex2var_epi32(b0g0r0a0, mask0, b1g1r1a1); - __m512i ga01 = _mm512_permutex2var_epi32(b0g0r0a0, mask1, b1g1r1a1); - __m512i br23 = _mm512_permutex2var_epi32(b2g2r2a2, mask0, b3g3r3a3); - __m512i ga23 = _mm512_permutex2var_epi32(b2g2r2a2, mask1, b3g3r3a3); - - a = v_uint8x64(_mm512_permutex2var_epi32(br01, mask0, br23)); - c = v_uint8x64(_mm512_permutex2var_epi32(br01, mask1, br23)); - b = v_uint8x64(_mm512_permutex2var_epi32(ga01, mask0, ga23)); - d = v_uint8x64(_mm512_permutex2var_epi32(ga01, mask1, ga23)); -#endif -} - -inline void v_load_deinterleave( const ushort* ptr, v_uint16x32& a, v_uint16x32& b, v_uint16x32& c, v_uint16x32& d ) -{ - __m512i bgra0 = _mm512_loadu_si512((const __m512i*)ptr); - __m512i bgra1 = _mm512_loadu_si512((const __m512i*)(ptr + 32)); - __m512i bgra2 = _mm512_loadu_si512((const __m512i*)(ptr + 64)); - __m512i bgra3 = _mm512_loadu_si512((const __m512i*)(ptr + 96)); - - __m512i mask0 = _v512_set_epu16(62, 60, 58, 56, 54, 52, 50, 48, 46, 44, 42, 40, 38, 36, 34, 32, - 30, 28, 26, 24, 22, 20, 18, 16, 14, 12, 10, 8, 6, 4, 2, 0); - __m512i mask1 = _v512_set_epu16(63, 61, 59, 57, 55, 53, 51, 49, 47, 45, 43, 41, 39, 37, 35, 33, - 31, 29, 27, 25, 23, 21, 19, 17, 15, 13, 11, 9, 7, 5, 3, 1); - - __m512i br01 = _mm512_permutex2var_epi16(bgra0, mask0, bgra1); - __m512i ga01 = _mm512_permutex2var_epi16(bgra0, mask1, bgra1); - __m512i br23 = _mm512_permutex2var_epi16(bgra2, mask0, bgra3); - __m512i ga23 = _mm512_permutex2var_epi16(bgra2, mask1, bgra3); - - a = v_uint16x32(_mm512_permutex2var_epi16(br01, mask0, br23)); - c = v_uint16x32(_mm512_permutex2var_epi16(br01, mask1, br23)); - b = v_uint16x32(_mm512_permutex2var_epi16(ga01, mask0, ga23)); - d = v_uint16x32(_mm512_permutex2var_epi16(ga01, mask1, ga23)); -} - -inline void v_load_deinterleave( const unsigned* ptr, v_uint32x16& a, v_uint32x16& b, v_uint32x16& c, v_uint32x16& d ) -{ - __m512i bgra0 = _mm512_loadu_si512((const __m512i*)ptr); - __m512i bgra1 = _mm512_loadu_si512((const __m512i*)(ptr + 16)); - __m512i bgra2 = _mm512_loadu_si512((const __m512i*)(ptr + 32)); - __m512i bgra3 = _mm512_loadu_si512((const __m512i*)(ptr + 48)); - - __m512i mask0 = _v512_set_epu32(30, 28, 26, 24, 22, 20, 18, 16, 14, 12, 10, 8, 6, 4, 2, 0); - __m512i mask1 = _v512_set_epu32(31, 29, 27, 25, 23, 21, 19, 17, 15, 13, 11, 9, 7, 5, 3, 1); - - __m512i br01 = _mm512_permutex2var_epi32(bgra0, mask0, bgra1); - __m512i ga01 = _mm512_permutex2var_epi32(bgra0, mask1, bgra1); - __m512i br23 = _mm512_permutex2var_epi32(bgra2, mask0, bgra3); - __m512i ga23 = _mm512_permutex2var_epi32(bgra2, mask1, bgra3); - - a = v_uint32x16(_mm512_permutex2var_epi32(br01, mask0, br23)); - c = v_uint32x16(_mm512_permutex2var_epi32(br01, mask1, br23)); - b = v_uint32x16(_mm512_permutex2var_epi32(ga01, mask0, ga23)); - d = v_uint32x16(_mm512_permutex2var_epi32(ga01, mask1, ga23)); -} - -inline void v_load_deinterleave( const uint64* ptr, v_uint64x8& a, v_uint64x8& b, v_uint64x8& c, v_uint64x8& d ) -{ - __m512i bgra0 = _mm512_loadu_si512((const __m512i*)ptr); - __m512i bgra1 = _mm512_loadu_si512((const __m512i*)(ptr + 8)); - __m512i bgra2 = _mm512_loadu_si512((const __m512i*)(ptr + 16)); - __m512i bgra3 = _mm512_loadu_si512((const __m512i*)(ptr + 24)); - - __m512i mask0 = _v512_set_epu64(14, 12, 10, 8, 6, 4, 2, 0); - __m512i mask1 = _v512_set_epu64(15, 13, 11, 9, 7, 5, 3, 1); - - __m512i br01 = _mm512_permutex2var_epi64(bgra0, mask0, bgra1); - __m512i ga01 = _mm512_permutex2var_epi64(bgra0, mask1, bgra1); - __m512i br23 = _mm512_permutex2var_epi64(bgra2, mask0, bgra3); - __m512i ga23 = _mm512_permutex2var_epi64(bgra2, mask1, bgra3); - - a = v_uint64x8(_mm512_permutex2var_epi64(br01, mask0, br23)); - c = v_uint64x8(_mm512_permutex2var_epi64(br01, mask1, br23)); - b = v_uint64x8(_mm512_permutex2var_epi64(ga01, mask0, ga23)); - d = v_uint64x8(_mm512_permutex2var_epi64(ga01, mask1, ga23)); -} - -///////////////////////////// store interleave ///////////////////////////////////// - -inline void v_store_interleave( uchar* ptr, const v_uint8x64& x, const v_uint8x64& y, - hal::StoreMode mode=hal::STORE_UNALIGNED ) -{ - v_uint8x64 low, high; - v_zip(x, y, low, high); - if( mode == hal::STORE_ALIGNED_NOCACHE ) - { - _mm512_stream_si512((__m512i*)ptr, low.val); - _mm512_stream_si512((__m512i*)(ptr + 64), high.val); - } - else if( mode == hal::STORE_ALIGNED ) - { - _mm512_store_si512((__m512i*)ptr, low.val); - _mm512_store_si512((__m512i*)(ptr + 64), high.val); - } - else - { - _mm512_storeu_si512((__m512i*)ptr, low.val); - _mm512_storeu_si512((__m512i*)(ptr + 64), high.val); - } -} - -inline void v_store_interleave( ushort* ptr, const v_uint16x32& x, const v_uint16x32& y, - hal::StoreMode mode=hal::STORE_UNALIGNED ) -{ - v_uint16x32 low, high; - v_zip(x, y, low, high); - if( mode == hal::STORE_ALIGNED_NOCACHE ) - { - _mm512_stream_si512((__m512i*)ptr, low.val); - _mm512_stream_si512((__m512i*)(ptr + 32), high.val); - } - else if( mode == hal::STORE_ALIGNED ) - { - _mm512_store_si512((__m512i*)ptr, low.val); - _mm512_store_si512((__m512i*)(ptr + 32), high.val); - } - else - { - _mm512_storeu_si512((__m512i*)ptr, low.val); - _mm512_storeu_si512((__m512i*)(ptr + 32), high.val); - } -} - -inline void v_store_interleave( unsigned* ptr, const v_uint32x16& x, const v_uint32x16& y, - hal::StoreMode mode=hal::STORE_UNALIGNED ) -{ - v_uint32x16 low, high; - v_zip(x, y, low, high); - if( mode == hal::STORE_ALIGNED_NOCACHE ) - { - _mm512_stream_si512((__m512i*)ptr, low.val); - _mm512_stream_si512((__m512i*)(ptr + 16), high.val); - } - else if( mode == hal::STORE_ALIGNED ) - { - _mm512_store_si512((__m512i*)ptr, low.val); - _mm512_store_si512((__m512i*)(ptr + 16), high.val); - } - else - { - _mm512_storeu_si512((__m512i*)ptr, low.val); - _mm512_storeu_si512((__m512i*)(ptr + 16), high.val); - } -} - -inline void v_store_interleave( uint64* ptr, const v_uint64x8& x, const v_uint64x8& y, - hal::StoreMode mode=hal::STORE_UNALIGNED ) -{ - v_uint64x8 low, high; - v_zip(x, y, low, high); - if( mode == hal::STORE_ALIGNED_NOCACHE ) - { - _mm512_stream_si512((__m512i*)ptr, low.val); - _mm512_stream_si512((__m512i*)(ptr + 8), high.val); - } - else if( mode == hal::STORE_ALIGNED ) - { - _mm512_store_si512((__m512i*)ptr, low.val); - _mm512_store_si512((__m512i*)(ptr + 8), high.val); - } - else - { - _mm512_storeu_si512((__m512i*)ptr, low.val); - _mm512_storeu_si512((__m512i*)(ptr + 8), high.val); - } -} - -inline void v_store_interleave( uchar* ptr, const v_uint8x64& a, const v_uint8x64& b, const v_uint8x64& c, - hal::StoreMode mode=hal::STORE_UNALIGNED ) -{ -#if CV_AVX_512VBMI - __m512i mask0 = _v512_set_epu8(127, 84, 20, 126, 83, 19, 125, 82, 18, 124, 81, 17, 123, 80, 16, 122, - 79, 15, 121, 78, 14, 120, 77, 13, 119, 76, 12, 118, 75, 11, 117, 74, - 10, 116, 73, 9, 115, 72, 8, 114, 71, 7, 113, 70, 6, 112, 69, 5, - 111, 68, 4, 110, 67, 3, 109, 66, 2, 108, 65, 1, 107, 64, 0, 106); - __m512i mask1 = _v512_set_epu8( 21, 42, 105, 20, 41, 104, 19, 40, 103, 18, 39, 102, 17, 38, 101, 16, - 37, 100, 15, 36, 99, 14, 35, 98, 13, 34, 97, 12, 33, 96, 11, 32, - 95, 10, 31, 94, 9, 30, 93, 8, 29, 92, 7, 28, 91, 6, 27, 90, - 5, 26, 89, 4, 25, 88, 3, 24, 87, 2, 23, 86, 1, 22, 85, 0); - __m512i mask2 = _v512_set_epu8(106, 127, 63, 105, 126, 62, 104, 125, 61, 103, 124, 60, 102, 123, 59, 101, - 122, 58, 100, 121, 57, 99, 120, 56, 98, 119, 55, 97, 118, 54, 96, 117, - 53, 95, 116, 52, 94, 115, 51, 93, 114, 50, 92, 113, 49, 91, 112, 48, - 90, 111, 47, 89, 110, 46, 88, 109, 45, 87, 108, 44, 86, 107, 43, 85); - __m512i r2g0r0 = _mm512_permutex2var_epi8(b.val, mask0, c.val); - __m512i b0r1b1 = _mm512_permutex2var_epi8(a.val, mask1, c.val); - __m512i g1b2g2 = _mm512_permutex2var_epi8(a.val, mask2, b.val); - - __m512i bgr0 = _mm512_mask_blend_epi8(0x9249249249249249, r2g0r0, b0r1b1); - __m512i bgr1 = _mm512_mask_blend_epi8(0x9249249249249249, b0r1b1, g1b2g2); - __m512i bgr2 = _mm512_mask_blend_epi8(0x9249249249249249, g1b2g2, r2g0r0); -#else - __m512i g1g0 = _mm512_shuffle_epi8(b.val, _mm512_set4_epi32(0x0e0f0c0d, 0x0a0b0809, 0x06070405, 0x02030001)); - __m512i b0g0 = _mm512_mask_blend_epi8(0xAAAAAAAAAAAAAAAA, a.val, g1g0); - __m512i r0b1 = _mm512_mask_blend_epi8(0xAAAAAAAAAAAAAAAA, c.val, a.val); - __m512i g1r1 = _mm512_mask_blend_epi8(0xAAAAAAAAAAAAAAAA, g1g0, c.val); - - __m512i mask0 = _v512_set_epu16(42, 10, 31, 41, 9, 30, 40, 8, 29, 39, 7, 28, 38, 6, 27, 37, - 5, 26, 36, 4, 25, 35, 3, 24, 34, 2, 23, 33, 1, 22, 32, 0); - __m512i mask1 = _v512_set_epu16(21, 52, 41, 20, 51, 40, 19, 50, 39, 18, 49, 38, 17, 48, 37, 16, - 47, 36, 15, 46, 35, 14, 45, 34, 13, 44, 33, 12, 43, 32, 11, 42); - __m512i mask2 = _v512_set_epu16(63, 31, 20, 62, 30, 19, 61, 29, 18, 60, 28, 17, 59, 27, 16, 58, - 26, 15, 57, 25, 14, 56, 24, 13, 55, 23, 12, 54, 22, 11, 53, 21); - __m512i b0g0b2 = _mm512_permutex2var_epi16(b0g0, mask0, r0b1); - __m512i r1b1r0 = _mm512_permutex2var_epi16(b0g0, mask1, g1r1); - __m512i g2r2g1 = _mm512_permutex2var_epi16(r0b1, mask2, g1r1); - - __m512i bgr0 = _mm512_mask_blend_epi16(0x24924924, b0g0b2, r1b1r0); - __m512i bgr1 = _mm512_mask_blend_epi16(0x24924924, r1b1r0, g2r2g1); - __m512i bgr2 = _mm512_mask_blend_epi16(0x24924924, g2r2g1, b0g0b2); -#endif - - if( mode == hal::STORE_ALIGNED_NOCACHE ) - { - _mm512_stream_si512((__m512i*)ptr, bgr0); - _mm512_stream_si512((__m512i*)(ptr + 64), bgr1); - _mm512_stream_si512((__m512i*)(ptr + 128), bgr2); - } - else if( mode == hal::STORE_ALIGNED ) - { - _mm512_store_si512((__m512i*)ptr, bgr0); - _mm512_store_si512((__m512i*)(ptr + 64), bgr1); - _mm512_store_si512((__m512i*)(ptr + 128), bgr2); - } - else - { - _mm512_storeu_si512((__m512i*)ptr, bgr0); - _mm512_storeu_si512((__m512i*)(ptr + 64), bgr1); - _mm512_storeu_si512((__m512i*)(ptr + 128), bgr2); - } -} - -inline void v_store_interleave( ushort* ptr, const v_uint16x32& a, const v_uint16x32& b, const v_uint16x32& c, - hal::StoreMode mode=hal::STORE_UNALIGNED ) -{ - __m512i mask0 = _v512_set_epu16(42, 10, 31, 41, 9, 30, 40, 8, 29, 39, 7, 28, 38, 6, 27, 37, - 5, 26, 36, 4, 25, 35, 3, 24, 34, 2, 23, 33, 1, 22, 32, 0); - __m512i mask1 = _v512_set_epu16(21, 52, 41, 20, 51, 40, 19, 50, 39, 18, 49, 38, 17, 48, 37, 16, - 47, 36, 15, 46, 35, 14, 45, 34, 13, 44, 33, 12, 43, 32, 11, 42); - __m512i mask2 = _v512_set_epu16(63, 31, 20, 62, 30, 19, 61, 29, 18, 60, 28, 17, 59, 27, 16, 58, - 26, 15, 57, 25, 14, 56, 24, 13, 55, 23, 12, 54, 22, 11, 53, 21); - __m512i b0g0b2 = _mm512_permutex2var_epi16(a.val, mask0, b.val); - __m512i r1b1r0 = _mm512_permutex2var_epi16(a.val, mask1, c.val); - __m512i g2r2g1 = _mm512_permutex2var_epi16(b.val, mask2, c.val); - - __m512i bgr0 = _mm512_mask_blend_epi16(0x24924924, b0g0b2, r1b1r0); - __m512i bgr1 = _mm512_mask_blend_epi16(0x24924924, r1b1r0, g2r2g1); - __m512i bgr2 = _mm512_mask_blend_epi16(0x24924924, g2r2g1, b0g0b2); - - if( mode == hal::STORE_ALIGNED_NOCACHE ) - { - _mm512_stream_si512((__m512i*)ptr, bgr0); - _mm512_stream_si512((__m512i*)(ptr + 32), bgr1); - _mm512_stream_si512((__m512i*)(ptr + 64), bgr2); - } - else if( mode == hal::STORE_ALIGNED ) - { - _mm512_store_si512((__m512i*)ptr, bgr0); - _mm512_store_si512((__m512i*)(ptr + 32), bgr1); - _mm512_store_si512((__m512i*)(ptr + 64), bgr2); - } - else - { - _mm512_storeu_si512((__m512i*)ptr, bgr0); - _mm512_storeu_si512((__m512i*)(ptr + 32), bgr1); - _mm512_storeu_si512((__m512i*)(ptr + 64), bgr2); - } -} - -inline void v_store_interleave( unsigned* ptr, const v_uint32x16& a, const v_uint32x16& b, const v_uint32x16& c, - hal::StoreMode mode=hal::STORE_UNALIGNED ) -{ - __m512i mask0 = _v512_set_epu32(26, 31, 15, 25, 30, 14, 24, 29, 13, 23, 28, 12, 22, 27, 11, 21); - __m512i mask1 = _v512_set_epu32(31, 10, 25, 30, 9, 24, 29, 8, 23, 28, 7, 22, 27, 6, 21, 26); - __m512i g1b2g2 = _mm512_permutex2var_epi32(a.val, mask0, b.val); - __m512i r2r1b1 = _mm512_permutex2var_epi32(a.val, mask1, c.val); - - __m512i bgr0 = _mm512_mask_expand_epi32(_mm512_mask_expand_epi32(_mm512_maskz_expand_epi32(0x9249, a.val), 0x2492, b.val), 0x4924, c.val); - __m512i bgr1 = _mm512_mask_blend_epi32(0x9249, r2r1b1, g1b2g2); - __m512i bgr2 = _mm512_mask_blend_epi32(0x9249, g1b2g2, r2r1b1); - - if( mode == hal::STORE_ALIGNED_NOCACHE ) - { - _mm512_stream_si512((__m512i*)ptr, bgr0); - _mm512_stream_si512((__m512i*)(ptr + 16), bgr1); - _mm512_stream_si512((__m512i*)(ptr + 32), bgr2); - } - else if( mode == hal::STORE_ALIGNED ) - { - _mm512_store_si512((__m512i*)ptr, bgr0); - _mm512_store_si512((__m512i*)(ptr + 16), bgr1); - _mm512_store_si512((__m512i*)(ptr + 32), bgr2); - } - else - { - _mm512_storeu_si512((__m512i*)ptr, bgr0); - _mm512_storeu_si512((__m512i*)(ptr + 16), bgr1); - _mm512_storeu_si512((__m512i*)(ptr + 32), bgr2); - } -} - -inline void v_store_interleave( uint64* ptr, const v_uint64x8& a, const v_uint64x8& b, const v_uint64x8& c, - hal::StoreMode mode=hal::STORE_UNALIGNED ) -{ - __m512i mask0 = _v512_set_epu64( 5, 12, 7, 4, 11, 6, 3, 10); - __m512i mask1 = _v512_set_epu64(15, 7, 4, 14, 6, 3, 13, 5); - __m512i r1b1b2 = _mm512_permutex2var_epi64(a.val, mask0, c.val); - __m512i g2r2g1 = _mm512_permutex2var_epi64(b.val, mask1, c.val); - - __m512i bgr0 = _mm512_mask_expand_epi64(_mm512_mask_expand_epi64(_mm512_maskz_expand_epi64(0x49, a.val), 0x92, b.val), 0x24, c.val); - __m512i bgr1 = _mm512_mask_blend_epi64(0xdb, g2r2g1, r1b1b2); - __m512i bgr2 = _mm512_mask_blend_epi64(0xdb, r1b1b2, g2r2g1); - - if( mode == hal::STORE_ALIGNED_NOCACHE ) - { - _mm512_stream_si512((__m512i*)ptr, bgr0); - _mm512_stream_si512((__m512i*)(ptr + 8), bgr1); - _mm512_stream_si512((__m512i*)(ptr + 16), bgr2); - } - else if( mode == hal::STORE_ALIGNED ) - { - _mm512_store_si512((__m512i*)ptr, bgr0); - _mm512_store_si512((__m512i*)(ptr + 8), bgr1); - _mm512_store_si512((__m512i*)(ptr + 16), bgr2); - } - else - { - _mm512_storeu_si512((__m512i*)ptr, bgr0); - _mm512_storeu_si512((__m512i*)(ptr + 8), bgr1); - _mm512_storeu_si512((__m512i*)(ptr + 16), bgr2); - } -} - -inline void v_store_interleave( uchar* ptr, const v_uint8x64& a, const v_uint8x64& b, - const v_uint8x64& c, const v_uint8x64& d, - hal::StoreMode mode=hal::STORE_UNALIGNED ) -{ - v_uint8x64 br01, br23, ga01, ga23; - v_zip(a, c, br01, br23); - v_zip(b, d, ga01, ga23); - v_uint8x64 bgra0, bgra1, bgra2, bgra3; - v_zip(br01, ga01, bgra0, bgra1); - v_zip(br23, ga23, bgra2, bgra3); - - if( mode == hal::STORE_ALIGNED_NOCACHE ) - { - _mm512_stream_si512((__m512i*)ptr, bgra0.val); - _mm512_stream_si512((__m512i*)(ptr + 64), bgra1.val); - _mm512_stream_si512((__m512i*)(ptr + 128), bgra2.val); - _mm512_stream_si512((__m512i*)(ptr + 192), bgra3.val); - } - else if( mode == hal::STORE_ALIGNED ) - { - _mm512_store_si512((__m512i*)ptr, bgra0.val); - _mm512_store_si512((__m512i*)(ptr + 64), bgra1.val); - _mm512_store_si512((__m512i*)(ptr + 128), bgra2.val); - _mm512_store_si512((__m512i*)(ptr + 192), bgra3.val); - } - else - { - _mm512_storeu_si512((__m512i*)ptr, bgra0.val); - _mm512_storeu_si512((__m512i*)(ptr + 64), bgra1.val); - _mm512_storeu_si512((__m512i*)(ptr + 128), bgra2.val); - _mm512_storeu_si512((__m512i*)(ptr + 192), bgra3.val); - } -} - -inline void v_store_interleave( ushort* ptr, const v_uint16x32& a, const v_uint16x32& b, - const v_uint16x32& c, const v_uint16x32& d, - hal::StoreMode mode=hal::STORE_UNALIGNED ) -{ - v_uint16x32 br01, br23, ga01, ga23; - v_zip(a, c, br01, br23); - v_zip(b, d, ga01, ga23); - v_uint16x32 bgra0, bgra1, bgra2, bgra3; - v_zip(br01, ga01, bgra0, bgra1); - v_zip(br23, ga23, bgra2, bgra3); - - if( mode == hal::STORE_ALIGNED_NOCACHE ) - { - _mm512_stream_si512((__m512i*)ptr, bgra0.val); - _mm512_stream_si512((__m512i*)(ptr + 32), bgra1.val); - _mm512_stream_si512((__m512i*)(ptr + 64), bgra2.val); - _mm512_stream_si512((__m512i*)(ptr + 96), bgra3.val); - } - else if( mode == hal::STORE_ALIGNED ) - { - _mm512_store_si512((__m512i*)ptr, bgra0.val); - _mm512_store_si512((__m512i*)(ptr + 32), bgra1.val); - _mm512_store_si512((__m512i*)(ptr + 64), bgra2.val); - _mm512_store_si512((__m512i*)(ptr + 96), bgra3.val); - } - else - { - _mm512_storeu_si512((__m512i*)ptr, bgra0.val); - _mm512_storeu_si512((__m512i*)(ptr + 32), bgra1.val); - _mm512_storeu_si512((__m512i*)(ptr + 64), bgra2.val); - _mm512_storeu_si512((__m512i*)(ptr + 96), bgra3.val); - } -} - -inline void v_store_interleave( unsigned* ptr, const v_uint32x16& a, const v_uint32x16& b, - const v_uint32x16& c, const v_uint32x16& d, - hal::StoreMode mode=hal::STORE_UNALIGNED ) -{ - v_uint32x16 br01, br23, ga01, ga23; - v_zip(a, c, br01, br23); - v_zip(b, d, ga01, ga23); - v_uint32x16 bgra0, bgra1, bgra2, bgra3; - v_zip(br01, ga01, bgra0, bgra1); - v_zip(br23, ga23, bgra2, bgra3); - - if( mode == hal::STORE_ALIGNED_NOCACHE ) - { - _mm512_stream_si512((__m512i*)ptr, bgra0.val); - _mm512_stream_si512((__m512i*)(ptr + 16), bgra1.val); - _mm512_stream_si512((__m512i*)(ptr + 32), bgra2.val); - _mm512_stream_si512((__m512i*)(ptr + 48), bgra3.val); - } - else if( mode == hal::STORE_ALIGNED ) - { - _mm512_store_si512((__m512i*)ptr, bgra0.val); - _mm512_store_si512((__m512i*)(ptr + 16), bgra1.val); - _mm512_store_si512((__m512i*)(ptr + 32), bgra2.val); - _mm512_store_si512((__m512i*)(ptr + 48), bgra3.val); - } - else - { - _mm512_storeu_si512((__m512i*)ptr, bgra0.val); - _mm512_storeu_si512((__m512i*)(ptr + 16), bgra1.val); - _mm512_storeu_si512((__m512i*)(ptr + 32), bgra2.val); - _mm512_storeu_si512((__m512i*)(ptr + 48), bgra3.val); - } -} - -inline void v_store_interleave( uint64* ptr, const v_uint64x8& a, const v_uint64x8& b, - const v_uint64x8& c, const v_uint64x8& d, - hal::StoreMode mode=hal::STORE_UNALIGNED ) -{ - v_uint64x8 br01, br23, ga01, ga23; - v_zip(a, c, br01, br23); - v_zip(b, d, ga01, ga23); - v_uint64x8 bgra0, bgra1, bgra2, bgra3; - v_zip(br01, ga01, bgra0, bgra1); - v_zip(br23, ga23, bgra2, bgra3); - - if( mode == hal::STORE_ALIGNED_NOCACHE ) - { - _mm512_stream_si512((__m512i*)ptr, bgra0.val); - _mm512_stream_si512((__m512i*)(ptr + 8), bgra1.val); - _mm512_stream_si512((__m512i*)(ptr + 16), bgra2.val); - _mm512_stream_si512((__m512i*)(ptr + 24), bgra3.val); - } - else if( mode == hal::STORE_ALIGNED ) - { - _mm512_store_si512((__m512i*)ptr, bgra0.val); - _mm512_store_si512((__m512i*)(ptr + 8), bgra1.val); - _mm512_store_si512((__m512i*)(ptr + 16), bgra2.val); - _mm512_store_si512((__m512i*)(ptr + 24), bgra3.val); - } - else - { - _mm512_storeu_si512((__m512i*)ptr, bgra0.val); - _mm512_storeu_si512((__m512i*)(ptr + 8), bgra1.val); - _mm512_storeu_si512((__m512i*)(ptr + 16), bgra2.val); - _mm512_storeu_si512((__m512i*)(ptr + 24), bgra3.val); - } -} - -#define OPENCV_HAL_IMPL_AVX512_LOADSTORE_INTERLEAVE(_Tpvec0, _Tp0, suffix0, _Tpvec1, _Tp1, suffix1) \ -inline void v_load_deinterleave( const _Tp0* ptr, _Tpvec0& a0, _Tpvec0& b0 ) \ -{ \ - _Tpvec1 a1, b1; \ - v_load_deinterleave((const _Tp1*)ptr, a1, b1); \ - a0 = v_reinterpret_as_##suffix0(a1); \ - b0 = v_reinterpret_as_##suffix0(b1); \ -} \ -inline void v_load_deinterleave( const _Tp0* ptr, _Tpvec0& a0, _Tpvec0& b0, _Tpvec0& c0 ) \ -{ \ - _Tpvec1 a1, b1, c1; \ - v_load_deinterleave((const _Tp1*)ptr, a1, b1, c1); \ - a0 = v_reinterpret_as_##suffix0(a1); \ - b0 = v_reinterpret_as_##suffix0(b1); \ - c0 = v_reinterpret_as_##suffix0(c1); \ -} \ -inline void v_load_deinterleave( const _Tp0* ptr, _Tpvec0& a0, _Tpvec0& b0, _Tpvec0& c0, _Tpvec0& d0 ) \ -{ \ - _Tpvec1 a1, b1, c1, d1; \ - v_load_deinterleave((const _Tp1*)ptr, a1, b1, c1, d1); \ - a0 = v_reinterpret_as_##suffix0(a1); \ - b0 = v_reinterpret_as_##suffix0(b1); \ - c0 = v_reinterpret_as_##suffix0(c1); \ - d0 = v_reinterpret_as_##suffix0(d1); \ -} \ -inline void v_store_interleave( _Tp0* ptr, const _Tpvec0& a0, const _Tpvec0& b0, \ - hal::StoreMode mode=hal::STORE_UNALIGNED ) \ -{ \ - _Tpvec1 a1 = v_reinterpret_as_##suffix1(a0); \ - _Tpvec1 b1 = v_reinterpret_as_##suffix1(b0); \ - v_store_interleave((_Tp1*)ptr, a1, b1, mode); \ -} \ -inline void v_store_interleave( _Tp0* ptr, const _Tpvec0& a0, const _Tpvec0& b0, const _Tpvec0& c0, \ - hal::StoreMode mode=hal::STORE_UNALIGNED ) \ -{ \ - _Tpvec1 a1 = v_reinterpret_as_##suffix1(a0); \ - _Tpvec1 b1 = v_reinterpret_as_##suffix1(b0); \ - _Tpvec1 c1 = v_reinterpret_as_##suffix1(c0); \ - v_store_interleave((_Tp1*)ptr, a1, b1, c1, mode); \ -} \ -inline void v_store_interleave( _Tp0* ptr, const _Tpvec0& a0, const _Tpvec0& b0, \ - const _Tpvec0& c0, const _Tpvec0& d0, \ - hal::StoreMode mode=hal::STORE_UNALIGNED ) \ -{ \ - _Tpvec1 a1 = v_reinterpret_as_##suffix1(a0); \ - _Tpvec1 b1 = v_reinterpret_as_##suffix1(b0); \ - _Tpvec1 c1 = v_reinterpret_as_##suffix1(c0); \ - _Tpvec1 d1 = v_reinterpret_as_##suffix1(d0); \ - v_store_interleave((_Tp1*)ptr, a1, b1, c1, d1, mode); \ -} - -OPENCV_HAL_IMPL_AVX512_LOADSTORE_INTERLEAVE(v_int8x64, schar, s8, v_uint8x64, uchar, u8) -OPENCV_HAL_IMPL_AVX512_LOADSTORE_INTERLEAVE(v_int16x32, short, s16, v_uint16x32, ushort, u16) -OPENCV_HAL_IMPL_AVX512_LOADSTORE_INTERLEAVE(v_int32x16, int, s32, v_uint32x16, unsigned, u32) -OPENCV_HAL_IMPL_AVX512_LOADSTORE_INTERLEAVE(v_float32x16, float, f32, v_uint32x16, unsigned, u32) -OPENCV_HAL_IMPL_AVX512_LOADSTORE_INTERLEAVE(v_int64x8, int64, s64, v_uint64x8, uint64, u64) -OPENCV_HAL_IMPL_AVX512_LOADSTORE_INTERLEAVE(v_float64x8, double, f64, v_uint64x8, uint64, u64) - -////////// Mask and checks ///////// - -/** Mask **/ -inline int64 v_signmask(const v_int8x64& a) { return (int64)_mm512_movepi8_mask(a.val); } -inline int v_signmask(const v_int16x32& a) { return (int)_mm512_cmp_epi16_mask(a.val, _mm512_setzero_si512(), _MM_CMPINT_LT); } -inline int v_signmask(const v_int32x16& a) { return (int)_mm512_cmp_epi32_mask(a.val, _mm512_setzero_si512(), _MM_CMPINT_LT); } -inline int v_signmask(const v_int64x8& a) { return (int)_mm512_cmp_epi64_mask(a.val, _mm512_setzero_si512(), _MM_CMPINT_LT); } - -inline int64 v_signmask(const v_uint8x64& a) { return v_signmask(v_reinterpret_as_s8(a)); } -inline int v_signmask(const v_uint16x32& a) { return v_signmask(v_reinterpret_as_s16(a)); } -inline int v_signmask(const v_uint32x16& a) { return v_signmask(v_reinterpret_as_s32(a)); } -inline int v_signmask(const v_uint64x8& a) { return v_signmask(v_reinterpret_as_s64(a)); } -inline int v_signmask(const v_float32x16& a) { return v_signmask(v_reinterpret_as_s32(a)); } -inline int v_signmask(const v_float64x8& a) { return v_signmask(v_reinterpret_as_s64(a)); } - -/** Checks **/ -inline bool v_check_all(const v_int8x64& a) { return !(bool)_mm512_cmp_epi8_mask(a.val, _mm512_setzero_si512(), _MM_CMPINT_NLT); } -inline bool v_check_any(const v_int8x64& a) { return (bool)_mm512_movepi8_mask(a.val); } -inline bool v_check_all(const v_int16x32& a) { return !(bool)_mm512_cmp_epi16_mask(a.val, _mm512_setzero_si512(), _MM_CMPINT_NLT); } -inline bool v_check_any(const v_int16x32& a) { return (bool)_mm512_cmp_epi16_mask(a.val, _mm512_setzero_si512(), _MM_CMPINT_LT); } -inline bool v_check_all(const v_int32x16& a) { return !(bool)_mm512_cmp_epi32_mask(a.val, _mm512_setzero_si512(), _MM_CMPINT_NLT); } -inline bool v_check_any(const v_int32x16& a) { return (bool)_mm512_cmp_epi32_mask(a.val, _mm512_setzero_si512(), _MM_CMPINT_LT); } -inline bool v_check_all(const v_int64x8& a) { return !(bool)_mm512_cmp_epi64_mask(a.val, _mm512_setzero_si512(), _MM_CMPINT_NLT); } -inline bool v_check_any(const v_int64x8& a) { return (bool)_mm512_cmp_epi64_mask(a.val, _mm512_setzero_si512(), _MM_CMPINT_LT); } - -inline bool v_check_all(const v_float32x16& a) { return v_check_all(v_reinterpret_as_s32(a)); } -inline bool v_check_any(const v_float32x16& a) { return v_check_any(v_reinterpret_as_s32(a)); } -inline bool v_check_all(const v_float64x8& a) { return v_check_all(v_reinterpret_as_s64(a)); } -inline bool v_check_any(const v_float64x8& a) { return v_check_any(v_reinterpret_as_s64(a)); } -inline bool v_check_all(const v_uint8x64& a) { return v_check_all(v_reinterpret_as_s8(a)); } -inline bool v_check_all(const v_uint16x32& a) { return v_check_all(v_reinterpret_as_s16(a)); } -inline bool v_check_all(const v_uint32x16& a) { return v_check_all(v_reinterpret_as_s32(a)); } -inline bool v_check_all(const v_uint64x8& a) { return v_check_all(v_reinterpret_as_s64(a)); } -inline bool v_check_any(const v_uint8x64& a) { return v_check_any(v_reinterpret_as_s8(a)); } -inline bool v_check_any(const v_uint16x32& a) { return v_check_any(v_reinterpret_as_s16(a)); } -inline bool v_check_any(const v_uint32x16& a) { return v_check_any(v_reinterpret_as_s32(a)); } -inline bool v_check_any(const v_uint64x8& a) { return v_check_any(v_reinterpret_as_s64(a)); } - -inline int v_scan_forward(const v_int8x64& a) -{ - int64 mask = _mm512_movepi8_mask(a.val); - int mask32 = (int)mask; - return mask != 0 ? mask32 != 0 ? trailingZeros32(mask32) : 32 + trailingZeros32((int)(mask >> 32)) : 0; -} -inline int v_scan_forward(const v_uint8x64& a) { return v_scan_forward(v_reinterpret_as_s8(a)); } -inline int v_scan_forward(const v_int16x32& a) { return trailingZeros32(v_signmask(v_reinterpret_as_s16(a))); } -inline int v_scan_forward(const v_uint16x32& a) { return trailingZeros32(v_signmask(v_reinterpret_as_s16(a))); } -inline int v_scan_forward(const v_int32x16& a) { return trailingZeros32(v_signmask(v_reinterpret_as_s16(a))) / 2; } -inline int v_scan_forward(const v_uint32x16& a) { return trailingZeros32(v_signmask(v_reinterpret_as_s16(a))) / 2; } -inline int v_scan_forward(const v_float32x16& a) { return trailingZeros32(v_signmask(v_reinterpret_as_s16(a))) / 2; } -inline int v_scan_forward(const v_int64x8& a) { return trailingZeros32(v_signmask(v_reinterpret_as_s16(a))) / 4; } -inline int v_scan_forward(const v_uint64x8& a) { return trailingZeros32(v_signmask(v_reinterpret_as_s16(a))) / 4; } -inline int v_scan_forward(const v_float64x8& a) { return trailingZeros32(v_signmask(v_reinterpret_as_s16(a))) / 4; } - -inline void v512_cleanup() { _mm256_zeroall(); } - -static inline void v_deinterleave(const v_float32x16& low, const v_float32x16& high, - v_float32x16& even, v_float32x16& odd) -{ - __m512i permute_mask1 = _mm512_setr_epi32(0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30); - __m512i permute_mask2 = _mm512_setr_epi32(1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31); - - even.val = _mm512_permutex2var_ps(low.val, permute_mask1, high.val); - odd.val = _mm512_permutex2var_ps(low.val, permute_mask2, high.val); -} - -static inline v_uint8x64 v_interleave_low(const v_uint8x64& a, const v_uint8x64& b) -{ - return v_uint8x64(_mm512_unpacklo_epi8(a.val, b.val)); -} - -static inline v_uint8x64 v_interleave_high(const v_uint8x64& a, const v_uint8x64& b) -{ - return v_uint8x64(_mm512_unpackhi_epi8(a.val, b.val)); -} - -static inline v_int16x32 v_interleave_low(const v_int16x32& a, const v_int16x32& b) -{ - return v_int16x32(_mm512_unpacklo_epi16(a.val, b.val)); -} - -static inline v_int16x32 v_interleave_high(const v_int16x32& a, const v_int16x32& b) -{ - return v_int16x32(_mm512_unpackhi_epi16(a.val, b.val)); -} - -static inline v_uint8x64 v_saturate_u8(const v_int16x32& a) -{ - v_uint8x64 r; - r.val = _mm512_packus_epi16(a.val, _mm512_setzero_si512()); - return r; -} - -static inline v_int16x32 v_saturate_s16(const v_int32x16& a) -{ - v_int16x32 r; - r.val = _mm512_packs_epi32(a.val, _mm512_setzero_si512()); - return r; -} - -static inline v_int32x16 v_madd(const v_int16x32& a, const v_int16x32& b) -{ - v_int32x16 r; - r.val = _mm512_madd_epi16(a.val, b.val); - return r; -} - -static inline v_int16x32 v_mulhi(const v_int16x32& a, short b) -{ - v_int16x32 r; - r.val = _mm512_mulhi_epi16(a.val, _mm512_set1_epi16(b)); - return r; -} - -static inline v_uint16x32 v_mulhi(const v_uint16x32& a, v_uint16x32 b) -{ - v_uint16x32 r; - r.val = _mm512_mulhi_epu16(a.val, b.val); - return r; -} - -static inline v_uint16x32 v_mulhi(const v_uint16x32& a, uint16_t b) -{ - v_uint16x32 r; - r.val = _mm512_mulhi_epu16(a.val, _mm512_set1_epi16(b)); - return r; -} - -static inline v_int16x32 v_mulhrs(const v_int16x32& a, const v_int16x32& b) -{ - return v_int16x32(_mm512_mulhrs_epi16(a.val, b.val)); -} - -static inline v_int16x32 v_mulhrs(const v_int16x32& a, short b) -{ - return v_mulhrs(a, v512_setall_s16(b)); -} - -static inline v_float32x16 v_fma(const v_float32x16& a, float b, const v_float32x16& c) -{ - return v_fma(a, v512_setall_f32(b), c); -} - -static inline v_int16x32 operator+ (const v_int16x32& a, short b) -{ - return a + v512_setall_s16(b); -} - -static inline v_int16x32 operator- (short a, const v_int16x32& b) -{ - return v512_setall_s16(a) - b; -} - -static inline v_float32x16 operator- (float a, const v_float32x16& b) -{ - return v512_setall_f32(a) - b; -} - -static inline v_float32x16 operator* (const v_float32x16& a, float b) -{ - return a * v512_setall_f32(b); -} - -template -static inline v_uint8x64 v_mask_blend_shiftleft(const v_uint8x64& a, const v_uint8x64& b) -{ - return v_uint8x64(_mm512_mask_blend_epi16(mask, - a.val, _mm512_bslli_epi128(b.val, shift))); -} - -template -static inline v_uint8x64 v_mask_blend_shiftright(const v_uint8x64& a, const v_uint8x64& b) -{ - return v_uint8x64(_mm512_mask_blend_epi16(mask, - _mm512_bsrli_epi128(a.val, shift), b.val)); -} - -static inline v_uint8x64 v_packus(const v_int16x32& a, const v_int16x32& b) -{ - return v_uint8x64(_mm512_packus_epi16(a.val, b.val)); -} - -#define word(b0, b1, b2, b3) \ - (((uint32_t)((uint8_t)(b0)) << 0*8) \ - | ((uint32_t)((uint8_t)(b1)) << 1*8) \ - | ((uint32_t)((uint8_t)(b2)) << 2*8) \ - | ((uint32_t)((uint8_t)(b3)) << 3*8)) - -static inline v_uint8x64 v_setr_s8(char b0, char b1, char b2, char b3, char b4, - char b5, char b6, char b7, char b8, char b9, - char b10, char b11, char b12, char b13, char b14, - char b15, char b16, char b17, char b18, char b19, - char b20, char b21, char b22, char b23, char b24, - char b25, char b26, char b27, char b28, char b29, - char b30, char b31, char b32, char b33, char b34, - char b35, char b36, char b37, char b38, char b39, - char b40, char b41, char b42, char b43, char b44, - char b45, char b46, char b47, char b48, char b49, - char b50, char b51, char b52, char b53, char b54, - char b55, char b56, char b57, char b58, char b59, - char b60, char b61, char b62, char b63) -{ - return v_uint8x64(_mm512_setr_epi32(word(b0, b1, b2, b3), word(b4, b5, b6, b7), word(b8, b9, b10, b11), - word(b12, b13, b14, b15), word(b16, b17, b18, b19), word(b20, b21, b22, b23), - word(b24, b25, b26, b27), word(b28, b29, b30, b31), word(b32, b33, b34, b35), - word(b36, b37, b38, b39), word(b40, b41, b42, b43), word(b44, b45, b46, b47), - word(b48, b49, b50, b51), word(b52, b53, b54, b55), word(b56, b57, b58, b59), - word(b60, b61, b62, b63))); -} - -static inline void v_deinterleave_expand(const v_uint8x64& src, v_int16x32& even, v_int16x32& odd) -{ - v_uint8x64 mask_even = v_setr_s8(0, -1, 2, -1, 4, -1, 6, -1, 8, -1, 10, -1, - 12, -1, 14, -1, 16, -1, 18, -1, 20, -1, 22, - -1, 24, -1, 26, -1, 28, -1, 30, -1, 32, -1, - 34, -1, 36, -1, 38, -1, 40, -1, 42, -1, 44, - -1, 46, -1, 48, -1, 50, -1, 52, -1, 54, -1, - 56, -1, 58, -1, 60, -1, 62, -1); - - v_uint8x64 mask_odd = v_setr_s8(1, -1, 3, -1, 5, -1, 7, -1, 9, -1, 11, -1, - 13, -1, 15, -1, 17, -1, 19, -1, 21, -1, 23, - -1, 25, -1, 27, -1, 29, -1, 31, -1, 33, -1, - 35, -1, 37, -1, 39, -1, 41, -1, 43, -1, 45, - -1, 47, -1, 49, -1, 51, -1, 53, -1, 55, -1, - 57, -1, 59, -1, 61, -1, 63, -1); - - even.val = _mm512_shuffle_epi8(src.val, mask_even.val); - odd.val = _mm512_shuffle_epi8(src.val, mask_odd.val); -} - -static inline v_uint64x8 v_set_s64(int b7, int b6, int b5, int b4, int b3, int b2, int b1, int b0) -{ - return v_uint64x8(_mm512_set_epi64(b7, b6, b5, b4, b3, b2, b1, b0)); -} - -static inline v_uint32x16 v_set_s32(int b15, int b14, int b13, int b12, int b11, int b10, int b9, int b8, - int b7, int b6, int b5, int b4, int b3, int b2, int b1, int b0) -{ - return v_uint32x16(_mm512_set_epi32(b15, b14, b13, b12, b11, b10, b9, b8, b7, b6, b5, b4, b3, b2, b1, b0)); -} - -static inline v_uint32x16 v_setr_s32(int b1, int b2, int b3, int b4, int b5, int b6, int b7, int b8, - int b9, int b10, int b11, int b12, int b13, int b14, int b15, int b16) -{ - return v_uint32x16(_mm512_setr_epi32(b1, b2, b3, b4, b5, b6, b7, b8, b9, b10, b11, b12, b13, b14, b15, b16)); -} - -static inline v_uint8x64 v_shuffle_s8(const v_uint8x64& a, const v_uint8x64& mask) -{ - return v_uint8x64(_mm512_shuffle_epi8(a.val, mask.val)); -} - -static inline v_int16x32 v_load_ccache_expand(const uchar* ptr) -{ - return v_int16x32(_mm512_cvtepu8_epi16(_mm256_lddqu_si256((const __m256i*)ptr))); \ -} - -static inline __m512i v512_insert_epi16(__m512i& target, const ushort x, const int index) -{ - return _mm512_mask_set1_epi16(target, 1UL << index, x); -} - -static inline __m512i v512_insert_epi32(__m512i& target, const int32_t x, const int index) -{ - return _mm512_mask_set1_epi32(target, 1UL << index, x); -} - -static inline __m512i v512_insert_epi64(__m512i& target, const int64_t x, const int index) -{ - return _mm512_mask_set1_epi64(target, 1UL << index, x); -} - -static inline void v_gather_channel(v_uint8x64& vec, const uint8_t tmp[], const short mapsx[], - int chanNum, int c, int x, int shift) -{ - __m256i vec1 = _mm256_setzero_si256(); - __m256i vec2 = _mm256_setzero_si256(); - - vec1 = _mm256_insert_epi32(vec1, *reinterpret_cast(&tmp[4 * (chanNum * mapsx[x + shift + 0] + c)]), 0); - vec1 = _mm256_insert_epi32(vec1, *reinterpret_cast(&tmp[4 * (chanNum * mapsx[x + shift + 1] + c)]), 1); - vec1 = _mm256_insert_epi32(vec1, *reinterpret_cast(&tmp[4 * (chanNum * mapsx[x + shift + 2] + c)]), 2); - vec1 = _mm256_insert_epi32(vec1, *reinterpret_cast(&tmp[4 * (chanNum * mapsx[x + shift + 3] + c)]), 3); - vec1 = _mm256_insert_epi32(vec1, *reinterpret_cast(&tmp[4 * (chanNum * mapsx[x + shift + 4] + c)]), 4); - vec1 = _mm256_insert_epi32(vec1, *reinterpret_cast(&tmp[4 * (chanNum * mapsx[x + shift + 5] + c)]), 5); - vec1 = _mm256_insert_epi32(vec1, *reinterpret_cast(&tmp[4 * (chanNum * mapsx[x + shift + 6] + c)]), 6); - vec1 = _mm256_insert_epi32(vec1, *reinterpret_cast(&tmp[4 * (chanNum * mapsx[x + shift + 7] + c)]), 7); - - vec2 = _mm256_insert_epi32(vec2, *reinterpret_cast(&tmp[4 * (chanNum * (mapsx[x + shift + 0] + 1) + c)]), 0); - vec2 = _mm256_insert_epi32(vec2, *reinterpret_cast(&tmp[4 * (chanNum * (mapsx[x + shift + 1] + 1) + c)]), 1); - vec2 = _mm256_insert_epi32(vec2, *reinterpret_cast(&tmp[4 * (chanNum * (mapsx[x + shift + 2] + 1) + c)]), 2); - vec2 = _mm256_insert_epi32(vec2, *reinterpret_cast(&tmp[4 * (chanNum * (mapsx[x + shift + 3] + 1) + c)]), 3); - vec2 = _mm256_insert_epi32(vec2, *reinterpret_cast(&tmp[4 * (chanNum * (mapsx[x + shift + 4] + 1) + c)]), 4); - vec2 = _mm256_insert_epi32(vec2, *reinterpret_cast(&tmp[4 * (chanNum * (mapsx[x + shift + 5] + 1) + c)]), 5); - vec2 = _mm256_insert_epi32(vec2, *reinterpret_cast(&tmp[4 * (chanNum * (mapsx[x + shift + 6] + 1) + c)]), 6); - vec2 = _mm256_insert_epi32(vec2, *reinterpret_cast(&tmp[4 * (chanNum * (mapsx[x + shift + 7] + 1) + c)]), 7); - - vec.val = _mm512_inserti32x8(_mm512_castsi256_si512(vec1), vec2, 1); -} - -static inline v_uint8x64 v_permutex2_s64(const v_uint8x64& a, const v_uint8x64& b, const v_uint64x8& idxs) -{ - return v_uint8x64(_mm512_permutex2var_epi64(a.val, idxs.val, b.val)); -} - -static inline v_uint8x64 v_permute32(const v_uint8x64& a, const v_uint64x8& idxs) -{ - return v_uint8x64(_mm512_permutexvar_epi32(idxs.val, a.val)); -} - -static inline v_uint8x64 v_permutex2_s32(const v_uint8x64& a, const v_uint8x64& b, const v_uint32x16& idxs) -{ - return v_uint8x64(_mm512_permutex2var_epi32(a.val, idxs.val, b.val)); -} - -static inline v_uint8x64 v_permute32(const v_uint8x64& a, const v_uint32x16& idxs) -{ - return v_uint8x64(_mm512_permutexvar_epi32(idxs.val, a.val)); -} - -static inline void v_set(v_uint8x64& val_0, v_uint8x64& val_1, - v_uint8x64& val_2, v_uint8x64& val_3, - uint8_t tmp[], const short mapsx[], - int x, int shift) -{ - val_0.val = _mm512_setr_epi64(*reinterpret_cast(&tmp[4 * (*(mapsx + x + 0))]), - *reinterpret_cast(&tmp[4 * (*(mapsx + x + 1))]), - *reinterpret_cast(&tmp[4 * (*(mapsx + x + 2))]), - *reinterpret_cast(&tmp[4 * (*(mapsx + x + 3))]), - *reinterpret_cast(&tmp[4 * (*(mapsx + x + 4))]), - *reinterpret_cast(&tmp[4 * (*(mapsx + x + 5))]), - *reinterpret_cast(&tmp[4 * (*(mapsx + x + 6))]), - *reinterpret_cast(&tmp[4 * (*(mapsx + x + 7))])); - - val_1.val = _mm512_setr_epi64(*reinterpret_cast(&tmp[4 * (*(mapsx + x + shift + 0))]), - *reinterpret_cast(&tmp[4 * (*(mapsx + x + shift + 1))]), - *reinterpret_cast(&tmp[4 * (*(mapsx + x + shift + 2))]), - *reinterpret_cast(&tmp[4 * (*(mapsx + x + shift + 3))]), - *reinterpret_cast(&tmp[4 * (*(mapsx + x + shift + 4))]), - *reinterpret_cast(&tmp[4 * (*(mapsx + x + shift + 5))]), - *reinterpret_cast(&tmp[4 * (*(mapsx + x + shift + 6))]), - *reinterpret_cast(&tmp[4 * (*(mapsx + x + shift + 7))])); - - val_2.val = _mm512_setr_epi64(*reinterpret_cast(&tmp[4 * (*(mapsx + x + 2 * shift + 0))]), - *reinterpret_cast(&tmp[4 * (*(mapsx + x + 2 * shift + 1))]), - *reinterpret_cast(&tmp[4 * (*(mapsx + x + 2 * shift + 2))]), - *reinterpret_cast(&tmp[4 * (*(mapsx + x + 2 * shift + 3))]), - *reinterpret_cast(&tmp[4 * (*(mapsx + x + 2 * shift + 4))]), - *reinterpret_cast(&tmp[4 * (*(mapsx + x + 2 * shift + 5))]), - *reinterpret_cast(&tmp[4 * (*(mapsx + x + 2 * shift + 6))]), - *reinterpret_cast(&tmp[4 * (*(mapsx + x + 2 * shift + 7))])); - - val_3.val = _mm512_setr_epi64(*reinterpret_cast(&tmp[4 * (*(mapsx + x + 3 * shift + 0))]), - *reinterpret_cast(&tmp[4 * (*(mapsx + x + 3 * shift + 1))]), - *reinterpret_cast(&tmp[4 * (*(mapsx + x + 3 * shift + 2))]), - *reinterpret_cast(&tmp[4 * (*(mapsx + x + 3 * shift + 3))]), - *reinterpret_cast(&tmp[4 * (*(mapsx + x + 3 * shift + 4))]), - *reinterpret_cast(&tmp[4 * (*(mapsx + x + 3 * shift + 5))]), - *reinterpret_cast(&tmp[4 * (*(mapsx + x + 3 * shift + 6))]), - *reinterpret_cast(&tmp[4 * (*(mapsx + x + 3 * shift + 7))])); -} - -#if defined(__GNUC__) && !defined(__INTEL_COMPILER) - -int _mm512_cvtsi512_si32(__m512i a) -{ - __v16si b = (__v16si)a; - return b[0]; -} - -#endif - -template -static inline int v512_extract_epi32(__m512i target) -{ - return _mm512_cvtsi512_si32(_mm512_mask_alignr_epi32(_mm512_setzero_si512(), (__mmask16)-1, target, target, index)); -} - -template -static inline int v512_extract_epi16(__m512i target) -{ - return (v512_extract_epi32(target) >> (index % 2 ? 16 : 0)) & 0xFFFF; -} - -static inline v_uint8x64 v_gather_pairs(const uchar src[], const v_int16x32& index) { - v_uint8x64 r; - - r.val = v512_insert_epi16(r.val, *reinterpret_cast(&src[v512_extract_epi16<0>(index.val)]), 0); - r.val = v512_insert_epi16(r.val, *reinterpret_cast(&src[v512_extract_epi16<1>(index.val)]), 1); - r.val = v512_insert_epi16(r.val, *reinterpret_cast(&src[v512_extract_epi16<2>(index.val)]), 2); - r.val = v512_insert_epi16(r.val, *reinterpret_cast(&src[v512_extract_epi16<3>(index.val)]), 3); - r.val = v512_insert_epi16(r.val, *reinterpret_cast(&src[v512_extract_epi16<4>(index.val)]), 4); - r.val = v512_insert_epi16(r.val, *reinterpret_cast(&src[v512_extract_epi16<5>(index.val)]), 5); - r.val = v512_insert_epi16(r.val, *reinterpret_cast(&src[v512_extract_epi16<6>(index.val)]), 6); - r.val = v512_insert_epi16(r.val, *reinterpret_cast(&src[v512_extract_epi16<7>(index.val)]), 7); - r.val = v512_insert_epi16(r.val, *reinterpret_cast(&src[v512_extract_epi16<8>(index.val)]), 8); - r.val = v512_insert_epi16(r.val, *reinterpret_cast(&src[v512_extract_epi16<9>(index.val)]), 9); - r.val = v512_insert_epi16(r.val, *reinterpret_cast(&src[v512_extract_epi16<10>(index.val)]), 10); - r.val = v512_insert_epi16(r.val, *reinterpret_cast(&src[v512_extract_epi16<11>(index.val)]), 11); - r.val = v512_insert_epi16(r.val, *reinterpret_cast(&src[v512_extract_epi16<12>(index.val)]), 12); - r.val = v512_insert_epi16(r.val, *reinterpret_cast(&src[v512_extract_epi16<13>(index.val)]), 13); - r.val = v512_insert_epi16(r.val, *reinterpret_cast(&src[v512_extract_epi16<14>(index.val)]), 14); - r.val = v512_insert_epi16(r.val, *reinterpret_cast(&src[v512_extract_epi16<15>(index.val)]), 15); - - r.val = v512_insert_epi16(r.val, *reinterpret_cast(&src[v512_extract_epi16<16>(index.val)]), 16); - r.val = v512_insert_epi16(r.val, *reinterpret_cast(&src[v512_extract_epi16<17>(index.val)]), 17); - r.val = v512_insert_epi16(r.val, *reinterpret_cast(&src[v512_extract_epi16<18>(index.val)]), 18); - r.val = v512_insert_epi16(r.val, *reinterpret_cast(&src[v512_extract_epi16<19>(index.val)]), 19); - r.val = v512_insert_epi16(r.val, *reinterpret_cast(&src[v512_extract_epi16<20>(index.val)]), 20); - r.val = v512_insert_epi16(r.val, *reinterpret_cast(&src[v512_extract_epi16<21>(index.val)]), 21); - r.val = v512_insert_epi16(r.val, *reinterpret_cast(&src[v512_extract_epi16<22>(index.val)]), 22); - r.val = v512_insert_epi16(r.val, *reinterpret_cast(&src[v512_extract_epi16<23>(index.val)]), 23); - r.val = v512_insert_epi16(r.val, *reinterpret_cast(&src[v512_extract_epi16<24>(index.val)]), 24); - r.val = v512_insert_epi16(r.val, *reinterpret_cast(&src[v512_extract_epi16<25>(index.val)]), 25); - r.val = v512_insert_epi16(r.val, *reinterpret_cast(&src[v512_extract_epi16<26>(index.val)]), 26); - r.val = v512_insert_epi16(r.val, *reinterpret_cast(&src[v512_extract_epi16<27>(index.val)]), 27); - r.val = v512_insert_epi16(r.val, *reinterpret_cast(&src[v512_extract_epi16<28>(index.val)]), 28); - r.val = v512_insert_epi16(r.val, *reinterpret_cast(&src[v512_extract_epi16<29>(index.val)]), 29); - r.val = v512_insert_epi16(r.val, *reinterpret_cast(&src[v512_extract_epi16<30>(index.val)]), 30); - r.val = v512_insert_epi16(r.val, *reinterpret_cast(&src[v512_extract_epi16<31>(index.val)]), 31); - - return r; -} - -static inline void v_gather_pairs(const float src[], const int mapsx[], int x, - v_float32x16& low, v_float32x16& high) { - __m512i lo = _mm512_castps_si512(low.val); - __m512i hi = _mm512_castps_si512(high.val); - - lo = v512_insert_epi64(lo, *reinterpret_cast(&src[mapsx[x]]), 0); - lo = v512_insert_epi64(lo, *reinterpret_cast(&src[mapsx[x + 1]]), 1); - lo = v512_insert_epi64(lo, *reinterpret_cast(&src[mapsx[x + 2]]), 2); - lo = v512_insert_epi64(lo, *reinterpret_cast(&src[mapsx[x + 3]]), 3); - lo = v512_insert_epi64(lo, *reinterpret_cast(&src[mapsx[x + 4]]), 4); - lo = v512_insert_epi64(lo, *reinterpret_cast(&src[mapsx[x + 5]]), 5); - lo = v512_insert_epi64(lo, *reinterpret_cast(&src[mapsx[x + 6]]), 6); - lo = v512_insert_epi64(lo, *reinterpret_cast(&src[mapsx[x + 7]]), 7); - - hi = v512_insert_epi64(hi, *reinterpret_cast(&src[mapsx[x + 8]]), 0); - hi = v512_insert_epi64(hi, *reinterpret_cast(&src[mapsx[x + 9]]), 1); - hi = v512_insert_epi64(hi, *reinterpret_cast(&src[mapsx[x + 10]]), 2); - hi = v512_insert_epi64(hi, *reinterpret_cast(&src[mapsx[x + 11]]), 3); - hi = v512_insert_epi64(hi, *reinterpret_cast(&src[mapsx[x + 12]]), 4); - hi = v512_insert_epi64(hi, *reinterpret_cast(&src[mapsx[x + 13]]), 5); - hi = v512_insert_epi64(hi, *reinterpret_cast(&src[mapsx[x + 14]]), 6); - hi = v512_insert_epi64(hi, *reinterpret_cast(&src[mapsx[x + 15]]), 7); - - low.val = _mm512_castsi512_ps(lo); - high.val = _mm512_castsi512_ps(hi); -} - -namespace { - template - static inline v_int16x32 v_gather_chan(const uchar src[], const v_int16x32& index, int channel, int pos) { - v_int16x32 r; - - r.val = v512_insert_epi16(r.val, *reinterpret_cast(&src[chanNum*(v512_extract_epi16<0>(index.val) + pos) + channel]), 0); - r.val = v512_insert_epi16(r.val, *reinterpret_cast(&src[chanNum*(v512_extract_epi16<1>(index.val) + pos) + channel]), 1); - r.val = v512_insert_epi16(r.val, *reinterpret_cast(&src[chanNum*(v512_extract_epi16<2>(index.val) + pos) + channel]), 2); - r.val = v512_insert_epi16(r.val, *reinterpret_cast(&src[chanNum*(v512_extract_epi16<3>(index.val) + pos) + channel]), 3); - r.val = v512_insert_epi16(r.val, *reinterpret_cast(&src[chanNum*(v512_extract_epi16<4>(index.val) + pos) + channel]), 4); - r.val = v512_insert_epi16(r.val, *reinterpret_cast(&src[chanNum*(v512_extract_epi16<5>(index.val) + pos) + channel]), 5); - r.val = v512_insert_epi16(r.val, *reinterpret_cast(&src[chanNum*(v512_extract_epi16<6>(index.val) + pos) + channel]), 6); - r.val = v512_insert_epi16(r.val, *reinterpret_cast(&src[chanNum*(v512_extract_epi16<7>(index.val) + pos) + channel]), 7); - r.val = v512_insert_epi16(r.val, *reinterpret_cast(&src[chanNum*(v512_extract_epi16<8>(index.val) + pos) + channel]), 8); - r.val = v512_insert_epi16(r.val, *reinterpret_cast(&src[chanNum*(v512_extract_epi16<9>(index.val) + pos) + channel]), 9); - r.val = v512_insert_epi16(r.val, *reinterpret_cast(&src[chanNum*(v512_extract_epi16<10>(index.val) + pos) + channel]), 10); - r.val = v512_insert_epi16(r.val, *reinterpret_cast(&src[chanNum*(v512_extract_epi16<11>(index.val) + pos) + channel]), 11); - r.val = v512_insert_epi16(r.val, *reinterpret_cast(&src[chanNum*(v512_extract_epi16<12>(index.val) + pos) + channel]), 12); - r.val = v512_insert_epi16(r.val, *reinterpret_cast(&src[chanNum*(v512_extract_epi16<13>(index.val) + pos) + channel]), 13); - r.val = v512_insert_epi16(r.val, *reinterpret_cast(&src[chanNum*(v512_extract_epi16<14>(index.val) + pos) + channel]), 14); - r.val = v512_insert_epi16(r.val, *reinterpret_cast(&src[chanNum*(v512_extract_epi16<15>(index.val) + pos) + channel]), 15); - r.val = v512_insert_epi16(r.val, *reinterpret_cast(&src[chanNum*(v512_extract_epi16<16>(index.val) + pos) + channel]), 16); - r.val = v512_insert_epi16(r.val, *reinterpret_cast(&src[chanNum*(v512_extract_epi16<17>(index.val) + pos) + channel]), 17); - r.val = v512_insert_epi16(r.val, *reinterpret_cast(&src[chanNum*(v512_extract_epi16<18>(index.val) + pos) + channel]), 18); - r.val = v512_insert_epi16(r.val, *reinterpret_cast(&src[chanNum*(v512_extract_epi16<19>(index.val) + pos) + channel]), 19); - r.val = v512_insert_epi16(r.val, *reinterpret_cast(&src[chanNum*(v512_extract_epi16<20>(index.val) + pos) + channel]), 20); - r.val = v512_insert_epi16(r.val, *reinterpret_cast(&src[chanNum*(v512_extract_epi16<21>(index.val) + pos) + channel]), 21); - r.val = v512_insert_epi16(r.val, *reinterpret_cast(&src[chanNum*(v512_extract_epi16<22>(index.val) + pos) + channel]), 22); - r.val = v512_insert_epi16(r.val, *reinterpret_cast(&src[chanNum*(v512_extract_epi16<23>(index.val) + pos) + channel]), 23); - r.val = v512_insert_epi16(r.val, *reinterpret_cast(&src[chanNum*(v512_extract_epi16<24>(index.val) + pos) + channel]), 24); - r.val = v512_insert_epi16(r.val, *reinterpret_cast(&src[chanNum*(v512_extract_epi16<25>(index.val) + pos) + channel]), 25); - r.val = v512_insert_epi16(r.val, *reinterpret_cast(&src[chanNum*(v512_extract_epi16<26>(index.val) + pos) + channel]), 26); - r.val = v512_insert_epi16(r.val, *reinterpret_cast(&src[chanNum*(v512_extract_epi16<27>(index.val) + pos) + channel]), 27); - r.val = v512_insert_epi16(r.val, *reinterpret_cast(&src[chanNum*(v512_extract_epi16<28>(index.val) + pos) + channel]), 28); - r.val = v512_insert_epi16(r.val, *reinterpret_cast(&src[chanNum*(v512_extract_epi16<29>(index.val) + pos) + channel]), 29); - r.val = v512_insert_epi16(r.val, *reinterpret_cast(&src[chanNum*(v512_extract_epi16<30>(index.val) + pos) + channel]), 30); - r.val = v512_insert_epi16(r.val, *reinterpret_cast(&src[chanNum*(v512_extract_epi16<31>(index.val) + pos) + channel]), 31); - - return r; - } -} // namespace - -CV_CPU_OPTIMIZATION_HAL_NAMESPACE_END - -//! @endcond - -} // cv:: - -#endif // OPENCV_HAL_INTRIN_AVX_HPP diff --git a/thirdparty/ocv/opencv_hal_intrin.hpp b/thirdparty/ocv/opencv_hal_intrin.hpp deleted file mode 100644 index 62795f7abe7..00000000000 --- a/thirdparty/ocv/opencv_hal_intrin.hpp +++ /dev/null @@ -1,572 +0,0 @@ -/*M/////////////////////////////////////////////////////////////////////////////////////// -// -// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. -// -// By downloading, copying, installing or using the software you agree to this license. -// If you do not agree to this license, do not download, install, -// copy or use the software. -// -// -// License Agreement -// For Open Source Computer Vision Library -// -// Copyright (C) 2000-2008, Intel Corporation, all rights reserved. -// Copyright (C) 2009, Willow Garage Inc., all rights reserved. -// Copyright (C) 2013, OpenCV Foundation, all rights reserved. -// Copyright (C) 2015, Itseez Inc., all rights reserved. -// Third party copyrights are property of their respective owners. -// -// Redistribution and use in source and binary forms, with or without modification, -// are permitted provided that the following conditions are met: -// -// * Redistribution's of source code must retain the above copyright notice, -// this list of conditions and the following disclaimer. -// -// * Redistribution's in binary form must reproduce the above copyright notice, -// this list of conditions and the following disclaimer in the documentation -// and/or other materials provided with the distribution. -// -// * The name of the copyright holders may not be used to endorse or promote products -// derived from this software without specific prior written permission. -// -// This software is provided by the copyright holders and contributors "as is" and -// any express or implied warranties, including, but not limited to, the implied -// warranties of merchantability and fitness for a particular purpose are disclaimed. -// In no event shall the Intel Corporation or contributors be liable for any direct, -// indirect, incidental, special, exemplary, or consequential damages -// (including, but not limited to, procurement of substitute goods or services; -// loss of use, data, or profits; or business interruption) however caused -// and on any theory of liability, whether in contract, strict liability, -// or tort (including negligence or otherwise) arising in any way out of -// the use of this software, even if advised of the possibility of such damage. -// -//M*/ - -#ifndef OPENCV_HAL_INTRIN_HPP -#define OPENCV_HAL_INTRIN_HPP - -#include -#include -#include - -//! @name Data types -//! primitive types -//! - schar - signed 1 byte integer -//! - uchar - unsigned 1 byte integer -//! - short - signed 2 byte integer -//! - ushort - unsigned 2 byte integer -//! - int - signed 4 byte integer -//! - uint - unsigned 4 byte integer -//! - int64 - signed 8 byte integer -//! - uint64 - unsigned 8 byte integer -//! @{ -#if !defined _MSC_VER && !defined __BORLANDC__ -# if defined __cplusplus && __cplusplus >= 201103L && !defined __APPLE__ -# include -# ifdef __NEWLIB__ - typedef unsigned int uint; -# else - typedef std::uint32_t uint; -# endif -# else -# include - typedef uint32_t uint; -# endif -#else - typedef unsigned uint; -#endif - -#if defined _MSC_VER || defined __BORLANDC__ - typedef __int64 int64; - typedef unsigned __int64 uint64; -# define CV_BIG_INT(n) n##I64 -# define CV_BIG_UINT(n) n##UI64 -#else - typedef int64_t int64; - typedef uint64_t uint64; -# define CV_BIG_INT(n) n##LL -# define CV_BIG_UINT(n) n##ULL -#endif - -#if defined(__cplusplus) -#if defined(_MSC_VER) && _MSC_VER < 1600 /* MSVS 2010 */ -namespace cv { -typedef signed char int8_t; -typedef unsigned char uint8_t; -typedef signed short int16_t; -typedef unsigned short uint16_t; -typedef signed int int32_t; -typedef unsigned int uint32_t; -typedef signed __int64 int64_t; -typedef unsigned __int64 uint64_t; -} -#elif defined(_MSC_VER) || __cplusplus >= 201103L -#include -namespace cv { -using std::int8_t; -using std::uint8_t; -using std::int16_t; -using std::uint16_t; -using std::int32_t; -using std::uint32_t; -using std::int64_t; -using std::uint64_t; -} -#else -#include -namespace cv { -typedef ::int8_t int8_t; -typedef ::uint8_t uint8_t; -typedef ::int16_t int16_t; -typedef ::uint16_t uint16_t; -typedef ::int32_t int32_t; -typedef ::uint32_t uint32_t; -typedef ::int64_t int64_t; -typedef ::uint64_t uint64_t; -} -#endif -#else // pure C -#include -#endif - -#define OPENCV_HAL_ADD(a, b) ((a) + (b)) -#define OPENCV_HAL_AND(a, b) ((a) & (b)) -#define OPENCV_HAL_NOP(a) (a) -#define OPENCV_HAL_1ST(a, b) (a) - -#ifdef __GNUC__ -# define CV_DECL_ALIGNED(x) __attribute__ ((aligned (x))) -#elif defined _MSC_VER -# define CV_DECL_ALIGNED(x) __declspec(align(x)) -#else -# define CV_DECL_ALIGNED(x) -#endif - -typedef union Cv32suf -{ - int i; - unsigned u; - float f; -} -Cv32suf; - -#if defined(_WIN32) && (defined(_M_IX86) || defined(_M_X64)) -// This is needed for _tzcnt_u32(). -#include -#endif - -namespace { -inline unsigned int trailingZeros32(unsigned int value) { -#if defined(_MSC_VER) -#if (_MSC_VER < 1700) || defined(_M_ARM) || defined(_M_ARM64) - unsigned long index = 0; - _BitScanForward(&index, value); - return (unsigned int)index; -#elif defined(__clang__) - // clang-cl doesn't export _tzcnt_u32 for non BMI systems - return value ? __builtin_ctz(value) : 32; -#else - return _tzcnt_u32(value); -#endif -#elif defined(__GNUC__) || defined(__GNUG__) - return __builtin_ctz(value); -#elif defined(__ICC) || defined(__INTEL_COMPILER) - return _bit_scan_forward(value); -#elif defined(__clang__) - return llvm.cttz.i32(value, true); -#else - static const int MultiplyDeBruijnBitPosition[32] = { - 0, 1, 28, 2, 29, 14, 24, 3, 30, 22, 20, 15, 25, 17, 4, 8, - 31, 27, 13, 23, 21, 19, 16, 7, 26, 12, 18, 6, 11, 5, 10, 9 }; - return MultiplyDeBruijnBitPosition[((uint32_t)((value & -value) * 0x077CB531U)) >> 27]; -#endif -} -} - -// unlike HAL API, which is in cv::hal, -// we put intrinsics into cv namespace to make its -// access from within opencv code more accessible -namespace cv { - -#ifndef CV_ALWAYS_INLINE -#if defined(__GNUC__) && (__GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 1)) -#define CV_ALWAYS_INLINE inline __attribute__((always_inline)) -#elif defined(_MSC_VER) -#define CV_ALWAYS_INLINE __forceinline -#else -#define CV_ALWAYS_INLINE inline -#endif -#endif - -namespace hal { - -enum StoreMode -{ - STORE_UNALIGNED = 0, - STORE_ALIGNED = 1, - STORE_ALIGNED_NOCACHE = 2 -}; - -} - -template struct V_TypeTraits -{ -}; - -#define CV_INTRIN_DEF_TYPE_TRAITS(type, int_type_, uint_type_, abs_type_, w_type_, q_type_, sum_type_, nlanes128_) \ - template<> struct V_TypeTraits \ - { \ - typedef type value_type; \ - typedef int_type_ int_type; \ - typedef abs_type_ abs_type; \ - typedef uint_type_ uint_type; \ - typedef w_type_ w_type; \ - typedef q_type_ q_type; \ - typedef sum_type_ sum_type; \ - enum { nlanes128 = nlanes128_ }; \ - \ - static inline int_type reinterpret_int(type x) \ - { \ - union { type l; int_type i; } v; \ - v.l = x; \ - return v.i; \ - } \ - \ - static inline type reinterpret_from_int(int_type x) \ - { \ - union { type l; int_type i; } v; \ - v.i = x; \ - return v.l; \ - } \ - } - -CV_INTRIN_DEF_TYPE_TRAITS(uchar, schar, uchar, uchar, ushort, unsigned, unsigned, 16); -CV_INTRIN_DEF_TYPE_TRAITS(schar, schar, uchar, uchar, short, int, int, 16); -CV_INTRIN_DEF_TYPE_TRAITS(ushort, short, ushort, ushort, unsigned, uint64, unsigned, 8); -CV_INTRIN_DEF_TYPE_TRAITS(short, short, ushort, ushort, int, int64, int, 8); -CV_INTRIN_DEF_TYPE_TRAITS(unsigned, int, unsigned, unsigned, uint64, void, unsigned, 4); -CV_INTRIN_DEF_TYPE_TRAITS(int, int, unsigned, unsigned, int64, void, int, 4); -CV_INTRIN_DEF_TYPE_TRAITS(float, int, unsigned, float, double, void, float, 4); -CV_INTRIN_DEF_TYPE_TRAITS(uint64, int64, uint64, uint64, void, void, uint64, 2); -CV_INTRIN_DEF_TYPE_TRAITS(int64, int64, uint64, uint64, void, void, int64, 2); -CV_INTRIN_DEF_TYPE_TRAITS(double, int64, uint64, double, void, void, double, 2); - -#ifndef CV_DOXYGEN - -#ifdef CV_CPU_DISPATCH_MODE - #define CV_CPU_OPTIMIZATION_HAL_NAMESPACE __CV_CAT(hal_, CV_CPU_DISPATCH_MODE) - #define CV_CPU_OPTIMIZATION_HAL_NAMESPACE_BEGIN namespace __CV_CAT(hal_, CV_CPU_DISPATCH_MODE) { - #define CV_CPU_OPTIMIZATION_HAL_NAMESPACE_END } -#else - #define CV_CPU_OPTIMIZATION_HAL_NAMESPACE hal_baseline - #define CV_CPU_OPTIMIZATION_HAL_NAMESPACE_BEGIN namespace hal_baseline { - #define CV_CPU_OPTIMIZATION_HAL_NAMESPACE_END } -#endif - -CV_CPU_OPTIMIZATION_HAL_NAMESPACE_BEGIN -CV_CPU_OPTIMIZATION_HAL_NAMESPACE_END -using namespace CV_CPU_OPTIMIZATION_HAL_NAMESPACE; -#endif -} - -/*#ifdef CV_DOXYGEN -# undef CV_AVX2 -# undef CV_SSE2 -# undef CV_NEON -# undef CV_VSX -# undef CV_FP16 -# undef CV_MSA -#endif*/ - -#if CV_SSE2 || CV_NEON -#define CV__SIMD_FORWARD 128 -#include "opencv_hal_intrin_forward.hpp" -#endif - -#if CV_SSE2 - -#include "opencv_hal_sse.hpp" - -#elif CV_NEON - -#include "opencv_hal_neon.hpp" - -#endif - -// AVX2 can be used together with SSE2, so -// we define those two sets of intrinsics at once. -// Most of the intrinsics do not conflict (the proper overloaded variant is -// resolved by the argument types, e.g. v_float32x4 ~ SSE2, v_float32x8 ~ AVX2), -// but some of AVX2 intrinsics get v256_ prefix instead of v_, e.g. v256_load() vs v_load(). -// Correspondingly, the wide intrinsics (which are mapped to the "widest" -// available instruction set) will get vx_ prefix -// (and will be mapped to v256_ counterparts) (e.g. vx_load() => v256_load()) -#if CV_AVX2 - -#define CV__SIMD_FORWARD 256 -#include "opencv_hal_intrin_forward.hpp" -#include "opencv_hal_avx.hpp" - -#endif - -// AVX512 can be used together with SSE2 and AVX2, so -// we define those sets of intrinsics at once. -// For some of AVX512 intrinsics get v512_ prefix instead of v_, e.g. v512_load() vs v_load(). -// Wide intrinsics will be mapped to v512_ counterparts in this case(e.g. vx_load() => v512_load()) -#if CV_AVX512_SKX - -#define CV__SIMD_FORWARD 512 -#include "opencv_hal_intrin_forward.hpp" -#include "opencv_hal_avx512.hpp" - -#endif - -//! @cond IGNORED - -namespace cv { - -#ifndef CV_DOXYGEN -CV_CPU_OPTIMIZATION_HAL_NAMESPACE_BEGIN -#endif - -#ifndef CV_SIMD128 -#define CV_SIMD128 0 -#endif - -#ifndef CV_SIMD128_64F -#define CV_SIMD128_64F 0 -#endif - -#ifndef CV_SIMD256 -#define CV_SIMD256 0 -#endif - -#ifndef CV_SIMD256_64F -#define CV_SIMD256_64F 0 -#endif - -#ifndef CV_SIMD512 -#define CV_SIMD512 0 -#endif - -#ifndef CV_SIMD512_64F -#define CV_SIMD512_64F 0 -#endif - -#ifndef CV_SIMD128_FP16 -#define CV_SIMD128_FP16 0 -#endif - -#ifndef CV_SIMD256_FP16 -#define CV_SIMD256_FP16 0 -#endif - -#ifndef CV_SIMD512_FP16 -#define CV_SIMD512_FP16 0 -#endif - -//================================================================================================== - -#define CV_INTRIN_DEFINE_WIDE_INTRIN(typ, vtyp, short_typ, prefix, loadsfx) \ - inline vtyp vx_setall_##short_typ(typ v) { return prefix##_setall_##short_typ(v); } \ - inline vtyp vx_setzero_##short_typ() { return prefix##_setzero_##short_typ(); } \ - inline vtyp vx_##loadsfx(const typ* ptr) { return prefix##_##loadsfx(ptr); } \ - inline vtyp vx_##loadsfx##_aligned(const typ* ptr) { return prefix##_##loadsfx##_aligned(ptr); } \ - inline vtyp vx_##loadsfx##_low(const typ* ptr) { return prefix##_##loadsfx##_low(ptr); } \ - inline vtyp vx_##loadsfx##_halves(const typ* ptr0, const typ* ptr1) { return prefix##_##loadsfx##_halves(ptr0, ptr1); } \ - inline void vx_store(typ* ptr, const vtyp& v) { return v_store(ptr, v); } \ - inline void vx_store_aligned(typ* ptr, const vtyp& v) { return v_store_aligned(ptr, v); } \ - inline vtyp vx_lut(const typ* ptr, const int* idx) { return prefix##_lut(ptr, idx); } \ - inline vtyp vx_lut_pairs(const typ* ptr, const int* idx) { return prefix##_lut_pairs(ptr, idx); } - -#define CV_INTRIN_DEFINE_WIDE_LUT_QUAD(typ, vtyp, prefix) \ - inline vtyp vx_lut_quads(const typ* ptr, const int* idx) { return prefix##_lut_quads(ptr, idx); } - -#define CV_INTRIN_DEFINE_WIDE_LOAD_EXPAND(typ, wtyp, prefix) \ - inline wtyp vx_load_expand(const typ* ptr) { return prefix##_load_expand(ptr); } - -#define CV_INTRIN_DEFINE_WIDE_LOAD_EXPAND_Q(typ, qtyp, prefix) \ - inline qtyp vx_load_expand_q(const typ* ptr) { return prefix##_load_expand_q(ptr); } - -#define CV_INTRIN_DEFINE_WIDE_INTRIN_WITH_EXPAND(typ, vtyp, short_typ, wtyp, qtyp, prefix, loadsfx) \ - CV_INTRIN_DEFINE_WIDE_INTRIN(typ, vtyp, short_typ, prefix, loadsfx) \ - CV_INTRIN_DEFINE_WIDE_LUT_QUAD(typ, vtyp, prefix) \ - CV_INTRIN_DEFINE_WIDE_LOAD_EXPAND(typ, wtyp, prefix) \ - CV_INTRIN_DEFINE_WIDE_LOAD_EXPAND_Q(typ, qtyp, prefix) - -#define CV_INTRIN_DEFINE_WIDE_INTRIN_ALL_TYPES(prefix) \ - CV_INTRIN_DEFINE_WIDE_INTRIN_WITH_EXPAND(uchar, v_uint8, u8, v_uint16, v_uint32, prefix, load) \ - CV_INTRIN_DEFINE_WIDE_INTRIN_WITH_EXPAND(schar, v_int8, s8, v_int16, v_int32, prefix, load) \ - CV_INTRIN_DEFINE_WIDE_INTRIN(ushort, v_uint16, u16, prefix, load) \ - CV_INTRIN_DEFINE_WIDE_LUT_QUAD(ushort, v_uint16, prefix) \ - CV_INTRIN_DEFINE_WIDE_LOAD_EXPAND(ushort, v_uint32, prefix) \ - CV_INTRIN_DEFINE_WIDE_INTRIN(short, v_int16, s16, prefix, load) \ - CV_INTRIN_DEFINE_WIDE_LUT_QUAD(short, v_int16, prefix) \ - CV_INTRIN_DEFINE_WIDE_LOAD_EXPAND(short, v_int32, prefix) \ - CV_INTRIN_DEFINE_WIDE_INTRIN(int, v_int32, s32, prefix, load) \ - CV_INTRIN_DEFINE_WIDE_LUT_QUAD(int, v_int32, prefix) \ - CV_INTRIN_DEFINE_WIDE_LOAD_EXPAND(int, v_int64, prefix) \ - CV_INTRIN_DEFINE_WIDE_INTRIN(unsigned, v_uint32, u32, prefix, load) \ - CV_INTRIN_DEFINE_WIDE_LUT_QUAD(unsigned, v_uint32, prefix) \ - CV_INTRIN_DEFINE_WIDE_LOAD_EXPAND(unsigned, v_uint64, prefix) \ - CV_INTRIN_DEFINE_WIDE_INTRIN(float, v_float32, f32, prefix, load) \ - CV_INTRIN_DEFINE_WIDE_LUT_QUAD(float, v_float32, prefix) \ - CV_INTRIN_DEFINE_WIDE_INTRIN(int64, v_int64, s64, prefix, load) \ - CV_INTRIN_DEFINE_WIDE_INTRIN(uint64, v_uint64, u64, prefix, load) \ - //CV_INTRIN_DEFINE_WIDE_LOAD_EXPAND(float16_t, v_float32, prefix) //Unsupported case - -template struct V_RegTraits -{ -}; - -#define CV_DEF_REG_TRAITS(prefix, _reg, lane_type, suffix, _u_reg, _w_reg, _q_reg, _int_reg, _round_reg) \ - template<> struct V_RegTraits<_reg> \ - { \ - typedef _reg reg; \ - typedef _u_reg u_reg; \ - typedef _w_reg w_reg; \ - typedef _q_reg q_reg; \ - typedef _int_reg int_reg; \ - typedef _round_reg round_reg; \ - } - -#if CV_SIMD128 || CV_SIMD128_CPP - CV_DEF_REG_TRAITS(v, v_uint8x16, uchar, u8, v_uint8x16, v_uint16x8, v_uint32x4, v_int8x16, void); - CV_DEF_REG_TRAITS(v, v_int8x16, schar, s8, v_uint8x16, v_int16x8, v_int32x4, v_int8x16, void); - CV_DEF_REG_TRAITS(v, v_uint16x8, ushort, u16, v_uint16x8, v_uint32x4, v_uint64x2, v_int16x8, void); - CV_DEF_REG_TRAITS(v, v_int16x8, short, s16, v_uint16x8, v_int32x4, v_int64x2, v_int16x8, void); - CV_DEF_REG_TRAITS(v, v_uint32x4, unsigned, u32, v_uint32x4, v_uint64x2, void, v_int32x4, void); - CV_DEF_REG_TRAITS(v, v_int32x4, int, s32, v_uint32x4, v_int64x2, void, v_int32x4, void); -#if CV_SIMD128_64F - CV_DEF_REG_TRAITS(v, v_float32x4, float, f32, v_float32x4, v_float64x2, void, v_int32x4, v_int32x4); -#else - CV_DEF_REG_TRAITS(v, v_float32x4, float, f32, v_float32x4, void, void, v_int32x4, v_int32x4); -#endif - CV_DEF_REG_TRAITS(v, v_uint64x2, uint64, u64, v_uint64x2, void, void, v_int64x2, void); - CV_DEF_REG_TRAITS(v, v_int64x2, int64, s64, v_uint64x2, void, void, v_int64x2, void); -#if CV_SIMD128_64F - CV_DEF_REG_TRAITS(v, v_float64x2, double, f64, v_float64x2, void, void, v_int64x2, v_int32x4); -#endif -#endif - -#if CV_SIMD256 - CV_DEF_REG_TRAITS(v256, v_uint8x32, uchar, u8, v_uint8x32, v_uint16x16, v_uint32x8, v_int8x32, void); - CV_DEF_REG_TRAITS(v256, v_int8x32, schar, s8, v_uint8x32, v_int16x16, v_int32x8, v_int8x32, void); - CV_DEF_REG_TRAITS(v256, v_uint16x16, ushort, u16, v_uint16x16, v_uint32x8, v_uint64x4, v_int16x16, void); - CV_DEF_REG_TRAITS(v256, v_int16x16, short, s16, v_uint16x16, v_int32x8, v_int64x4, v_int16x16, void); - CV_DEF_REG_TRAITS(v256, v_uint32x8, unsigned, u32, v_uint32x8, v_uint64x4, void, v_int32x8, void); - CV_DEF_REG_TRAITS(v256, v_int32x8, int, s32, v_uint32x8, v_int64x4, void, v_int32x8, void); - CV_DEF_REG_TRAITS(v256, v_float32x8, float, f32, v_float32x8, v_float64x4, void, v_int32x8, v_int32x8); - CV_DEF_REG_TRAITS(v256, v_uint64x4, uint64, u64, v_uint64x4, void, void, v_int64x4, void); - CV_DEF_REG_TRAITS(v256, v_int64x4, int64, s64, v_uint64x4, void, void, v_int64x4, void); - CV_DEF_REG_TRAITS(v256, v_float64x4, double, f64, v_float64x4, void, void, v_int64x4, v_int32x8); -#endif - -#if CV_SIMD512 - CV_DEF_REG_TRAITS(v512, v_uint8x64, uchar, u8, v_uint8x64, v_uint16x32, v_uint32x16, v_int8x64, void); - CV_DEF_REG_TRAITS(v512, v_int8x64, schar, s8, v_uint8x64, v_int16x32, v_int32x16, v_int8x64, void); - CV_DEF_REG_TRAITS(v512, v_uint16x32, ushort, u16, v_uint16x32, v_uint32x16, v_uint64x8, v_int16x32, void); - CV_DEF_REG_TRAITS(v512, v_int16x32, short, s16, v_uint16x32, v_int32x16, v_int64x8, v_int16x32, void); - CV_DEF_REG_TRAITS(v512, v_uint32x16, unsigned, u32, v_uint32x16, v_uint64x8, void, v_int32x16, void); - CV_DEF_REG_TRAITS(v512, v_int32x16, int, s32, v_uint32x16, v_int64x8, void, v_int32x16, void); - CV_DEF_REG_TRAITS(v512, v_float32x16, float, f32, v_float32x16, v_float64x8, void, v_int32x16, v_int32x16); - CV_DEF_REG_TRAITS(v512, v_uint64x8, uint64, u64, v_uint64x8, void, void, v_int64x8, void); - CV_DEF_REG_TRAITS(v512, v_int64x8, int64, s64, v_uint64x8, void, void, v_int64x8, void); - CV_DEF_REG_TRAITS(v512, v_float64x8, double, f64, v_float64x8, void, void, v_int64x8, v_int32x16); -#endif - -#if CV_SIMD512// && (!defined(CV__SIMD_FORCE_WIDTH) || CV__SIMD_FORCE_WIDTH == 512) -#define CV__SIMD_NAMESPACE simd512 -namespace CV__SIMD_NAMESPACE { - #define CV_SIMD 1 - #define CV_SIMD_64F CV_SIMD512_64F - #define CV_SIMD_FP16 CV_SIMD512_FP16 - #define CV_SIMD_WIDTH 64 - typedef v_uint8x64 v_uint8; - typedef v_int8x64 v_int8; - typedef v_uint16x32 v_uint16; - typedef v_int16x32 v_int16; - typedef v_uint32x16 v_uint32; - typedef v_int32x16 v_int32; - typedef v_uint64x8 v_uint64; - typedef v_int64x8 v_int64; - typedef v_float32x16 v_float32; - CV_INTRIN_DEFINE_WIDE_INTRIN_ALL_TYPES(v512) -#if CV_SIMD512_64F - typedef v_float64x8 v_float64; - CV_INTRIN_DEFINE_WIDE_INTRIN(double, v_float64, f64, v512, load) -#endif - inline void vx_cleanup() { v512_cleanup(); } -} // namespace -using namespace CV__SIMD_NAMESPACE; -#elif CV_SIMD256// && (!defined(CV__SIMD_FORCE_WIDTH) || CV__SIMD_FORCE_WIDTH == 256) -#define CV__SIMD_NAMESPACE simd256 -namespace CV__SIMD_NAMESPACE { - #define CV_SIMD 1 - #define CV_SIMD_64F CV_SIMD256_64F - #define CV_SIMD_FP16 CV_SIMD256_FP16 - #define CV_SIMD_WIDTH 32 - typedef v_uint8x32 v_uint8; - typedef v_int8x32 v_int8; - typedef v_uint16x16 v_uint16; - typedef v_int16x16 v_int16; - typedef v_uint32x8 v_uint32; - typedef v_int32x8 v_int32; - typedef v_uint64x4 v_uint64; - typedef v_int64x4 v_int64; - typedef v_float32x8 v_float32; - CV_INTRIN_DEFINE_WIDE_INTRIN_ALL_TYPES(v256) - #if CV_SIMD256_64F - typedef v_float64x4 v_float64; - CV_INTRIN_DEFINE_WIDE_INTRIN(double, v_float64, f64, v256, load) - #endif - inline void vx_cleanup() { v256_cleanup(); } -} // namespace -using namespace CV__SIMD_NAMESPACE; -#elif (CV_SIMD128 || CV_SIMD128_CPP) && (!defined(CV__SIMD_FORCE_WIDTH) || CV__SIMD_FORCE_WIDTH == 128) -#define CV__SIMD_NAMESPACE simd128 -namespace CV__SIMD_NAMESPACE { - #define CV_SIMD CV_SIMD128 - #define CV_SIMD_64F CV_SIMD128_64F - #define CV_SIMD_WIDTH 16 - typedef v_uint8x16 v_uint8; - typedef v_int8x16 v_int8; - typedef v_uint16x8 v_uint16; - typedef v_int16x8 v_int16; - typedef v_uint32x4 v_uint32; - typedef v_int32x4 v_int32; - typedef v_uint64x2 v_uint64; - typedef v_int64x2 v_int64; - typedef v_float32x4 v_float32; - CV_INTRIN_DEFINE_WIDE_INTRIN_ALL_TYPES(v) - #if CV_SIMD128_64F - typedef v_float64x2 v_float64; - CV_INTRIN_DEFINE_WIDE_INTRIN(double, v_float64, f64, v, load) - #endif - inline void vx_cleanup() { v_cleanup(); } -} // namespace -using namespace CV__SIMD_NAMESPACE; -#endif - -#ifndef CV_DOXYGEN -CV_CPU_OPTIMIZATION_HAL_NAMESPACE_END -#endif - -#ifndef CV_SIMD_64F -#define CV_SIMD_64F 0 -#endif - -#ifndef CV_SIMD_FP16 -#define CV_SIMD_FP16 0 //!< Defined to 1 on native support of operations with float16x8_t / float16x16_t (SIMD256) types -#endif - - -#ifndef CV_SIMD -#define CV_SIMD 0 -#endif - -} // cv:: - -//! @endcond - -#endif diff --git a/thirdparty/ocv/opencv_hal_intrin_forward.hpp b/thirdparty/ocv/opencv_hal_intrin_forward.hpp deleted file mode 100644 index 28f67cc9ef9..00000000000 --- a/thirdparty/ocv/opencv_hal_intrin_forward.hpp +++ /dev/null @@ -1,191 +0,0 @@ -// This file is part of OpenCV project. -// It is subject to the license terms in the LICENSE file found in the top-level directory -// of this distribution and at http://opencv.org/license.html - -#ifndef CV__SIMD_FORWARD -#error "Need to pre-define forward width" -#endif - -namespace cv -{ - -//! @cond IGNORED - -CV_CPU_OPTIMIZATION_HAL_NAMESPACE_BEGIN - -/** Types **/ -#if CV__SIMD_FORWARD == 1024 -// [todo] 1024 -#error "1024-long ops not implemented yet" -#elif CV__SIMD_FORWARD == 512 -// 512 -#define __CV_VX(fun) v512_##fun -#define __CV_V_UINT8 v_uint8x64 -#define __CV_V_INT8 v_int8x64 -#define __CV_V_UINT16 v_uint16x32 -#define __CV_V_INT16 v_int16x32 -#define __CV_V_UINT32 v_uint32x16 -#define __CV_V_INT32 v_int32x16 -#define __CV_V_UINT64 v_uint64x8 -#define __CV_V_INT64 v_int64x8 -#define __CV_V_FLOAT32 v_float32x16 -#define __CV_V_FLOAT64 v_float64x8 -struct v_uint8x64; -struct v_int8x64; -struct v_uint16x32; -struct v_int16x32; -struct v_uint32x16; -struct v_int32x16; -struct v_uint64x8; -struct v_int64x8; -struct v_float32x16; -struct v_float64x8; -#elif CV__SIMD_FORWARD == 256 -// 256 -#define __CV_VX(fun) v256_##fun -#define __CV_V_UINT8 v_uint8x32 -#define __CV_V_INT8 v_int8x32 -#define __CV_V_UINT16 v_uint16x16 -#define __CV_V_INT16 v_int16x16 -#define __CV_V_UINT32 v_uint32x8 -#define __CV_V_INT32 v_int32x8 -#define __CV_V_UINT64 v_uint64x4 -#define __CV_V_INT64 v_int64x4 -#define __CV_V_FLOAT32 v_float32x8 -#define __CV_V_FLOAT64 v_float64x4 -struct v_uint8x32; -struct v_int8x32; -struct v_uint16x16; -struct v_int16x16; -struct v_uint32x8; -struct v_int32x8; -struct v_uint64x4; -struct v_int64x4; -struct v_float32x8; -struct v_float64x4; -#else -// 128 -#define __CV_VX(fun) v_##fun -#define __CV_V_UINT8 v_uint8x16 -#define __CV_V_INT8 v_int8x16 -#define __CV_V_UINT16 v_uint16x8 -#define __CV_V_INT16 v_int16x8 -#define __CV_V_UINT32 v_uint32x4 -#define __CV_V_INT32 v_int32x4 -#define __CV_V_UINT64 v_uint64x2 -#define __CV_V_INT64 v_int64x2 -#define __CV_V_FLOAT32 v_float32x4 -#define __CV_V_FLOAT64 v_float64x2 -struct v_uint8x16; -struct v_int8x16; -struct v_uint16x8; -struct v_int16x8; -struct v_uint32x4; -struct v_int32x4; -struct v_uint64x2; -struct v_int64x2; -struct v_float32x4; -struct v_float64x2; -#endif - -/** Value reordering **/ - -// Expansion -void v_expand(const __CV_V_UINT8&, __CV_V_UINT16&, __CV_V_UINT16&); -void v_expand(const __CV_V_INT8&, __CV_V_INT16&, __CV_V_INT16&); -void v_expand(const __CV_V_UINT16&, __CV_V_UINT32&, __CV_V_UINT32&); -void v_expand(const __CV_V_INT16&, __CV_V_INT32&, __CV_V_INT32&); -void v_expand(const __CV_V_UINT32&, __CV_V_UINT64&, __CV_V_UINT64&); -void v_expand(const __CV_V_INT32&, __CV_V_INT64&, __CV_V_INT64&); -// Low Expansion -__CV_V_UINT16 v_expand_low(const __CV_V_UINT8&); -__CV_V_INT16 v_expand_low(const __CV_V_INT8&); -__CV_V_UINT32 v_expand_low(const __CV_V_UINT16&); -__CV_V_INT32 v_expand_low(const __CV_V_INT16&); -__CV_V_UINT64 v_expand_low(const __CV_V_UINT32&); -__CV_V_INT64 v_expand_low(const __CV_V_INT32&); -// High Expansion -__CV_V_UINT16 v_expand_high(const __CV_V_UINT8&); -__CV_V_INT16 v_expand_high(const __CV_V_INT8&); -__CV_V_UINT32 v_expand_high(const __CV_V_UINT16&); -__CV_V_INT32 v_expand_high(const __CV_V_INT16&); -__CV_V_UINT64 v_expand_high(const __CV_V_UINT32&); -__CV_V_INT64 v_expand_high(const __CV_V_INT32&); -// Load & Low Expansion -__CV_V_UINT16 __CV_VX(load_expand)(const uchar*); -__CV_V_INT16 __CV_VX(load_expand)(const schar*); -__CV_V_UINT32 __CV_VX(load_expand)(const ushort*); -__CV_V_INT32 __CV_VX(load_expand)(const short*); -__CV_V_UINT64 __CV_VX(load_expand)(const uint*); -__CV_V_INT64 __CV_VX(load_expand)(const int*); -// Load lower 8-bit and expand into 32-bit -__CV_V_UINT32 __CV_VX(load_expand_q)(const uchar*); -__CV_V_INT32 __CV_VX(load_expand_q)(const schar*); - -// Saturating Pack -__CV_V_UINT8 v_pack(const __CV_V_UINT16&, const __CV_V_UINT16&); -__CV_V_INT8 v_pack(const __CV_V_INT16&, const __CV_V_INT16&); -__CV_V_UINT16 v_pack(const __CV_V_UINT32&, const __CV_V_UINT32&); -__CV_V_INT16 v_pack(const __CV_V_INT32&, const __CV_V_INT32&); -// Non-saturating Pack -__CV_V_UINT32 v_pack(const __CV_V_UINT64&, const __CV_V_UINT64&); -__CV_V_INT32 v_pack(const __CV_V_INT64&, const __CV_V_INT64&); -// Pack signed integers with unsigned saturation -__CV_V_UINT8 v_pack_u(const __CV_V_INT16&, const __CV_V_INT16&); -__CV_V_UINT16 v_pack_u(const __CV_V_INT32&, const __CV_V_INT32&); - -/** Arithmetic, bitwise and comparison operations **/ - -// Non-saturating multiply -#if CV_VSX -template -Tvec v_mul_wrap(const Tvec& a, const Tvec& b); -#else -__CV_V_UINT8 v_mul_wrap(const __CV_V_UINT8&, const __CV_V_UINT8&); -__CV_V_INT8 v_mul_wrap(const __CV_V_INT8&, const __CV_V_INT8&); -__CV_V_UINT16 v_mul_wrap(const __CV_V_UINT16&, const __CV_V_UINT16&); -__CV_V_INT16 v_mul_wrap(const __CV_V_INT16&, const __CV_V_INT16&); -#endif - -// Multiply and expand -#if CV_VSX -template -void v_mul_expand(const Tvec& a, const Tvec& b, Twvec& c, Twvec& d); -#else -void v_mul_expand(const __CV_V_UINT8&, const __CV_V_UINT8&, __CV_V_UINT16&, __CV_V_UINT16&); -void v_mul_expand(const __CV_V_INT8&, const __CV_V_INT8&, __CV_V_INT16&, __CV_V_INT16&); -void v_mul_expand(const __CV_V_UINT16&, const __CV_V_UINT16&, __CV_V_UINT32&, __CV_V_UINT32&); -void v_mul_expand(const __CV_V_INT16&, const __CV_V_INT16&, __CV_V_INT32&, __CV_V_INT32&); -void v_mul_expand(const __CV_V_UINT32&, const __CV_V_UINT32&, __CV_V_UINT64&, __CV_V_UINT64&); -void v_mul_expand(const __CV_V_INT32&, const __CV_V_INT32&, __CV_V_INT64&, __CV_V_INT64&); -#endif - -// Conversions -__CV_V_FLOAT32 v_cvt_f32(const __CV_V_INT32& a); -__CV_V_FLOAT32 v_cvt_f32(const __CV_V_FLOAT64& a); -__CV_V_FLOAT32 v_cvt_f32(const __CV_V_FLOAT64& a, const __CV_V_FLOAT64& b); -__CV_V_FLOAT64 v_cvt_f64(const __CV_V_INT32& a); -__CV_V_FLOAT64 v_cvt_f64_high(const __CV_V_INT32& a); -__CV_V_FLOAT64 v_cvt_f64(const __CV_V_FLOAT32& a); -__CV_V_FLOAT64 v_cvt_f64_high(const __CV_V_FLOAT32& a); -__CV_V_FLOAT64 v_cvt_f64(const __CV_V_INT64& a); - -/** Cleanup **/ -#undef CV__SIMD_FORWARD -#undef __CV_VX -#undef __CV_V_UINT8 -#undef __CV_V_INT8 -#undef __CV_V_UINT16 -#undef __CV_V_INT16 -#undef __CV_V_UINT32 -#undef __CV_V_INT32 -#undef __CV_V_UINT64 -#undef __CV_V_INT64 -#undef __CV_V_FLOAT32 -#undef __CV_V_FLOAT64 - -CV_CPU_OPTIMIZATION_HAL_NAMESPACE_END - -//! @endcond - -} // cv:: diff --git a/thirdparty/ocv/opencv_hal_neon.hpp b/thirdparty/ocv/opencv_hal_neon.hpp deleted file mode 100644 index 00f2036c451..00000000000 --- a/thirdparty/ocv/opencv_hal_neon.hpp +++ /dev/null @@ -1,2661 +0,0 @@ -/*M/////////////////////////////////////////////////////////////////////////////////////// -// -// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. -// -// By downloading, copying, installing or using the software you agree to this license. -// If you do not agree to this license, do not download, install, -// copy or use the software. -// -// -// License Agreement -// For Open Source Computer Vision Library -// -// Copyright (C) 2000-2008, Intel Corporation, all rights reserved. -// Copyright (C) 2009, Willow Garage Inc., all rights reserved. -// Copyright (C) 2013, OpenCV Foundation, all rights reserved. -// Copyright (C) 2015, Itseez Inc., all rights reserved. -// Third party copyrights are property of their respective owners. -// -// Redistribution and use in source and binary forms, with or without modification, -// are permitted provided that the following conditions are met: -// -// * Redistribution's of source code must retain the above copyright notice, -// this list of conditions and the following disclaimer. -// -// * Redistribution's in binary form must reproduce the above copyright notice, -// this list of conditions and the following disclaimer in the documentation -// and/or other materials provided with the distribution. -// -// * The name of the copyright holders may not be used to endorse or promote products -// derived from this software without specific prior written permission. -// -// This software is provided by the copyright holders and contributors "as is" and -// any express or implied warranties, including, but not limited to, the implied -// warranties of merchantability and fitness for a particular purpose are disclaimed. -// In no event shall the Intel Corporation or contributors be liable for any direct, -// indirect, incidental, special, exemplary, or consequential damages -// (including, but not limited to, procurement of substitute goods or services; -// loss of use, data, or profits; or business interruption) however caused -// and on any theory of liability, whether in contract, strict liability, -// or tort (including negligence or otherwise) arising in any way out of -// the use of this software, even if advised of the possibility of such damage. -// -//M*/ - -#ifndef OPENCV_HAL_INTRIN_NEON_HPP -#define OPENCV_HAL_INTRIN_NEON_HPP - -#include -#include - - -namespace cv -{ - -// This nessecary for actual type of compiler intrinsic argument. - -static_assert(sizeof(uint8_t) == sizeof(unsigned char), "Size of uint8_t doesn't equal to size of uchar"); -using uchar = uint8_t; - -static_assert(sizeof(int8_t) == sizeof(char), "Size of int8_t doesn't equal to size of schar"); -using schar = int8_t; - -static_assert(sizeof(uint16_t) == sizeof(unsigned short), "Size of uint16_t doesn't equal to size of ushort"); -using ushort = uint16_t; - -static_assert(sizeof(int16_t) == sizeof(short), "Size of int16_t doesn't equal to size of short"); -typedef short int16_t ; - -static_assert(sizeof(uint32_t) == sizeof(unsigned), "Size of uint32_t doesn't equal to size of unsigned"); -typedef unsigned uint32_t; - -static_assert(sizeof(int32_t) == sizeof(int), "Size of int32_t doesn't equal to size of int"); -typedef int int32_t; - -//! @cond IGNORED - -CV_CPU_OPTIMIZATION_HAL_NAMESPACE_BEGIN - -#define CV_SIMD128 1 -#if defined(__aarch64__) -#define CV_SIMD128_64F 1 -#else -#define CV_SIMD128_64F 0 -#endif - -#define CV_UNUSED(name) (void)name - -// TODO -#define CV_NEON_DOT 0 - -//////////// Utils //////////// - -#if CV_SIMD128_64F -#define OPENCV_HAL_IMPL_NEON_UNZIP(_Tpv, _Tpvx2, suffix) \ - inline void _v128_unzip(const _Tpv& a, const _Tpv& b, _Tpv& c, _Tpv& d) \ - { c = vuzp1q_##suffix(a, b); d = vuzp2q_##suffix(a, b); } -#define OPENCV_HAL_IMPL_NEON_UNZIP_L(_Tpv, _Tpvx2, suffix) \ - inline void _v128_unzip(const _Tpv&a, const _Tpv&b, _Tpv& c, _Tpv& d) \ - { c = vuzp1_##suffix(a, b); d = vuzp2_##suffix(a, b); } -#else -#define OPENCV_HAL_IMPL_NEON_UNZIP(_Tpv, _Tpvx2, suffix) \ - inline void _v128_unzip(const _Tpv& a, const _Tpv& b, _Tpv& c, _Tpv& d) \ - { _Tpvx2 ab = vuzpq_##suffix(a, b); c = ab.val[0]; d = ab.val[1]; } -#define OPENCV_HAL_IMPL_NEON_UNZIP_L(_Tpv, _Tpvx2, suffix) \ - inline void _v128_unzip(const _Tpv& a, const _Tpv& b, _Tpv& c, _Tpv& d) \ - { _Tpvx2 ab = vuzp_##suffix(a, b); c = ab.val[0]; d = ab.val[1]; } -#endif - -#if CV_SIMD128_64F -#define OPENCV_HAL_IMPL_NEON_REINTERPRET(_Tpv, suffix) \ - template static inline \ - _Tpv vreinterpretq_##suffix##_f64(T a) { return (_Tpv) a; } \ - template static inline \ - float64x2_t vreinterpretq_f64_##suffix(T a) { return (float64x2_t) a; } -#else -#define OPENCV_HAL_IMPL_NEON_REINTERPRET(_Tpv, suffix) -#endif - -#define OPENCV_HAL_IMPL_NEON_UTILS_SUFFIX(_Tpv, _Tpvl, suffix) \ - OPENCV_HAL_IMPL_NEON_UNZIP(_Tpv##_t, _Tpv##x2_t, suffix) \ - OPENCV_HAL_IMPL_NEON_UNZIP_L(_Tpvl##_t, _Tpvl##x2_t, suffix) \ - OPENCV_HAL_IMPL_NEON_REINTERPRET(_Tpv##_t, suffix) - -#define OPENCV_HAL_IMPL_NEON_UTILS_SUFFIX_I64(_Tpv, _Tpvl, suffix) \ - OPENCV_HAL_IMPL_NEON_REINTERPRET(_Tpv##_t, suffix) - -#define OPENCV_HAL_IMPL_NEON_UTILS_SUFFIX_F64(_Tpv, _Tpvl, suffix) \ - OPENCV_HAL_IMPL_NEON_UNZIP(_Tpv##_t, _Tpv##x2_t, suffix) - -OPENCV_HAL_IMPL_NEON_UTILS_SUFFIX(uint8x16, uint8x8, u8) -OPENCV_HAL_IMPL_NEON_UTILS_SUFFIX(int8x16, int8x8, s8) -OPENCV_HAL_IMPL_NEON_UTILS_SUFFIX(uint16x8, uint16x4, u16) -OPENCV_HAL_IMPL_NEON_UTILS_SUFFIX(int16x8, int16x4, s16) -OPENCV_HAL_IMPL_NEON_UTILS_SUFFIX(uint32x4, uint32x2, u32) -OPENCV_HAL_IMPL_NEON_UTILS_SUFFIX(int32x4, int32x2, s32) -OPENCV_HAL_IMPL_NEON_UTILS_SUFFIX(float32x4, float32x2, f32) -OPENCV_HAL_IMPL_NEON_UTILS_SUFFIX_I64(uint64x2, uint64x1, u64) -OPENCV_HAL_IMPL_NEON_UTILS_SUFFIX_I64(int64x2, int64x1, s64) -#if CV_SIMD128_64F -OPENCV_HAL_IMPL_NEON_UTILS_SUFFIX_F64(float64x2, float64x1,f64) -#endif - -//////////// Types //////////// - -struct v_uint8x16 -{ - typedef uchar lane_type; - enum { nlanes = 16 }; - - v_uint8x16() {} - explicit v_uint8x16(uint8x16_t v) : val(v) {} - v_uint8x16(uchar v0, uchar v1, uchar v2, uchar v3, uchar v4, uchar v5, uchar v6, uchar v7, - uchar v8, uchar v9, uchar v10, uchar v11, uchar v12, uchar v13, uchar v14, uchar v15) - { - uchar v[] = {v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15}; - val = vld1q_u8(v); - } - uchar get0() const - { - return vgetq_lane_u8(val, 0); - } - - uint8x16_t val; -}; - -struct v_int8x16 -{ - typedef schar lane_type; - enum { nlanes = 16 }; - - v_int8x16() {} - explicit v_int8x16(int8x16_t v) : val(v) {} - v_int8x16(schar v0, schar v1, schar v2, schar v3, schar v4, schar v5, schar v6, schar v7, - schar v8, schar v9, schar v10, schar v11, schar v12, schar v13, schar v14, schar v15) - { - schar v[] = {v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15}; - val = vld1q_s8(v); - } - schar get0() const - { - return vgetq_lane_s8(val, 0); - } - - int8x16_t val; -}; - -struct v_uint16x8 -{ - typedef ushort lane_type; - enum { nlanes = 8 }; - - v_uint16x8() {} - explicit v_uint16x8(uint16x8_t v) : val(v) {} - v_uint16x8(ushort v0, ushort v1, ushort v2, ushort v3, ushort v4, ushort v5, ushort v6, ushort v7) - { - ushort v[] = {v0, v1, v2, v3, v4, v5, v6, v7}; - val = vld1q_u16(v); - } - ushort get0() const - { - return vgetq_lane_u16(val, 0); - } - - uint16x8_t val; -}; - -struct v_int16x8 -{ - typedef short lane_type; - enum { nlanes = 8 }; - - v_int16x8() {} - explicit v_int16x8(int16x8_t v) : val(v) {} - v_int16x8(short v0, short v1, short v2, short v3, short v4, short v5, short v6, short v7) - { - short v[] = {v0, v1, v2, v3, v4, v5, v6, v7}; - val = vld1q_s16(v); - } - short get0() const - { - return vgetq_lane_s16(val, 0); - } - - int16x8_t val; -}; - -struct v_uint32x4 -{ - typedef unsigned lane_type; - enum { nlanes = 4 }; - - v_uint32x4() {} - explicit v_uint32x4(uint32x4_t v) : val(v) {} - v_uint32x4(unsigned v0, unsigned v1, unsigned v2, unsigned v3) - { - unsigned v[] = {v0, v1, v2, v3}; - val = vld1q_u32(v); - } - unsigned get0() const - { - return vgetq_lane_u32(val, 0); - } - - uint32x4_t val; -}; - -struct v_int32x4 -{ - typedef int lane_type; - enum { nlanes = 4 }; - - v_int32x4() {} - explicit v_int32x4(int32x4_t v) : val(v) {} - v_int32x4(int v0, int v1, int v2, int v3) - { - int v[] = {v0, v1, v2, v3}; - val = vld1q_s32(v); - } - int get0() const - { - return vgetq_lane_s32(val, 0); - } - int32x4_t val; -}; - -struct v_float32x4 -{ - typedef float lane_type; - enum { nlanes = 4 }; - - v_float32x4() {} - explicit v_float32x4(float32x4_t v) : val(v) {} - v_float32x4(float v0, float v1, float v2, float v3) - { - float v[] = {v0, v1, v2, v3}; - val = vld1q_f32(v); - } - float get0() const - { - return vgetq_lane_f32(val, 0); - } - float32x4_t val; -}; - -struct v_uint64x2 -{ - typedef uint64 lane_type; - enum { nlanes = 2 }; - - v_uint64x2() {} - explicit v_uint64x2(uint64x2_t v) : val(v) {} - v_uint64x2(uint64 v0, uint64 v1) - { - uint64 v[] = {v0, v1}; - val = vld1q_u64(v); - } - uint64 get0() const - { - return vgetq_lane_u64(val, 0); - } - uint64x2_t val; -}; - -struct v_int64x2 -{ - typedef int64 lane_type; - enum { nlanes = 2 }; - - v_int64x2() {} - explicit v_int64x2(int64x2_t v) : val(v) {} - v_int64x2(int64 v0, int64 v1) - { - int64 v[] = {v0, v1}; - val = vld1q_s64(v); - } - int64 get0() const - { - return vgetq_lane_s64(val, 0); - } - int64x2_t val; -}; - -#if CV_SIMD128_64F -struct v_float64x2 -{ - typedef double lane_type; - enum { nlanes = 2 }; - - v_float64x2() {} - explicit v_float64x2(float64x2_t v) : val(v) {} - v_float64x2(double v0, double v1) - { - double v[] = {v0, v1}; - val = vld1q_f64(v); - } - double get0() const - { - return vgetq_lane_f64(val, 0); - } - float64x2_t val; -}; -#endif - -#define OPENCV_HAL_IMPL_NEON_INIT(_Tpv, _Tp, suffix) \ -inline v_##_Tpv v_setzero_##suffix() { return v_##_Tpv(vdupq_n_##suffix((_Tp)0)); } \ -inline v_##_Tpv v_setall_##suffix(_Tp v) { return v_##_Tpv(vdupq_n_##suffix(v)); } \ -inline _Tpv##_t vreinterpretq_##suffix##_##suffix(_Tpv##_t v) { return v; } \ -inline v_uint8x16 v_reinterpret_as_u8(const v_##_Tpv& v) { return v_uint8x16(vreinterpretq_u8_##suffix(v.val)); } \ -inline v_int8x16 v_reinterpret_as_s8(const v_##_Tpv& v) { return v_int8x16(vreinterpretq_s8_##suffix(v.val)); } \ -inline v_uint16x8 v_reinterpret_as_u16(const v_##_Tpv& v) { return v_uint16x8(vreinterpretq_u16_##suffix(v.val)); } \ -inline v_int16x8 v_reinterpret_as_s16(const v_##_Tpv& v) { return v_int16x8(vreinterpretq_s16_##suffix(v.val)); } \ -inline v_uint32x4 v_reinterpret_as_u32(const v_##_Tpv& v) { return v_uint32x4(vreinterpretq_u32_##suffix(v.val)); } \ -inline v_int32x4 v_reinterpret_as_s32(const v_##_Tpv& v) { return v_int32x4(vreinterpretq_s32_##suffix(v.val)); } \ -inline v_uint64x2 v_reinterpret_as_u64(const v_##_Tpv& v) { return v_uint64x2(vreinterpretq_u64_##suffix(v.val)); } \ -inline v_int64x2 v_reinterpret_as_s64(const v_##_Tpv& v) { return v_int64x2(vreinterpretq_s64_##suffix(v.val)); } \ -inline v_float32x4 v_reinterpret_as_f32(const v_##_Tpv& v) { return v_float32x4(vreinterpretq_f32_##suffix(v.val)); } - -OPENCV_HAL_IMPL_NEON_INIT(uint8x16, uchar, u8) -OPENCV_HAL_IMPL_NEON_INIT(int8x16, schar, s8) -OPENCV_HAL_IMPL_NEON_INIT(uint16x8, ushort, u16) -OPENCV_HAL_IMPL_NEON_INIT(int16x8, short, s16) -OPENCV_HAL_IMPL_NEON_INIT(uint32x4, unsigned, u32) -OPENCV_HAL_IMPL_NEON_INIT(int32x4, int, s32) -OPENCV_HAL_IMPL_NEON_INIT(uint64x2, uint64, u64) -OPENCV_HAL_IMPL_NEON_INIT(int64x2, int64, s64) -OPENCV_HAL_IMPL_NEON_INIT(float32x4, float, f32) -#if CV_SIMD128_64F -#define OPENCV_HAL_IMPL_NEON_INIT_64(_Tpv, suffix) \ -inline v_float64x2 v_reinterpret_as_f64(const v_##_Tpv& v) { return v_float64x2(vreinterpretq_f64_##suffix(v.val)); } -OPENCV_HAL_IMPL_NEON_INIT(float64x2, double, f64) -OPENCV_HAL_IMPL_NEON_INIT_64(uint8x16, u8) -OPENCV_HAL_IMPL_NEON_INIT_64(int8x16, s8) -OPENCV_HAL_IMPL_NEON_INIT_64(uint16x8, u16) -OPENCV_HAL_IMPL_NEON_INIT_64(int16x8, s16) -OPENCV_HAL_IMPL_NEON_INIT_64(uint32x4, u32) -OPENCV_HAL_IMPL_NEON_INIT_64(int32x4, s32) -OPENCV_HAL_IMPL_NEON_INIT_64(uint64x2, u64) -OPENCV_HAL_IMPL_NEON_INIT_64(int64x2, s64) -OPENCV_HAL_IMPL_NEON_INIT_64(float32x4, f32) -OPENCV_HAL_IMPL_NEON_INIT_64(float64x2, f64) -#endif - -#define OPENCV_HAL_IMPL_NEON_PACK(_Tpvec, _Tp, hreg, suffix, _Tpwvec, pack, mov, rshr) \ -inline _Tpvec v_##pack(const _Tpwvec& a, const _Tpwvec& b) \ -{ \ - hreg a1 = mov(a.val), b1 = mov(b.val); \ - return _Tpvec(vcombine_##suffix(a1, b1)); \ -} \ -inline void v_##pack##_store(_Tp* ptr, const _Tpwvec& a) \ -{ \ - hreg a1 = mov(a.val); \ - vst1_##suffix(ptr, a1); \ -} \ -template inline \ -_Tpvec v_rshr_##pack(const _Tpwvec& a, const _Tpwvec& b) \ -{ \ - hreg a1 = rshr(a.val, n); \ - hreg b1 = rshr(b.val, n); \ - return _Tpvec(vcombine_##suffix(a1, b1)); \ -} \ -template inline \ -void v_rshr_##pack##_store(_Tp* ptr, const _Tpwvec& a) \ -{ \ - hreg a1 = rshr(a.val, n); \ - vst1_##suffix(ptr, a1); \ -} - -OPENCV_HAL_IMPL_NEON_PACK(v_uint8x16, uchar, uint8x8_t, u8, v_uint16x8, pack, vqmovn_u16, vqrshrn_n_u16) -OPENCV_HAL_IMPL_NEON_PACK(v_int8x16, schar, int8x8_t, s8, v_int16x8, pack, vqmovn_s16, vqrshrn_n_s16) -OPENCV_HAL_IMPL_NEON_PACK(v_uint16x8, ushort, uint16x4_t, u16, v_uint32x4, pack, vqmovn_u32, vqrshrn_n_u32) -OPENCV_HAL_IMPL_NEON_PACK(v_int16x8, short, int16x4_t, s16, v_int32x4, pack, vqmovn_s32, vqrshrn_n_s32) -OPENCV_HAL_IMPL_NEON_PACK(v_uint32x4, unsigned, uint32x2_t, u32, v_uint64x2, pack, vmovn_u64, vrshrn_n_u64) -OPENCV_HAL_IMPL_NEON_PACK(v_int32x4, int, int32x2_t, s32, v_int64x2, pack, vmovn_s64, vrshrn_n_s64) - -OPENCV_HAL_IMPL_NEON_PACK(v_uint8x16, uchar, uint8x8_t, u8, v_int16x8, pack_u, vqmovun_s16, vqrshrun_n_s16) -OPENCV_HAL_IMPL_NEON_PACK(v_uint16x8, ushort, uint16x4_t, u16, v_int32x4, pack_u, vqmovun_s32, vqrshrun_n_s32) - -// pack boolean -inline v_uint8x16 v_pack_b(const v_uint16x8& a, const v_uint16x8& b) -{ - uint8x16_t ab = vcombine_u8(vmovn_u16(a.val), vmovn_u16(b.val)); - return v_uint8x16(ab); -} - -inline v_uint8x16 v_pack_b(const v_uint32x4& a, const v_uint32x4& b, - const v_uint32x4& c, const v_uint32x4& d) -{ - uint16x8_t nab = vcombine_u16(vmovn_u32(a.val), vmovn_u32(b.val)); - uint16x8_t ncd = vcombine_u16(vmovn_u32(c.val), vmovn_u32(d.val)); - return v_uint8x16(vcombine_u8(vmovn_u16(nab), vmovn_u16(ncd))); -} - -inline v_uint8x16 v_pack_b(const v_uint64x2& a, const v_uint64x2& b, const v_uint64x2& c, - const v_uint64x2& d, const v_uint64x2& e, const v_uint64x2& f, - const v_uint64x2& g, const v_uint64x2& h) -{ - uint32x4_t ab = vcombine_u32(vmovn_u64(a.val), vmovn_u64(b.val)); - uint32x4_t cd = vcombine_u32(vmovn_u64(c.val), vmovn_u64(d.val)); - uint32x4_t ef = vcombine_u32(vmovn_u64(e.val), vmovn_u64(f.val)); - uint32x4_t gh = vcombine_u32(vmovn_u64(g.val), vmovn_u64(h.val)); - - uint16x8_t abcd = vcombine_u16(vmovn_u32(ab), vmovn_u32(cd)); - uint16x8_t efgh = vcombine_u16(vmovn_u32(ef), vmovn_u32(gh)); - return v_uint8x16(vcombine_u8(vmovn_u16(abcd), vmovn_u16(efgh))); -} - -inline v_float32x4 v_matmul(const v_float32x4& v, const v_float32x4& m0, - const v_float32x4& m1, const v_float32x4& m2, - const v_float32x4& m3) -{ - float32x2_t vl = vget_low_f32(v.val), vh = vget_high_f32(v.val); - float32x4_t res = vmulq_lane_f32(m0.val, vl, 0); - res = vmlaq_lane_f32(res, m1.val, vl, 1); - res = vmlaq_lane_f32(res, m2.val, vh, 0); - res = vmlaq_lane_f32(res, m3.val, vh, 1); - return v_float32x4(res); -} - -#define OPENCV_HAL_IMPL_NEON_BIN_OP(bin_op, _Tpvec, intrin) \ -inline _Tpvec operator bin_op (const _Tpvec& a, const _Tpvec& b) \ -{ \ - return _Tpvec(intrin(a.val, b.val)); \ -} \ -inline _Tpvec& operator bin_op##= (_Tpvec& a, const _Tpvec& b) \ -{ \ - a.val = intrin(a.val, b.val); \ - return a; \ -} - -OPENCV_HAL_IMPL_NEON_BIN_OP(+, v_uint8x16, vqaddq_u8) -OPENCV_HAL_IMPL_NEON_BIN_OP(-, v_uint8x16, vqsubq_u8) -OPENCV_HAL_IMPL_NEON_BIN_OP(+, v_int8x16, vqaddq_s8) -OPENCV_HAL_IMPL_NEON_BIN_OP(-, v_int8x16, vqsubq_s8) -OPENCV_HAL_IMPL_NEON_BIN_OP(+, v_uint16x8, vqaddq_u16) -OPENCV_HAL_IMPL_NEON_BIN_OP(-, v_uint16x8, vqsubq_u16) -OPENCV_HAL_IMPL_NEON_BIN_OP(+, v_int16x8, vqaddq_s16) -OPENCV_HAL_IMPL_NEON_BIN_OP(-, v_int16x8, vqsubq_s16) -OPENCV_HAL_IMPL_NEON_BIN_OP(+, v_int32x4, vaddq_s32) -OPENCV_HAL_IMPL_NEON_BIN_OP(-, v_int32x4, vsubq_s32) -OPENCV_HAL_IMPL_NEON_BIN_OP(*, v_int32x4, vmulq_s32) -OPENCV_HAL_IMPL_NEON_BIN_OP(+, v_uint32x4, vaddq_u32) -OPENCV_HAL_IMPL_NEON_BIN_OP(-, v_uint32x4, vsubq_u32) -OPENCV_HAL_IMPL_NEON_BIN_OP(*, v_uint32x4, vmulq_u32) -OPENCV_HAL_IMPL_NEON_BIN_OP(+, v_float32x4, vaddq_f32) -OPENCV_HAL_IMPL_NEON_BIN_OP(-, v_float32x4, vsubq_f32) -OPENCV_HAL_IMPL_NEON_BIN_OP(*, v_float32x4, vmulq_f32) -OPENCV_HAL_IMPL_NEON_BIN_OP(+, v_int64x2, vaddq_s64) -OPENCV_HAL_IMPL_NEON_BIN_OP(-, v_int64x2, vsubq_s64) -OPENCV_HAL_IMPL_NEON_BIN_OP(+, v_uint64x2, vaddq_u64) -OPENCV_HAL_IMPL_NEON_BIN_OP(-, v_uint64x2, vsubq_u64) -#if CV_SIMD128_64F -OPENCV_HAL_IMPL_NEON_BIN_OP(/, v_float32x4, vdivq_f32) -OPENCV_HAL_IMPL_NEON_BIN_OP(+, v_float64x2, vaddq_f64) -OPENCV_HAL_IMPL_NEON_BIN_OP(-, v_float64x2, vsubq_f64) -OPENCV_HAL_IMPL_NEON_BIN_OP(*, v_float64x2, vmulq_f64) -OPENCV_HAL_IMPL_NEON_BIN_OP(/, v_float64x2, vdivq_f64) -#else -inline v_float32x4 operator / (const v_float32x4& a, const v_float32x4& b) -{ - float32x4_t reciprocal = vrecpeq_f32(b.val); - reciprocal = vmulq_f32(vrecpsq_f32(b.val, reciprocal), reciprocal); - reciprocal = vmulq_f32(vrecpsq_f32(b.val, reciprocal), reciprocal); - return v_float32x4(vmulq_f32(a.val, reciprocal)); -} -inline v_float32x4& operator /= (v_float32x4& a, const v_float32x4& b) -{ - float32x4_t reciprocal = vrecpeq_f32(b.val); - reciprocal = vmulq_f32(vrecpsq_f32(b.val, reciprocal), reciprocal); - reciprocal = vmulq_f32(vrecpsq_f32(b.val, reciprocal), reciprocal); - a.val = vmulq_f32(a.val, reciprocal); - return a; -} -#endif - -static inline v_float32x4 v_matmuladd(const v_float32x4& v, const v_float32x4& m0, - const v_float32x4& m1, const v_float32x4& m2, - const v_float32x4& a) -{ - float32x2_t vl = vget_low_f32(v.val), vh = vget_high_f32(v.val); - float32x4_t res = vmulq_lane_f32(m0.val, vl, 0); - res = vmlaq_lane_f32(res, m1.val, vl, 1); - res = vmlaq_lane_f32(res, m2.val, vh, 0); - res = vaddq_f32(res, a.val); - return v_float32x4(res); -} - -// saturating multiply 8-bit, 16-bit -#define OPENCV_HAL_IMPL_NEON_MUL_SAT(_Tpvec, _Tpwvec) \ - inline _Tpvec operator * (const _Tpvec& a, const _Tpvec& b) \ - { \ - _Tpwvec c, d; \ - v_mul_expand(a, b, c, d); \ - return v_pack(c, d); \ - } \ - inline _Tpvec& operator *= (_Tpvec& a, const _Tpvec& b) \ - { a = a * b; return a; } - -OPENCV_HAL_IMPL_NEON_MUL_SAT(v_int8x16, v_int16x8) -OPENCV_HAL_IMPL_NEON_MUL_SAT(v_uint8x16, v_uint16x8) -OPENCV_HAL_IMPL_NEON_MUL_SAT(v_int16x8, v_int32x4) -OPENCV_HAL_IMPL_NEON_MUL_SAT(v_uint16x8, v_uint32x4) - -// Multiply and expand -inline void v_mul_expand(const v_int8x16& a, const v_int8x16& b, - v_int16x8& c, v_int16x8& d) -{ - c.val = vmull_s8(vget_low_s8(a.val), vget_low_s8(b.val)); - d.val = vmull_s8(vget_high_s8(a.val), vget_high_s8(b.val)); -} - -inline void v_mul_expand(const v_uint8x16& a, const v_uint8x16& b, - v_uint16x8& c, v_uint16x8& d) -{ - c.val = vmull_u8(vget_low_u8(a.val), vget_low_u8(b.val)); - d.val = vmull_u8(vget_high_u8(a.val), vget_high_u8(b.val)); -} - -inline void v_mul_expand(const v_int16x8& a, const v_int16x8& b, - v_int32x4& c, v_int32x4& d) -{ - c.val = vmull_s16(vget_low_s16(a.val), vget_low_s16(b.val)); - d.val = vmull_s16(vget_high_s16(a.val), vget_high_s16(b.val)); -} - -inline void v_mul_expand(const v_uint16x8& a, const v_uint16x8& b, - v_uint32x4& c, v_uint32x4& d) -{ - c.val = vmull_u16(vget_low_u16(a.val), vget_low_u16(b.val)); - d.val = vmull_u16(vget_high_u16(a.val), vget_high_u16(b.val)); -} - -inline void v_mul_expand(const v_uint32x4& a, const v_uint32x4& b, - v_uint64x2& c, v_uint64x2& d) -{ - c.val = vmull_u32(vget_low_u32(a.val), vget_low_u32(b.val)); - d.val = vmull_u32(vget_high_u32(a.val), vget_high_u32(b.val)); -} - -inline v_int16x8 v_mul_hi(const v_int16x8& a, const v_int16x8& b) -{ - return v_int16x8(vcombine_s16( - vshrn_n_s32(vmull_s16( vget_low_s16(a.val), vget_low_s16(b.val)), 16), - vshrn_n_s32(vmull_s16(vget_high_s16(a.val), vget_high_s16(b.val)), 16) - )); -} -inline v_uint16x8 v_mul_hi(const v_uint16x8& a, const v_uint16x8& b) -{ - return v_uint16x8(vcombine_u16( - vshrn_n_u32(vmull_u16( vget_low_u16(a.val), vget_low_u16(b.val)), 16), - vshrn_n_u32(vmull_u16(vget_high_u16(a.val), vget_high_u16(b.val)), 16) - )); -} - -//////// Dot Product //////// - -// 16 >> 32 -inline v_int32x4 v_dotprod(const v_int16x8& a, const v_int16x8& b) -{ - int16x8_t uzp1, uzp2; - _v128_unzip(a.val, b.val, uzp1, uzp2); - int16x4_t a0 = vget_low_s16(uzp1); - int16x4_t b0 = vget_high_s16(uzp1); - int16x4_t a1 = vget_low_s16(uzp2); - int16x4_t b1 = vget_high_s16(uzp2); - int32x4_t p = vmull_s16(a0, b0); - return v_int32x4(vmlal_s16(p, a1, b1)); -} -inline v_int32x4 v_dotprod(const v_int16x8& a, const v_int16x8& b, const v_int32x4& c) -{ - int16x8_t uzp1, uzp2; - _v128_unzip(a.val, b.val, uzp1, uzp2); - int16x4_t a0 = vget_low_s16(uzp1); - int16x4_t b0 = vget_high_s16(uzp1); - int16x4_t a1 = vget_low_s16(uzp2); - int16x4_t b1 = vget_high_s16(uzp2); - int32x4_t p = vmlal_s16(c.val, a0, b0); - return v_int32x4(vmlal_s16(p, a1, b1)); -} - -// 32 >> 64 -inline v_int64x2 v_dotprod(const v_int32x4& a, const v_int32x4& b) -{ - int32x4_t uzp1, uzp2; - _v128_unzip(a.val, b.val, uzp1, uzp2); - int32x2_t a0 = vget_low_s32(uzp1); - int32x2_t b0 = vget_high_s32(uzp1); - int32x2_t a1 = vget_low_s32(uzp2); - int32x2_t b1 = vget_high_s32(uzp2); - int64x2_t p = vmull_s32(a0, b0); - return v_int64x2(vmlal_s32(p, a1, b1)); -} -inline v_int64x2 v_dotprod(const v_int32x4& a, const v_int32x4& b, const v_int64x2& c) -{ - int32x4_t uzp1, uzp2; - _v128_unzip(a.val, b.val, uzp1, uzp2); - int32x2_t a0 = vget_low_s32(uzp1); - int32x2_t b0 = vget_high_s32(uzp1); - int32x2_t a1 = vget_low_s32(uzp2); - int32x2_t b1 = vget_high_s32(uzp2); - int64x2_t p = vmlal_s32(c.val, a0, b0); - return v_int64x2(vmlal_s32(p, a1, b1)); -} - -// 8 >> 32 -inline v_uint32x4 v_dotprod_expand(const v_uint8x16& a, const v_uint8x16& b) -{ -#if CV_NEON_DOT - return v_uint32x4(vdotq_u32(vdupq_n_u32(0), a.val, b.val)); -#else - const uint8x16_t zero = vreinterpretq_u8_u32(vdupq_n_u32(0)); - const uint8x16_t mask = vreinterpretq_u8_u32(vdupq_n_u32(0x00FF00FF)); - const uint16x8_t zero32 = vreinterpretq_u16_u32(vdupq_n_u32(0)); - const uint16x8_t mask32 = vreinterpretq_u16_u32(vdupq_n_u32(0x0000FFFF)); - - uint16x8_t even = vmulq_u16(vreinterpretq_u16_u8(vbslq_u8(mask, a.val, zero)), - vreinterpretq_u16_u8(vbslq_u8(mask, b.val, zero))); - uint16x8_t odd = vmulq_u16(vshrq_n_u16(vreinterpretq_u16_u8(a.val), 8), - vshrq_n_u16(vreinterpretq_u16_u8(b.val), 8)); - - uint32x4_t s0 = vaddq_u32(vreinterpretq_u32_u16(vbslq_u16(mask32, even, zero32)), - vreinterpretq_u32_u16(vbslq_u16(mask32, odd, zero32))); - uint32x4_t s1 = vaddq_u32(vshrq_n_u32(vreinterpretq_u32_u16(even), 16), - vshrq_n_u32(vreinterpretq_u32_u16(odd), 16)); - return v_uint32x4(vaddq_u32(s0, s1)); -#endif -} -inline v_uint32x4 v_dotprod_expand(const v_uint8x16& a, const v_uint8x16& b, - const v_uint32x4& c) -{ -#if CV_NEON_DOT - return v_uint32x4(vdotq_u32(c.val, a.val, b.val)); -#else - return v_dotprod_expand(a, b) + c; -#endif -} - -inline v_int32x4 v_dotprod_expand(const v_int8x16& a, const v_int8x16& b) -{ -#if CV_NEON_DOT - return v_int32x4(vdotq_s32(vdupq_n_s32(0), a.val, b.val)); -#else - int16x8_t p0 = vmull_s8(vget_low_s8(a.val), vget_low_s8(b.val)); - int16x8_t p1 = vmull_s8(vget_high_s8(a.val), vget_high_s8(b.val)); - int16x8_t uzp1, uzp2; - _v128_unzip(p0, p1, uzp1, uzp2); - int16x8_t sum = vaddq_s16(uzp1, uzp2); - int16x4_t uzpl1, uzpl2; - _v128_unzip(vget_low_s16(sum), vget_high_s16(sum), uzpl1, uzpl2); - return v_int32x4(vaddl_s16(uzpl1, uzpl2)); -#endif -} -inline v_int32x4 v_dotprod_expand(const v_int8x16& a, const v_int8x16& b, - const v_int32x4& c) -{ -#if CV_NEON_DOT - return v_int32x4(vdotq_s32(c.val, a.val, b.val)); -#else - return v_dotprod_expand(a, b) + c; -#endif -} - -// 16 >> 64 -inline v_uint64x2 v_dotprod_expand(const v_uint16x8& a, const v_uint16x8& b) -{ - const uint16x8_t zero = vreinterpretq_u16_u32(vdupq_n_u32(0)); - const uint16x8_t mask = vreinterpretq_u16_u32(vdupq_n_u32(0x0000FFFF)); - - uint32x4_t even = vmulq_u32(vreinterpretq_u32_u16(vbslq_u16(mask, a.val, zero)), - vreinterpretq_u32_u16(vbslq_u16(mask, b.val, zero))); - uint32x4_t odd = vmulq_u32(vshrq_n_u32(vreinterpretq_u32_u16(a.val), 16), - vshrq_n_u32(vreinterpretq_u32_u16(b.val), 16)); - uint32x4_t uzp1, uzp2; - _v128_unzip(even, odd, uzp1, uzp2); - uint64x2_t s0 = vaddl_u32(vget_low_u32(uzp1), vget_high_u32(uzp1)); - uint64x2_t s1 = vaddl_u32(vget_low_u32(uzp2), vget_high_u32(uzp2)); - return v_uint64x2(vaddq_u64(s0, s1)); -} -inline v_uint64x2 v_dotprod_expand(const v_uint16x8& a, const v_uint16x8& b, const v_uint64x2& c) -{ return v_dotprod_expand(a, b) + c; } - -inline v_int64x2 v_dotprod_expand(const v_int16x8& a, const v_int16x8& b) -{ - int32x4_t p0 = vmull_s16(vget_low_s16(a.val), vget_low_s16(b.val)); - int32x4_t p1 = vmull_s16(vget_high_s16(a.val), vget_high_s16(b.val)); - - int32x4_t uzp1, uzp2; - _v128_unzip(p0, p1, uzp1, uzp2); - int32x4_t sum = vaddq_s32(uzp1, uzp2); - - int32x2_t uzpl1, uzpl2; - _v128_unzip(vget_low_s32(sum), vget_high_s32(sum), uzpl1, uzpl2); - return v_int64x2(vaddl_s32(uzpl1, uzpl2)); -} -inline v_int64x2 v_dotprod_expand(const v_int16x8& a, const v_int16x8& b, - const v_int64x2& c) -{ return v_dotprod_expand(a, b) + c; } - -// 32 >> 64f -#if CV_SIMD128_64F -inline v_float64x2 v_dotprod_expand(const v_int32x4& a, const v_int32x4& b) -{ return v_cvt_f64(v_dotprod(a, b)); } -inline v_float64x2 v_dotprod_expand(const v_int32x4& a, const v_int32x4& b, - const v_float64x2& c) -{ return v_dotprod_expand(a, b) + c; } -#endif - -//////// Fast Dot Product //////// - -// 16 >> 32 -inline v_int32x4 v_dotprod_fast(const v_int16x8& a, const v_int16x8& b) -{ - int16x4_t a0 = vget_low_s16(a.val); - int16x4_t a1 = vget_high_s16(a.val); - int16x4_t b0 = vget_low_s16(b.val); - int16x4_t b1 = vget_high_s16(b.val); - int32x4_t p = vmull_s16(a0, b0); - return v_int32x4(vmlal_s16(p, a1, b1)); -} -inline v_int32x4 v_dotprod_fast(const v_int16x8& a, const v_int16x8& b, const v_int32x4& c) -{ - int16x4_t a0 = vget_low_s16(a.val); - int16x4_t a1 = vget_high_s16(a.val); - int16x4_t b0 = vget_low_s16(b.val); - int16x4_t b1 = vget_high_s16(b.val); - int32x4_t p = vmlal_s16(c.val, a0, b0); - return v_int32x4(vmlal_s16(p, a1, b1)); -} - -// 32 >> 64 -inline v_int64x2 v_dotprod_fast(const v_int32x4& a, const v_int32x4& b) -{ - int32x2_t a0 = vget_low_s32(a.val); - int32x2_t a1 = vget_high_s32(a.val); - int32x2_t b0 = vget_low_s32(b.val); - int32x2_t b1 = vget_high_s32(b.val); - int64x2_t p = vmull_s32(a0, b0); - return v_int64x2(vmlal_s32(p, a1, b1)); -} -inline v_int64x2 v_dotprod_fast(const v_int32x4& a, const v_int32x4& b, const v_int64x2& c) -{ - int32x2_t a0 = vget_low_s32(a.val); - int32x2_t a1 = vget_high_s32(a.val); - int32x2_t b0 = vget_low_s32(b.val); - int32x2_t b1 = vget_high_s32(b.val); - int64x2_t p = vmlal_s32(c.val, a0, b0); - return v_int64x2(vmlal_s32(p, a1, b1)); -} - -// 8 >> 32 -inline v_uint32x4 v_dotprod_expand_fast(const v_uint8x16& a, const v_uint8x16& b) -{ -#if CV_NEON_DOT - return v_uint32x4(vdotq_u32(vdupq_n_u32(0), a.val, b.val)); -#else - uint16x8_t p0 = vmull_u8(vget_low_u8(a.val), vget_low_u8(b.val)); - uint16x8_t p1 = vmull_u8(vget_high_u8(a.val), vget_high_u8(b.val)); - uint32x4_t s0 = vaddl_u16(vget_low_u16(p0), vget_low_u16(p1)); - uint32x4_t s1 = vaddl_u16(vget_high_u16(p0), vget_high_u16(p1)); - return v_uint32x4(vaddq_u32(s0, s1)); -#endif -} -inline v_uint32x4 v_dotprod_expand_fast(const v_uint8x16& a, const v_uint8x16& b, const v_uint32x4& c) -{ -#if CV_NEON_DOT - return v_uint32x4(vdotq_u32(c.val, a.val, b.val)); -#else - return v_dotprod_expand_fast(a, b) + c; -#endif -} - -inline v_int32x4 v_dotprod_expand_fast(const v_int8x16& a, const v_int8x16& b) -{ -#if CV_NEON_DOT - return v_int32x4(vdotq_s32(vdupq_n_s32(0), a.val, b.val)); -#else - int16x8_t prod = vmull_s8(vget_low_s8(a.val), vget_low_s8(b.val)); - prod = vmlal_s8(prod, vget_high_s8(a.val), vget_high_s8(b.val)); - return v_int32x4(vaddl_s16(vget_low_s16(prod), vget_high_s16(prod))); -#endif -} -inline v_int32x4 v_dotprod_expand_fast(const v_int8x16& a, const v_int8x16& b, const v_int32x4& c) -{ -#if CV_NEON_DOT - return v_int32x4(vdotq_s32(c.val, a.val, b.val)); -#else - return v_dotprod_expand_fast(a, b) + c; -#endif -} - -// 16 >> 64 -inline v_uint64x2 v_dotprod_expand_fast(const v_uint16x8& a, const v_uint16x8& b) -{ - uint32x4_t p0 = vmull_u16(vget_low_u16(a.val), vget_low_u16(b.val)); - uint32x4_t p1 = vmull_u16(vget_high_u16(a.val), vget_high_u16(b.val)); - uint64x2_t s0 = vaddl_u32(vget_low_u32(p0), vget_high_u32(p0)); - uint64x2_t s1 = vaddl_u32(vget_low_u32(p1), vget_high_u32(p1)); - return v_uint64x2(vaddq_u64(s0, s1)); -} -inline v_uint64x2 v_dotprod_expand_fast(const v_uint16x8& a, const v_uint16x8& b, const v_uint64x2& c) -{ return v_dotprod_expand_fast(a, b) + c; } - -inline v_int64x2 v_dotprod_expand_fast(const v_int16x8& a, const v_int16x8& b) -{ - int32x4_t prod = vmull_s16(vget_low_s16(a.val), vget_low_s16(b.val)); - prod = vmlal_s16(prod, vget_high_s16(a.val), vget_high_s16(b.val)); - return v_int64x2(vaddl_s32(vget_low_s32(prod), vget_high_s32(prod))); -} -inline v_int64x2 v_dotprod_expand_fast(const v_int16x8& a, const v_int16x8& b, const v_int64x2& c) -{ return v_dotprod_expand_fast(a, b) + c; } - -// 32 >> 64f -#if CV_SIMD128_64F -inline v_float64x2 v_dotprod_expand_fast(const v_int32x4& a, const v_int32x4& b) -{ return v_cvt_f64(v_dotprod_fast(a, b)); } -inline v_float64x2 v_dotprod_expand_fast(const v_int32x4& a, const v_int32x4& b, const v_float64x2& c) -{ return v_dotprod_expand_fast(a, b) + c; } -#endif - - -#define OPENCV_HAL_IMPL_NEON_LOGIC_OP(_Tpvec, suffix) \ - OPENCV_HAL_IMPL_NEON_BIN_OP(&, _Tpvec, vandq_##suffix) \ - OPENCV_HAL_IMPL_NEON_BIN_OP(|, _Tpvec, vorrq_##suffix) \ - OPENCV_HAL_IMPL_NEON_BIN_OP(^, _Tpvec, veorq_##suffix) \ - inline _Tpvec operator ~ (const _Tpvec& a) \ - { \ - return _Tpvec(vreinterpretq_##suffix##_u8(vmvnq_u8(vreinterpretq_u8_##suffix(a.val)))); \ - } - -OPENCV_HAL_IMPL_NEON_LOGIC_OP(v_uint8x16, u8) -OPENCV_HAL_IMPL_NEON_LOGIC_OP(v_int8x16, s8) -OPENCV_HAL_IMPL_NEON_LOGIC_OP(v_uint16x8, u16) -OPENCV_HAL_IMPL_NEON_LOGIC_OP(v_int16x8, s16) -OPENCV_HAL_IMPL_NEON_LOGIC_OP(v_uint32x4, u32) -OPENCV_HAL_IMPL_NEON_LOGIC_OP(v_int32x4, s32) -OPENCV_HAL_IMPL_NEON_LOGIC_OP(v_uint64x2, u64) -OPENCV_HAL_IMPL_NEON_LOGIC_OP(v_int64x2, s64) - -#define OPENCV_HAL_IMPL_NEON_FLT_BIT_OP(bin_op, intrin) \ -inline v_float32x4 operator bin_op (const v_float32x4& a, const v_float32x4& b) \ -{ \ - return v_float32x4(vreinterpretq_f32_s32(intrin(vreinterpretq_s32_f32(a.val), vreinterpretq_s32_f32(b.val)))); \ -} \ -inline v_float32x4& operator bin_op##= (v_float32x4& a, const v_float32x4& b) \ -{ \ - a.val = vreinterpretq_f32_s32(intrin(vreinterpretq_s32_f32(a.val), vreinterpretq_s32_f32(b.val))); \ - return a; \ -} - -OPENCV_HAL_IMPL_NEON_FLT_BIT_OP(&, vandq_s32) -OPENCV_HAL_IMPL_NEON_FLT_BIT_OP(|, vorrq_s32) -OPENCV_HAL_IMPL_NEON_FLT_BIT_OP(^, veorq_s32) - -inline v_float32x4 operator ~ (const v_float32x4& a) -{ - return v_float32x4(vreinterpretq_f32_s32(vmvnq_s32(vreinterpretq_s32_f32(a.val)))); -} - -#if CV_SIMD128_64F -inline v_float32x4 v_sqrt(const v_float32x4& x) -{ - return v_float32x4(vsqrtq_f32(x.val)); -} - -inline v_float32x4 v_invsqrt(const v_float32x4& x) -{ - v_float32x4 one = v_setall_f32(1.0f); - return one / v_sqrt(x); -} -#else -inline v_float32x4 v_sqrt(const v_float32x4& x) -{ - float32x4_t x1 = vmaxq_f32(x.val, vdupq_n_f32(FLT_MIN)); - float32x4_t e = vrsqrteq_f32(x1); - e = vmulq_f32(vrsqrtsq_f32(vmulq_f32(x1, e), e), e); - e = vmulq_f32(vrsqrtsq_f32(vmulq_f32(x1, e), e), e); - return v_float32x4(vmulq_f32(x.val, e)); -} - -inline v_float32x4 v_invsqrt(const v_float32x4& x) -{ - float32x4_t e = vrsqrteq_f32(x.val); - e = vmulq_f32(vrsqrtsq_f32(vmulq_f32(x.val, e), e), e); - e = vmulq_f32(vrsqrtsq_f32(vmulq_f32(x.val, e), e), e); - return v_float32x4(e); -} -#endif - -#define OPENCV_HAL_IMPL_NEON_ABS(_Tpuvec, _Tpsvec, usuffix, ssuffix) \ -inline _Tpuvec v_abs(const _Tpsvec& a) { return v_reinterpret_as_##usuffix(_Tpsvec(vabsq_##ssuffix(a.val))); } - -OPENCV_HAL_IMPL_NEON_ABS(v_uint8x16, v_int8x16, u8, s8) -OPENCV_HAL_IMPL_NEON_ABS(v_uint16x8, v_int16x8, u16, s16) -OPENCV_HAL_IMPL_NEON_ABS(v_uint32x4, v_int32x4, u32, s32) - -inline v_float32x4 v_abs(v_float32x4 x) -{ return v_float32x4(vabsq_f32(x.val)); } - -#if CV_SIMD128_64F -#define OPENCV_HAL_IMPL_NEON_DBL_BIT_OP(bin_op, intrin) \ -inline v_float64x2 operator bin_op (const v_float64x2& a, const v_float64x2& b) \ -{ \ - return v_float64x2(vreinterpretq_f64_s64(intrin(vreinterpretq_s64_f64(a.val), vreinterpretq_s64_f64(b.val)))); \ -} \ -inline v_float64x2& operator bin_op##= (v_float64x2& a, const v_float64x2& b) \ -{ \ - a.val = vreinterpretq_f64_s64(intrin(vreinterpretq_s64_f64(a.val), vreinterpretq_s64_f64(b.val))); \ - return a; \ -} - -OPENCV_HAL_IMPL_NEON_DBL_BIT_OP(&, vandq_s64) -OPENCV_HAL_IMPL_NEON_DBL_BIT_OP(|, vorrq_s64) -OPENCV_HAL_IMPL_NEON_DBL_BIT_OP(^, veorq_s64) - -inline v_float64x2 operator ~ (const v_float64x2& a) -{ - return v_float64x2(vreinterpretq_f64_s32(vmvnq_s32(vreinterpretq_s32_f64(a.val)))); -} - -inline v_float64x2 v_sqrt(const v_float64x2& x) -{ - return v_float64x2(vsqrtq_f64(x.val)); -} - -inline v_float64x2 v_invsqrt(const v_float64x2& x) -{ - v_float64x2 one = v_setall_f64(1.0f); - return one / v_sqrt(x); -} - -inline v_float64x2 v_abs(v_float64x2 x) -{ return v_float64x2(vabsq_f64(x.val)); } -#endif - -// TODO: exp, log, sin, cos - -#define OPENCV_HAL_IMPL_NEON_BIN_FUNC(_Tpvec, func, intrin) \ -inline _Tpvec func(const _Tpvec& a, const _Tpvec& b) \ -{ \ - return _Tpvec(intrin(a.val, b.val)); \ -} - -OPENCV_HAL_IMPL_NEON_BIN_FUNC(v_uint8x16, v_min, vminq_u8) -OPENCV_HAL_IMPL_NEON_BIN_FUNC(v_uint8x16, v_max, vmaxq_u8) -OPENCV_HAL_IMPL_NEON_BIN_FUNC(v_int8x16, v_min, vminq_s8) -OPENCV_HAL_IMPL_NEON_BIN_FUNC(v_int8x16, v_max, vmaxq_s8) -OPENCV_HAL_IMPL_NEON_BIN_FUNC(v_uint16x8, v_min, vminq_u16) -OPENCV_HAL_IMPL_NEON_BIN_FUNC(v_uint16x8, v_max, vmaxq_u16) -OPENCV_HAL_IMPL_NEON_BIN_FUNC(v_int16x8, v_min, vminq_s16) -OPENCV_HAL_IMPL_NEON_BIN_FUNC(v_int16x8, v_max, vmaxq_s16) -OPENCV_HAL_IMPL_NEON_BIN_FUNC(v_uint32x4, v_min, vminq_u32) -OPENCV_HAL_IMPL_NEON_BIN_FUNC(v_uint32x4, v_max, vmaxq_u32) -OPENCV_HAL_IMPL_NEON_BIN_FUNC(v_int32x4, v_min, vminq_s32) -OPENCV_HAL_IMPL_NEON_BIN_FUNC(v_int32x4, v_max, vmaxq_s32) -OPENCV_HAL_IMPL_NEON_BIN_FUNC(v_float32x4, v_min, vminq_f32) -OPENCV_HAL_IMPL_NEON_BIN_FUNC(v_float32x4, v_max, vmaxq_f32) -#if CV_SIMD128_64F -OPENCV_HAL_IMPL_NEON_BIN_FUNC(v_float64x2, v_min, vminq_f64) -OPENCV_HAL_IMPL_NEON_BIN_FUNC(v_float64x2, v_max, vmaxq_f64) -#endif - -#if CV_SIMD128_64F -inline int64x2_t vmvnq_s64(int64x2_t a) -{ - int64x2_t vx = vreinterpretq_s64_u32(vdupq_n_u32(0xFFFFFFFF)); - return veorq_s64(a, vx); -} -inline uint64x2_t vmvnq_u64(uint64x2_t a) -{ - uint64x2_t vx = vreinterpretq_u64_u32(vdupq_n_u32(0xFFFFFFFF)); - return veorq_u64(a, vx); -} -#endif -#define OPENCV_HAL_IMPL_NEON_INT_CMP_OP(_Tpvec, cast, suffix, not_suffix) \ -inline _Tpvec operator == (const _Tpvec& a, const _Tpvec& b) \ -{ return _Tpvec(cast(vceqq_##suffix(a.val, b.val))); } \ -inline _Tpvec operator != (const _Tpvec& a, const _Tpvec& b) \ -{ return _Tpvec(cast(vmvnq_##not_suffix(vceqq_##suffix(a.val, b.val)))); } \ -inline _Tpvec operator < (const _Tpvec& a, const _Tpvec& b) \ -{ return _Tpvec(cast(vcltq_##suffix(a.val, b.val))); } \ -inline _Tpvec operator > (const _Tpvec& a, const _Tpvec& b) \ -{ return _Tpvec(cast(vcgtq_##suffix(a.val, b.val))); } \ -inline _Tpvec operator <= (const _Tpvec& a, const _Tpvec& b) \ -{ return _Tpvec(cast(vcleq_##suffix(a.val, b.val))); } \ -inline _Tpvec operator >= (const _Tpvec& a, const _Tpvec& b) \ -{ return _Tpvec(cast(vcgeq_##suffix(a.val, b.val))); } - -OPENCV_HAL_IMPL_NEON_INT_CMP_OP(v_uint8x16, OPENCV_HAL_NOP, u8, u8) -OPENCV_HAL_IMPL_NEON_INT_CMP_OP(v_int8x16, vreinterpretq_s8_u8, s8, u8) -OPENCV_HAL_IMPL_NEON_INT_CMP_OP(v_uint16x8, OPENCV_HAL_NOP, u16, u16) -OPENCV_HAL_IMPL_NEON_INT_CMP_OP(v_int16x8, vreinterpretq_s16_u16, s16, u16) -OPENCV_HAL_IMPL_NEON_INT_CMP_OP(v_uint32x4, OPENCV_HAL_NOP, u32, u32) -OPENCV_HAL_IMPL_NEON_INT_CMP_OP(v_int32x4, vreinterpretq_s32_u32, s32, u32) -OPENCV_HAL_IMPL_NEON_INT_CMP_OP(v_float32x4, vreinterpretq_f32_u32, f32, u32) -#if CV_SIMD128_64F -OPENCV_HAL_IMPL_NEON_INT_CMP_OP(v_uint64x2, OPENCV_HAL_NOP, u64, u64) -OPENCV_HAL_IMPL_NEON_INT_CMP_OP(v_int64x2, vreinterpretq_s64_u64, s64, u64) -OPENCV_HAL_IMPL_NEON_INT_CMP_OP(v_float64x2, vreinterpretq_f64_u64, f64, u64) -#endif - -inline v_float32x4 v_not_nan(const v_float32x4& a) -{ return v_float32x4(vreinterpretq_f32_u32(vceqq_f32(a.val, a.val))); } -#if CV_SIMD128_64F -inline v_float64x2 v_not_nan(const v_float64x2& a) -{ return v_float64x2(vreinterpretq_f64_u64(vceqq_f64(a.val, a.val))); } -#endif - -OPENCV_HAL_IMPL_NEON_BIN_FUNC(v_uint8x16, v_add_wrap, vaddq_u8) -OPENCV_HAL_IMPL_NEON_BIN_FUNC(v_int8x16, v_add_wrap, vaddq_s8) -OPENCV_HAL_IMPL_NEON_BIN_FUNC(v_uint16x8, v_add_wrap, vaddq_u16) -OPENCV_HAL_IMPL_NEON_BIN_FUNC(v_int16x8, v_add_wrap, vaddq_s16) -OPENCV_HAL_IMPL_NEON_BIN_FUNC(v_uint8x16, v_sub_wrap, vsubq_u8) -OPENCV_HAL_IMPL_NEON_BIN_FUNC(v_int8x16, v_sub_wrap, vsubq_s8) -OPENCV_HAL_IMPL_NEON_BIN_FUNC(v_uint16x8, v_sub_wrap, vsubq_u16) -OPENCV_HAL_IMPL_NEON_BIN_FUNC(v_int16x8, v_sub_wrap, vsubq_s16) -OPENCV_HAL_IMPL_NEON_BIN_FUNC(v_uint8x16, v_mul_wrap, vmulq_u8) -OPENCV_HAL_IMPL_NEON_BIN_FUNC(v_int8x16, v_mul_wrap, vmulq_s8) -OPENCV_HAL_IMPL_NEON_BIN_FUNC(v_uint16x8, v_mul_wrap, vmulq_u16) -OPENCV_HAL_IMPL_NEON_BIN_FUNC(v_int16x8, v_mul_wrap, vmulq_s16) - -OPENCV_HAL_IMPL_NEON_BIN_FUNC(v_uint8x16, v_absdiff, vabdq_u8) -OPENCV_HAL_IMPL_NEON_BIN_FUNC(v_uint16x8, v_absdiff, vabdq_u16) -OPENCV_HAL_IMPL_NEON_BIN_FUNC(v_uint32x4, v_absdiff, vabdq_u32) -OPENCV_HAL_IMPL_NEON_BIN_FUNC(v_float32x4, v_absdiff, vabdq_f32) -#if CV_SIMD128_64F -OPENCV_HAL_IMPL_NEON_BIN_FUNC(v_float64x2, v_absdiff, vabdq_f64) -#endif - -/** Saturating absolute difference **/ -inline v_int8x16 v_absdiffs(const v_int8x16& a, const v_int8x16& b) -{ return v_int8x16(vqabsq_s8(vqsubq_s8(a.val, b.val))); } -inline v_int16x8 v_absdiffs(const v_int16x8& a, const v_int16x8& b) -{ return v_int16x8(vqabsq_s16(vqsubq_s16(a.val, b.val))); } - -#define OPENCV_HAL_IMPL_NEON_BIN_FUNC2(_Tpvec, _Tpvec2, cast, func, intrin) \ -inline _Tpvec2 func(const _Tpvec& a, const _Tpvec& b) \ -{ \ - return _Tpvec2(cast(intrin(a.val, b.val))); \ -} - -OPENCV_HAL_IMPL_NEON_BIN_FUNC2(v_int8x16, v_uint8x16, vreinterpretq_u8_s8, v_absdiff, vabdq_s8) -OPENCV_HAL_IMPL_NEON_BIN_FUNC2(v_int16x8, v_uint16x8, vreinterpretq_u16_s16, v_absdiff, vabdq_s16) -OPENCV_HAL_IMPL_NEON_BIN_FUNC2(v_int32x4, v_uint32x4, vreinterpretq_u32_s32, v_absdiff, vabdq_s32) - -inline v_float32x4 v_magnitude(const v_float32x4& a, const v_float32x4& b) -{ - v_float32x4 x(vmlaq_f32(vmulq_f32(a.val, a.val), b.val, b.val)); - return v_sqrt(x); -} - -inline v_float32x4 v_sqr_magnitude(const v_float32x4& a, const v_float32x4& b) -{ - return v_float32x4(vmlaq_f32(vmulq_f32(a.val, a.val), b.val, b.val)); -} - -inline v_float32x4 v_fma(const v_float32x4& a, const v_float32x4& b, const v_float32x4& c) -{ -#if CV_SIMD128_64F - // ARMv8, which adds support for 64-bit floating-point (so CV_SIMD128_64F is defined), - // also adds FMA support both for single- and double-precision floating-point vectors - return v_float32x4(vfmaq_f32(c.val, a.val, b.val)); -#else - return v_float32x4(vmlaq_f32(c.val, a.val, b.val)); -#endif -} - -inline v_int32x4 v_fma(const v_int32x4& a, const v_int32x4& b, const v_int32x4& c) -{ - return v_int32x4(vmlaq_s32(c.val, a.val, b.val)); -} - -inline v_float32x4 v_muladd(const v_float32x4& a, const v_float32x4& b, const v_float32x4& c) -{ - return v_fma(a, b, c); -} - -inline v_int32x4 v_muladd(const v_int32x4& a, const v_int32x4& b, const v_int32x4& c) -{ - return v_fma(a, b, c); -} - -#if CV_SIMD128_64F -inline v_float64x2 v_magnitude(const v_float64x2& a, const v_float64x2& b) -{ - v_float64x2 x(vaddq_f64(vmulq_f64(a.val, a.val), vmulq_f64(b.val, b.val))); - return v_sqrt(x); -} - -inline v_float64x2 v_sqr_magnitude(const v_float64x2& a, const v_float64x2& b) -{ - return v_float64x2(vaddq_f64(vmulq_f64(a.val, a.val), vmulq_f64(b.val, b.val))); -} - -inline v_float64x2 v_fma(const v_float64x2& a, const v_float64x2& b, const v_float64x2& c) -{ - return v_float64x2(vfmaq_f64(c.val, a.val, b.val)); -} - -inline v_float64x2 v_muladd(const v_float64x2& a, const v_float64x2& b, const v_float64x2& c) -{ - return v_fma(a, b, c); -} -#endif - -// trade efficiency for convenience -#define OPENCV_HAL_IMPL_NEON_SHIFT_OP(_Tpvec, suffix, _Tps, ssuffix) \ -inline _Tpvec operator << (const _Tpvec& a, int n) \ -{ return _Tpvec(vshlq_##suffix(a.val, vdupq_n_##ssuffix((_Tps)n))); } \ -inline _Tpvec operator >> (const _Tpvec& a, int n) \ -{ return _Tpvec(vshlq_##suffix(a.val, vdupq_n_##ssuffix((_Tps)-n))); } \ -template inline _Tpvec v_shl(const _Tpvec& a) \ -{ return _Tpvec(vshlq_n_##suffix(a.val, n)); } \ -template inline _Tpvec v_shr(const _Tpvec& a) \ -{ return _Tpvec(vshrq_n_##suffix(a.val, n)); } \ -template inline _Tpvec v_rshr(const _Tpvec& a) \ -{ return _Tpvec(vrshrq_n_##suffix(a.val, n)); } - -OPENCV_HAL_IMPL_NEON_SHIFT_OP(v_uint8x16, u8, schar, s8) -OPENCV_HAL_IMPL_NEON_SHIFT_OP(v_int8x16, s8, schar, s8) -OPENCV_HAL_IMPL_NEON_SHIFT_OP(v_uint16x8, u16, short, s16) -OPENCV_HAL_IMPL_NEON_SHIFT_OP(v_int16x8, s16, short, s16) -OPENCV_HAL_IMPL_NEON_SHIFT_OP(v_uint32x4, u32, int, s32) -OPENCV_HAL_IMPL_NEON_SHIFT_OP(v_int32x4, s32, int, s32) -OPENCV_HAL_IMPL_NEON_SHIFT_OP(v_uint64x2, u64, int64, s64) -OPENCV_HAL_IMPL_NEON_SHIFT_OP(v_int64x2, s64, int64, s64) - -#define OPENCV_HAL_IMPL_NEON_ROTATE_OP(_Tpvec, suffix) \ -template inline _Tpvec v_rotate_right(const _Tpvec& a) \ -{ return _Tpvec(vextq_##suffix(a.val, vdupq_n_##suffix(0), n)); } \ -template inline _Tpvec v_rotate_left(const _Tpvec& a) \ -{ return _Tpvec(vextq_##suffix(vdupq_n_##suffix(0), a.val, _Tpvec::nlanes - n)); } \ -template<> inline _Tpvec v_rotate_left<0>(const _Tpvec& a) \ -{ return a; } \ -template inline _Tpvec v_rotate_right(const _Tpvec& a, const _Tpvec& b) \ -{ return _Tpvec(vextq_##suffix(a.val, b.val, n)); } \ -template inline _Tpvec v_rotate_left(const _Tpvec& a, const _Tpvec& b) \ -{ return _Tpvec(vextq_##suffix(b.val, a.val, _Tpvec::nlanes - n)); } \ -template<> inline _Tpvec v_rotate_left<0>(const _Tpvec& a, const _Tpvec& b) \ -{ CV_UNUSED(b); return a; } - -OPENCV_HAL_IMPL_NEON_ROTATE_OP(v_uint8x16, u8) -OPENCV_HAL_IMPL_NEON_ROTATE_OP(v_int8x16, s8) -OPENCV_HAL_IMPL_NEON_ROTATE_OP(v_uint16x8, u16) -OPENCV_HAL_IMPL_NEON_ROTATE_OP(v_int16x8, s16) -OPENCV_HAL_IMPL_NEON_ROTATE_OP(v_uint32x4, u32) -OPENCV_HAL_IMPL_NEON_ROTATE_OP(v_int32x4, s32) -OPENCV_HAL_IMPL_NEON_ROTATE_OP(v_float32x4, f32) -OPENCV_HAL_IMPL_NEON_ROTATE_OP(v_uint64x2, u64) -OPENCV_HAL_IMPL_NEON_ROTATE_OP(v_int64x2, s64) -#if CV_SIMD128_64F -OPENCV_HAL_IMPL_NEON_ROTATE_OP(v_float64x2, f64) -#endif - -#if defined(__clang__) && defined(__aarch64__) -// avoid LD2 instruction. details: https://github.com/opencv/opencv/issues/14863 -#define OPENCV_HAL_IMPL_NEON_LOAD_LOW_OP(_Tpvec, _Tp, suffix) \ -inline _Tpvec v_load_low(const _Tp* ptr) \ -{ \ -typedef uint64 CV_DECL_ALIGNED(1) unaligned_uint64; \ -uint64 v = *(unaligned_uint64*)ptr; \ -return _Tpvec(v_reinterpret_as_##suffix(v_uint64x2(v, (uint64)123456))); \ -} -#else -#define OPENCV_HAL_IMPL_NEON_LOAD_LOW_OP(_Tpvec, _Tp, suffix) \ -inline _Tpvec v_load_low(const _Tp* ptr) \ -{ return _Tpvec(vcombine_##suffix(vld1_##suffix(ptr), vdup_n_##suffix((_Tp)0))); } -#endif - -#define OPENCV_HAL_IMPL_NEON_LOADSTORE_OP(_Tpvec, _Tp, suffix) \ -inline _Tpvec v_load(const _Tp* ptr) \ -{ return _Tpvec(vld1q_##suffix(ptr)); } \ -inline _Tpvec v_load_aligned(const _Tp* ptr) \ -{ return _Tpvec(vld1q_##suffix(ptr)); } \ -OPENCV_HAL_IMPL_NEON_LOAD_LOW_OP(_Tpvec, _Tp, suffix) \ -inline _Tpvec v_load_halves(const _Tp* ptr0, const _Tp* ptr1) \ -{ return _Tpvec(vcombine_##suffix(vld1_##suffix(ptr0), vld1_##suffix(ptr1))); } \ -inline void v_store(_Tp* ptr, const _Tpvec& a) \ -{ vst1q_##suffix(ptr, a.val); } \ -inline void v_store_aligned(_Tp* ptr, const _Tpvec& a) \ -{ vst1q_##suffix(ptr, a.val); } \ -inline void v_store_aligned_nocache(_Tp* ptr, const _Tpvec& a) \ -{ vst1q_##suffix(ptr, a.val); } \ -inline void v_store(_Tp* ptr, const _Tpvec& a, hal::StoreMode /*mode*/) \ -{ vst1q_##suffix(ptr, a.val); } \ -inline void v_store_low(_Tp* ptr, const _Tpvec& a) \ -{ vst1_##suffix(ptr, vget_low_##suffix(a.val)); } \ -inline void v_store_high(_Tp* ptr, const _Tpvec& a) \ -{ vst1_##suffix(ptr, vget_high_##suffix(a.val)); } - -OPENCV_HAL_IMPL_NEON_LOADSTORE_OP(v_uint8x16, uchar, u8) -OPENCV_HAL_IMPL_NEON_LOADSTORE_OP(v_int8x16, schar, s8) -OPENCV_HAL_IMPL_NEON_LOADSTORE_OP(v_uint16x8, ushort, u16) -OPENCV_HAL_IMPL_NEON_LOADSTORE_OP(v_int16x8, short, s16) -OPENCV_HAL_IMPL_NEON_LOADSTORE_OP(v_uint32x4, unsigned, u32) -OPENCV_HAL_IMPL_NEON_LOADSTORE_OP(v_int32x4, int, s32) -OPENCV_HAL_IMPL_NEON_LOADSTORE_OP(v_uint64x2, uint64, u64) -OPENCV_HAL_IMPL_NEON_LOADSTORE_OP(v_int64x2, int64, s64) -OPENCV_HAL_IMPL_NEON_LOADSTORE_OP(v_float32x4, float, f32) - -#if CV_SIMD128_64F -OPENCV_HAL_IMPL_NEON_LOADSTORE_OP(v_float64x2, double, f64) -#endif - -inline unsigned v_reduce_sum(const v_uint8x16& a) -{ - uint32x4_t t0 = vpaddlq_u16(vpaddlq_u8(a.val)); - uint32x2_t t1 = vpadd_u32(vget_low_u32(t0), vget_high_u32(t0)); - return vget_lane_u32(vpadd_u32(t1, t1), 0); -} -inline int v_reduce_sum(const v_int8x16& a) -{ - int32x4_t t0 = vpaddlq_s16(vpaddlq_s8(a.val)); - int32x2_t t1 = vpadd_s32(vget_low_s32(t0), vget_high_s32(t0)); - return vget_lane_s32(vpadd_s32(t1, t1), 0); -} -inline unsigned v_reduce_sum(const v_uint16x8& a) -{ - uint32x4_t t0 = vpaddlq_u16(a.val); - uint32x2_t t1 = vpadd_u32(vget_low_u32(t0), vget_high_u32(t0)); - return vget_lane_u32(vpadd_u32(t1, t1), 0); -} -inline int v_reduce_sum(const v_int16x8& a) -{ - int32x4_t t0 = vpaddlq_s16(a.val); - int32x2_t t1 = vpadd_s32(vget_low_s32(t0), vget_high_s32(t0)); - return vget_lane_s32(vpadd_s32(t1, t1), 0); -} - -#define OPENCV_HAL_IMPL_NEON_REDUCE_OP_8(_Tpvec, _Tpnvec, scalartype, func, vectorfunc, suffix) \ -inline scalartype v_reduce_##func(const _Tpvec& a) \ -{ \ - _Tpnvec##_t a0 = vp##vectorfunc##_##suffix(vget_low_##suffix(a.val), vget_high_##suffix(a.val)); \ - a0 = vp##vectorfunc##_##suffix(a0, a0); \ - return (scalartype)vget_lane_##suffix(vp##vectorfunc##_##suffix(a0, a0),0); \ -} - -OPENCV_HAL_IMPL_NEON_REDUCE_OP_8(v_uint16x8, uint16x4, unsigned int, max, max, u16) -OPENCV_HAL_IMPL_NEON_REDUCE_OP_8(v_uint16x8, uint16x4, unsigned int, min, min, u16) -OPENCV_HAL_IMPL_NEON_REDUCE_OP_8(v_int16x8, int16x4, int, max, max, s16) -OPENCV_HAL_IMPL_NEON_REDUCE_OP_8(v_int16x8, int16x4, int, min, min, s16) - -#define OPENCV_HAL_IMPL_NEON_REDUCE_OP_4(_Tpvec, _Tpnvec, scalartype, func, vectorfunc, suffix) \ -inline scalartype v_reduce_##func(const _Tpvec& a) \ -{ \ - _Tpnvec##_t a0 = vp##vectorfunc##_##suffix(vget_low_##suffix(a.val), vget_high_##suffix(a.val)); \ - return (scalartype)vget_lane_##suffix(vp##vectorfunc##_##suffix(a0, vget_high_##suffix(a.val)),0); \ -} - -OPENCV_HAL_IMPL_NEON_REDUCE_OP_4(v_uint32x4, uint32x2, unsigned, sum, add, u32) -OPENCV_HAL_IMPL_NEON_REDUCE_OP_4(v_uint32x4, uint32x2, unsigned, max, max, u32) -OPENCV_HAL_IMPL_NEON_REDUCE_OP_4(v_uint32x4, uint32x2, unsigned, min, min, u32) -OPENCV_HAL_IMPL_NEON_REDUCE_OP_4(v_int32x4, int32x2, int, sum, add, s32) -OPENCV_HAL_IMPL_NEON_REDUCE_OP_4(v_int32x4, int32x2, int, max, max, s32) -OPENCV_HAL_IMPL_NEON_REDUCE_OP_4(v_int32x4, int32x2, int, min, min, s32) -OPENCV_HAL_IMPL_NEON_REDUCE_OP_4(v_float32x4, float32x2, float, sum, add, f32) -OPENCV_HAL_IMPL_NEON_REDUCE_OP_4(v_float32x4, float32x2, float, max, max, f32) -OPENCV_HAL_IMPL_NEON_REDUCE_OP_4(v_float32x4, float32x2, float, min, min, f32) - -inline uint64 v_reduce_sum(const v_uint64x2& a) -{ return vget_lane_u64(vadd_u64(vget_low_u64(a.val), vget_high_u64(a.val)),0); } -inline int64 v_reduce_sum(const v_int64x2& a) -{ return vget_lane_s64(vadd_s64(vget_low_s64(a.val), vget_high_s64(a.val)),0); } -#if CV_SIMD128_64F -inline double v_reduce_sum(const v_float64x2& a) -{ - return vgetq_lane_f64(a.val, 0) + vgetq_lane_f64(a.val, 1); -} -#endif - -inline v_float32x4 v_reduce_sum4(const v_float32x4& a, const v_float32x4& b, - const v_float32x4& c, const v_float32x4& d) -{ - float32x4x2_t ab = vtrnq_f32(a.val, b.val); - float32x4x2_t cd = vtrnq_f32(c.val, d.val); - - float32x4_t u0 = vaddq_f32(ab.val[0], ab.val[1]); // a0+a1 b0+b1 a2+a3 b2+b3 - float32x4_t u1 = vaddq_f32(cd.val[0], cd.val[1]); // c0+c1 d0+d1 c2+c3 d2+d3 - - float32x4_t v0 = vcombine_f32(vget_low_f32(u0), vget_low_f32(u1)); - float32x4_t v1 = vcombine_f32(vget_high_f32(u0), vget_high_f32(u1)); - - return v_float32x4(vaddq_f32(v0, v1)); -} - -inline unsigned v_reduce_sad(const v_uint8x16& a, const v_uint8x16& b) -{ - uint32x4_t t0 = vpaddlq_u16(vpaddlq_u8(vabdq_u8(a.val, b.val))); - uint32x2_t t1 = vpadd_u32(vget_low_u32(t0), vget_high_u32(t0)); - return vget_lane_u32(vpadd_u32(t1, t1), 0); -} -inline unsigned v_reduce_sad(const v_int8x16& a, const v_int8x16& b) -{ - uint32x4_t t0 = vpaddlq_u16(vpaddlq_u8(vreinterpretq_u8_s8(vabdq_s8(a.val, b.val)))); - uint32x2_t t1 = vpadd_u32(vget_low_u32(t0), vget_high_u32(t0)); - return vget_lane_u32(vpadd_u32(t1, t1), 0); -} -inline unsigned v_reduce_sad(const v_uint16x8& a, const v_uint16x8& b) -{ - uint32x4_t t0 = vpaddlq_u16(vabdq_u16(a.val, b.val)); - uint32x2_t t1 = vpadd_u32(vget_low_u32(t0), vget_high_u32(t0)); - return vget_lane_u32(vpadd_u32(t1, t1), 0); -} -inline unsigned v_reduce_sad(const v_int16x8& a, const v_int16x8& b) -{ - uint32x4_t t0 = vpaddlq_u16(vreinterpretq_u16_s16(vabdq_s16(a.val, b.val))); - uint32x2_t t1 = vpadd_u32(vget_low_u32(t0), vget_high_u32(t0)); - return vget_lane_u32(vpadd_u32(t1, t1), 0); -} -inline unsigned v_reduce_sad(const v_uint32x4& a, const v_uint32x4& b) -{ - uint32x4_t t0 = vabdq_u32(a.val, b.val); - uint32x2_t t1 = vpadd_u32(vget_low_u32(t0), vget_high_u32(t0)); - return vget_lane_u32(vpadd_u32(t1, t1), 0); -} -inline unsigned v_reduce_sad(const v_int32x4& a, const v_int32x4& b) -{ - uint32x4_t t0 = vreinterpretq_u32_s32(vabdq_s32(a.val, b.val)); - uint32x2_t t1 = vpadd_u32(vget_low_u32(t0), vget_high_u32(t0)); - return vget_lane_u32(vpadd_u32(t1, t1), 0); -} -inline float v_reduce_sad(const v_float32x4& a, const v_float32x4& b) -{ - float32x4_t t0 = vabdq_f32(a.val, b.val); - float32x2_t t1 = vpadd_f32(vget_low_f32(t0), vget_high_f32(t0)); - return vget_lane_f32(vpadd_f32(t1, t1), 0); -} - -inline v_uint8x16 v_popcount(const v_uint8x16& a) -{ return v_uint8x16(vcntq_u8(a.val)); } -inline v_uint8x16 v_popcount(const v_int8x16& a) -{ return v_uint8x16(vcntq_u8(vreinterpretq_u8_s8(a.val))); } -inline v_uint16x8 v_popcount(const v_uint16x8& a) -{ return v_uint16x8(vpaddlq_u8(vcntq_u8(vreinterpretq_u8_u16(a.val)))); } -inline v_uint16x8 v_popcount(const v_int16x8& a) -{ return v_uint16x8(vpaddlq_u8(vcntq_u8(vreinterpretq_u8_s16(a.val)))); } -inline v_uint32x4 v_popcount(const v_uint32x4& a) -{ return v_uint32x4(vpaddlq_u16(vpaddlq_u8(vcntq_u8(vreinterpretq_u8_u32(a.val))))); } -inline v_uint32x4 v_popcount(const v_int32x4& a) -{ return v_uint32x4(vpaddlq_u16(vpaddlq_u8(vcntq_u8(vreinterpretq_u8_s32(a.val))))); } -inline v_uint64x2 v_popcount(const v_uint64x2& a) -{ return v_uint64x2(vpaddlq_u32(vpaddlq_u16(vpaddlq_u8(vcntq_u8(vreinterpretq_u8_u64(a.val)))))); } -inline v_uint64x2 v_popcount(const v_int64x2& a) -{ return v_uint64x2(vpaddlq_u32(vpaddlq_u16(vpaddlq_u8(vcntq_u8(vreinterpretq_u8_s64(a.val)))))); } - -inline int v_signmask(const v_uint8x16& a) -{ - int8x8_t m0 = vcreate_s8(CV_BIG_UINT(0x0706050403020100)); - uint8x16_t v0 = vshlq_u8(vshrq_n_u8(a.val, 7), vcombine_s8(m0, m0)); - uint64x2_t v1 = vpaddlq_u32(vpaddlq_u16(vpaddlq_u8(v0))); - return (int)vgetq_lane_u64(v1, 0) + ((int)vgetq_lane_u64(v1, 1) << 8); -} -inline int v_signmask(const v_int8x16& a) -{ return v_signmask(v_reinterpret_as_u8(a)); } - -inline int v_signmask(const v_uint16x8& a) -{ - int16x4_t m0 = vcreate_s16(CV_BIG_UINT(0x0003000200010000)); - uint16x8_t v0 = vshlq_u16(vshrq_n_u16(a.val, 15), vcombine_s16(m0, m0)); - uint64x2_t v1 = vpaddlq_u32(vpaddlq_u16(v0)); - return (int)vgetq_lane_u64(v1, 0) + ((int)vgetq_lane_u64(v1, 1) << 4); -} -inline int v_signmask(const v_int16x8& a) -{ return v_signmask(v_reinterpret_as_u16(a)); } - -inline int v_signmask(const v_uint32x4& a) -{ - int32x2_t m0 = vcreate_s32(CV_BIG_UINT(0x0000000100000000)); - uint32x4_t v0 = vshlq_u32(vshrq_n_u32(a.val, 31), vcombine_s32(m0, m0)); - uint64x2_t v1 = vpaddlq_u32(v0); - return (int)vgetq_lane_u64(v1, 0) + ((int)vgetq_lane_u64(v1, 1) << 2); -} -inline int v_signmask(const v_int32x4& a) -{ return v_signmask(v_reinterpret_as_u32(a)); } -inline int v_signmask(const v_float32x4& a) -{ return v_signmask(v_reinterpret_as_u32(a)); } -inline int v_signmask(const v_uint64x2& a) -{ - int64x1_t m0 = vdup_n_s64(0); - uint64x2_t v0 = vshlq_u64(vshrq_n_u64(a.val, 63), vcombine_s64(m0, m0)); - return (int)vgetq_lane_u64(v0, 0) + ((int)vgetq_lane_u64(v0, 1) << 1); -} -inline int v_signmask(const v_int64x2& a) -{ return v_signmask(v_reinterpret_as_u64(a)); } -#if CV_SIMD128_64F -inline int v_signmask(const v_float64x2& a) -{ return v_signmask(v_reinterpret_as_u64(a)); } -#endif - -inline int v_scan_forward(const v_int8x16& a) { return trailingZeros32(v_signmask(a)); } -inline int v_scan_forward(const v_uint8x16& a) { return trailingZeros32(v_signmask(a)); } -inline int v_scan_forward(const v_int16x8& a) { return trailingZeros32(v_signmask(a)); } -inline int v_scan_forward(const v_uint16x8& a) { return trailingZeros32(v_signmask(a)); } -inline int v_scan_forward(const v_int32x4& a) { return trailingZeros32(v_signmask(a)); } -inline int v_scan_forward(const v_uint32x4& a) { return trailingZeros32(v_signmask(a)); } -inline int v_scan_forward(const v_float32x4& a) { return trailingZeros32(v_signmask(a)); } -inline int v_scan_forward(const v_int64x2& a) { return trailingZeros32(v_signmask(a)); } -inline int v_scan_forward(const v_uint64x2& a) { return trailingZeros32(v_signmask(a)); } -#if CV_SIMD128_64F -inline int v_scan_forward(const v_float64x2& a) { return trailingZeros32(v_signmask(a)); } -#endif - -#define OPENCV_HAL_IMPL_NEON_CHECK_ALLANY(_Tpvec, suffix, shift) \ -inline bool v_check_all(const v_##_Tpvec& a) \ -{ \ - _Tpvec##_t v0 = vshrq_n_##suffix(vmvnq_##suffix(a.val), shift); \ - uint64x2_t v1 = vreinterpretq_u64_##suffix(v0); \ - return (vgetq_lane_u64(v1, 0) | vgetq_lane_u64(v1, 1)) == 0; \ -} \ -inline bool v_check_any(const v_##_Tpvec& a) \ -{ \ - _Tpvec##_t v0 = vshrq_n_##suffix(a.val, shift); \ - uint64x2_t v1 = vreinterpretq_u64_##suffix(v0); \ - return (vgetq_lane_u64(v1, 0) | vgetq_lane_u64(v1, 1)) != 0; \ -} - -OPENCV_HAL_IMPL_NEON_CHECK_ALLANY(uint8x16, u8, 7) -OPENCV_HAL_IMPL_NEON_CHECK_ALLANY(uint16x8, u16, 15) -OPENCV_HAL_IMPL_NEON_CHECK_ALLANY(uint32x4, u32, 31) - -inline bool v_check_all(const v_uint64x2& a) -{ - uint64x2_t v0 = vshrq_n_u64(a.val, 63); - return (vgetq_lane_u64(v0, 0) & vgetq_lane_u64(v0, 1)) == 1; -} -inline bool v_check_any(const v_uint64x2& a) -{ - uint64x2_t v0 = vshrq_n_u64(a.val, 63); - return (vgetq_lane_u64(v0, 0) | vgetq_lane_u64(v0, 1)) != 0; -} - -inline bool v_check_all(const v_int8x16& a) -{ return v_check_all(v_reinterpret_as_u8(a)); } -inline bool v_check_all(const v_int16x8& a) -{ return v_check_all(v_reinterpret_as_u16(a)); } -inline bool v_check_all(const v_int32x4& a) -{ return v_check_all(v_reinterpret_as_u32(a)); } -inline bool v_check_all(const v_float32x4& a) -{ return v_check_all(v_reinterpret_as_u32(a)); } - -inline bool v_check_any(const v_int8x16& a) -{ return v_check_any(v_reinterpret_as_u8(a)); } -inline bool v_check_any(const v_int16x8& a) -{ return v_check_any(v_reinterpret_as_u16(a)); } -inline bool v_check_any(const v_int32x4& a) -{ return v_check_any(v_reinterpret_as_u32(a)); } -inline bool v_check_any(const v_float32x4& a) -{ return v_check_any(v_reinterpret_as_u32(a)); } - -inline bool v_check_all(const v_int64x2& a) -{ return v_check_all(v_reinterpret_as_u64(a)); } -inline bool v_check_any(const v_int64x2& a) -{ return v_check_any(v_reinterpret_as_u64(a)); } -#if CV_SIMD128_64F -inline bool v_check_all(const v_float64x2& a) -{ return v_check_all(v_reinterpret_as_u64(a)); } -inline bool v_check_any(const v_float64x2& a) -{ return v_check_any(v_reinterpret_as_u64(a)); } -#endif - -#define OPENCV_HAL_IMPL_NEON_SELECT(_Tpvec, suffix, usuffix) \ -inline _Tpvec v_select(const _Tpvec& mask, const _Tpvec& a, const _Tpvec& b) \ -{ \ - return _Tpvec(vbslq_##suffix(vreinterpretq_##usuffix##_##suffix(mask.val), a.val, b.val)); \ -} - -OPENCV_HAL_IMPL_NEON_SELECT(v_uint8x16, u8, u8) -OPENCV_HAL_IMPL_NEON_SELECT(v_int8x16, s8, u8) -OPENCV_HAL_IMPL_NEON_SELECT(v_uint16x8, u16, u16) -OPENCV_HAL_IMPL_NEON_SELECT(v_int16x8, s16, u16) -OPENCV_HAL_IMPL_NEON_SELECT(v_uint32x4, u32, u32) -OPENCV_HAL_IMPL_NEON_SELECT(v_int32x4, s32, u32) -OPENCV_HAL_IMPL_NEON_SELECT(v_float32x4, f32, u32) -#if CV_SIMD128_64F -OPENCV_HAL_IMPL_NEON_SELECT(v_float64x2, f64, u64) -#endif - -#define OPENCV_HAL_IMPL_NEON_EXPAND(_Tpvec, _Tpwvec, _Tp, suffix) \ -inline void v_expand(const _Tpvec& a, _Tpwvec& b0, _Tpwvec& b1) \ -{ \ - b0.val = vmovl_##suffix(vget_low_##suffix(a.val)); \ - b1.val = vmovl_##suffix(vget_high_##suffix(a.val)); \ -} \ -inline _Tpwvec v_expand_low(const _Tpvec& a) \ -{ \ - return _Tpwvec(vmovl_##suffix(vget_low_##suffix(a.val))); \ -} \ -inline _Tpwvec v_expand_high(const _Tpvec& a) \ -{ \ - return _Tpwvec(vmovl_##suffix(vget_high_##suffix(a.val))); \ -} \ -inline _Tpwvec v_load_expand(const _Tp* ptr) \ -{ \ - return _Tpwvec(vmovl_##suffix(vld1_##suffix(ptr))); \ -} - -OPENCV_HAL_IMPL_NEON_EXPAND(v_uint8x16, v_uint16x8, uchar, u8) -OPENCV_HAL_IMPL_NEON_EXPAND(v_int8x16, v_int16x8, schar, s8) -OPENCV_HAL_IMPL_NEON_EXPAND(v_uint16x8, v_uint32x4, ushort, u16) -OPENCV_HAL_IMPL_NEON_EXPAND(v_int16x8, v_int32x4, short, s16) -OPENCV_HAL_IMPL_NEON_EXPAND(v_uint32x4, v_uint64x2, uint, u32) -OPENCV_HAL_IMPL_NEON_EXPAND(v_int32x4, v_int64x2, int, s32) - -inline v_uint32x4 v_load_expand_q(const uchar* ptr) -{ - typedef unsigned int CV_DECL_ALIGNED(1) unaligned_uint; - uint8x8_t v0 = vcreate_u8(*(unaligned_uint*)ptr); - uint16x4_t v1 = vget_low_u16(vmovl_u8(v0)); - return v_uint32x4(vmovl_u16(v1)); -} - -inline v_int32x4 v_load_expand_q(const schar* ptr) -{ - typedef unsigned int CV_DECL_ALIGNED(1) unaligned_uint; - int8x8_t v0 = vcreate_s8(*(unaligned_uint*)ptr); - int16x4_t v1 = vget_low_s16(vmovl_s8(v0)); - return v_int32x4(vmovl_s16(v1)); -} - -#if defined(__aarch64__) -#define OPENCV_HAL_IMPL_NEON_UNPACKS(_Tpvec, suffix) \ -inline void v_zip(const v_##_Tpvec& a0, const v_##_Tpvec& a1, v_##_Tpvec& b0, v_##_Tpvec& b1) \ -{ \ - b0.val = vzip1q_##suffix(a0.val, a1.val); \ - b1.val = vzip2q_##suffix(a0.val, a1.val); \ -} \ -inline v_##_Tpvec v_combine_low(const v_##_Tpvec& a, const v_##_Tpvec& b) \ -{ \ - return v_##_Tpvec(vcombine_##suffix(vget_low_##suffix(a.val), vget_low_##suffix(b.val))); \ -} \ -inline v_##_Tpvec v_combine_high(const v_##_Tpvec& a, const v_##_Tpvec& b) \ -{ \ - return v_##_Tpvec(vcombine_##suffix(vget_high_##suffix(a.val), vget_high_##suffix(b.val))); \ -} \ -inline void v_recombine(const v_##_Tpvec& a, const v_##_Tpvec& b, v_##_Tpvec& c, v_##_Tpvec& d) \ -{ \ - c.val = vcombine_##suffix(vget_low_##suffix(a.val), vget_low_##suffix(b.val)); \ - d.val = vcombine_##suffix(vget_high_##suffix(a.val), vget_high_##suffix(b.val)); \ -} -#else -#define OPENCV_HAL_IMPL_NEON_UNPACKS(_Tpvec, suffix) \ -inline void v_zip(const v_##_Tpvec& a0, const v_##_Tpvec& a1, v_##_Tpvec& b0, v_##_Tpvec& b1) \ -{ \ - _Tpvec##x2_t p = vzipq_##suffix(a0.val, a1.val); \ - b0.val = p.val[0]; \ - b1.val = p.val[1]; \ -} \ -inline v_##_Tpvec v_combine_low(const v_##_Tpvec& a, const v_##_Tpvec& b) \ -{ \ - return v_##_Tpvec(vcombine_##suffix(vget_low_##suffix(a.val), vget_low_##suffix(b.val))); \ -} \ -inline v_##_Tpvec v_combine_high(const v_##_Tpvec& a, const v_##_Tpvec& b) \ -{ \ - return v_##_Tpvec(vcombine_##suffix(vget_high_##suffix(a.val), vget_high_##suffix(b.val))); \ -} \ -inline void v_recombine(const v_##_Tpvec& a, const v_##_Tpvec& b, v_##_Tpvec& c, v_##_Tpvec& d) \ -{ \ - c.val = vcombine_##suffix(vget_low_##suffix(a.val), vget_low_##suffix(b.val)); \ - d.val = vcombine_##suffix(vget_high_##suffix(a.val), vget_high_##suffix(b.val)); \ -} -#endif - -OPENCV_HAL_IMPL_NEON_UNPACKS(uint8x16, u8) -OPENCV_HAL_IMPL_NEON_UNPACKS(int8x16, s8) -OPENCV_HAL_IMPL_NEON_UNPACKS(uint16x8, u16) -OPENCV_HAL_IMPL_NEON_UNPACKS(int16x8, s16) -OPENCV_HAL_IMPL_NEON_UNPACKS(uint32x4, u32) -OPENCV_HAL_IMPL_NEON_UNPACKS(int32x4, s32) -OPENCV_HAL_IMPL_NEON_UNPACKS(float32x4, f32) -#if CV_SIMD128_64F -OPENCV_HAL_IMPL_NEON_UNPACKS(float64x2, f64) -#endif - -#define OPENCV_HAL_IMPL_NEON_EXTRACT(_Tpvec, suffix) \ -template \ -inline v_##_Tpvec v_extract(const v_##_Tpvec& a, const v_##_Tpvec& b) \ -{ \ - return v_##_Tpvec(vextq_##suffix(a.val, b.val, s)); \ -} - -OPENCV_HAL_IMPL_NEON_EXTRACT(uint8x16, u8) -OPENCV_HAL_IMPL_NEON_EXTRACT(int8x16, s8) -OPENCV_HAL_IMPL_NEON_EXTRACT(uint16x8, u16) -OPENCV_HAL_IMPL_NEON_EXTRACT(int16x8, s16) -OPENCV_HAL_IMPL_NEON_EXTRACT(uint32x4, u32) -OPENCV_HAL_IMPL_NEON_EXTRACT(int32x4, s32) -OPENCV_HAL_IMPL_NEON_EXTRACT(uint64x2, u64) -OPENCV_HAL_IMPL_NEON_EXTRACT(int64x2, s64) -OPENCV_HAL_IMPL_NEON_EXTRACT(float32x4, f32) -#if CV_SIMD128_64F -OPENCV_HAL_IMPL_NEON_EXTRACT(float64x2, f64) -#endif - -#if CV_SIMD128_64F -inline v_int32x4 v_round(const v_float32x4& a) -{ - float32x4_t a_ = a.val; - int32x4_t result; - __asm__ ("fcvtns %0.4s, %1.4s" - : "=w"(result) - : "w"(a_) - : /* No clobbers */); - return v_int32x4(result); -} -#else -inline v_int32x4 v_round(const v_float32x4& a) -{ - static const int32x4_t v_sign = vdupq_n_s32(1 << 31), - v_05 = vreinterpretq_s32_f32(vdupq_n_f32(0.5f)); - - int32x4_t v_addition = vorrq_s32(v_05, vandq_s32(v_sign, vreinterpretq_s32_f32(a.val))); - return v_int32x4(vcvtq_s32_f32(vaddq_f32(a.val, vreinterpretq_f32_s32(v_addition)))); -} -#endif -inline v_int32x4 v_floor(const v_float32x4& a) -{ - int32x4_t a1 = vcvtq_s32_f32(a.val); - uint32x4_t mask = vcgtq_f32(vcvtq_f32_s32(a1), a.val); - return v_int32x4(vaddq_s32(a1, vreinterpretq_s32_u32(mask))); -} - -inline v_int32x4 v_ceil(const v_float32x4& a) -{ - int32x4_t a1 = vcvtq_s32_f32(a.val); - uint32x4_t mask = vcgtq_f32(a.val, vcvtq_f32_s32(a1)); - return v_int32x4(vsubq_s32(a1, vreinterpretq_s32_u32(mask))); -} - -inline v_int32x4 v_trunc(const v_float32x4& a) -{ return v_int32x4(vcvtq_s32_f32(a.val)); } - -#if CV_SIMD128_64F -inline v_int32x4 v_round(const v_float64x2& a) -{ - static const int32x2_t zero = vdup_n_s32(0); - return v_int32x4(vcombine_s32(vmovn_s64(vcvtaq_s64_f64(a.val)), zero)); -} - -inline v_int32x4 v_round(const v_float64x2& a, const v_float64x2& b) -{ - return v_int32x4(vcombine_s32(vmovn_s64(vcvtaq_s64_f64(a.val)), vmovn_s64(vcvtaq_s64_f64(b.val)))); -} - -inline v_int32x4 v_floor(const v_float64x2& a) -{ - static const int32x2_t zero = vdup_n_s32(0); - int64x2_t a1 = vcvtq_s64_f64(a.val); - uint64x2_t mask = vcgtq_f64(vcvtq_f64_s64(a1), a.val); - a1 = vaddq_s64(a1, vreinterpretq_s64_u64(mask)); - return v_int32x4(vcombine_s32(vmovn_s64(a1), zero)); -} - -inline v_int32x4 v_ceil(const v_float64x2& a) -{ - static const int32x2_t zero = vdup_n_s32(0); - int64x2_t a1 = vcvtq_s64_f64(a.val); - uint64x2_t mask = vcgtq_f64(a.val, vcvtq_f64_s64(a1)); - a1 = vsubq_s64(a1, vreinterpretq_s64_u64(mask)); - return v_int32x4(vcombine_s32(vmovn_s64(a1), zero)); -} - -inline v_int32x4 v_trunc(const v_float64x2& a) -{ - static const int32x2_t zero = vdup_n_s32(0); - return v_int32x4(vcombine_s32(vmovn_s64(vcvtaq_s64_f64(a.val)), zero)); -} -#endif - -#define OPENCV_HAL_IMPL_NEON_TRANSPOSE4x4(_Tpvec, suffix) \ -inline void v_transpose4x4(const v_##_Tpvec& a0, const v_##_Tpvec& a1, \ - const v_##_Tpvec& a2, const v_##_Tpvec& a3, \ - v_##_Tpvec& b0, v_##_Tpvec& b1, \ - v_##_Tpvec& b2, v_##_Tpvec& b3) \ -{ \ - /* m00 m01 m02 m03 */ \ - /* m10 m11 m12 m13 */ \ - /* m20 m21 m22 m23 */ \ - /* m30 m31 m32 m33 */ \ - _Tpvec##x2_t t0 = vtrnq_##suffix(a0.val, a1.val); \ - _Tpvec##x2_t t1 = vtrnq_##suffix(a2.val, a3.val); \ - /* m00 m10 m02 m12 */ \ - /* m01 m11 m03 m13 */ \ - /* m20 m30 m22 m32 */ \ - /* m21 m31 m23 m33 */ \ - b0.val = vcombine_##suffix(vget_low_##suffix(t0.val[0]), vget_low_##suffix(t1.val[0])); \ - b1.val = vcombine_##suffix(vget_low_##suffix(t0.val[1]), vget_low_##suffix(t1.val[1])); \ - b2.val = vcombine_##suffix(vget_high_##suffix(t0.val[0]), vget_high_##suffix(t1.val[0])); \ - b3.val = vcombine_##suffix(vget_high_##suffix(t0.val[1]), vget_high_##suffix(t1.val[1])); \ -} - -OPENCV_HAL_IMPL_NEON_TRANSPOSE4x4(uint32x4, u32) -OPENCV_HAL_IMPL_NEON_TRANSPOSE4x4(int32x4, s32) -OPENCV_HAL_IMPL_NEON_TRANSPOSE4x4(float32x4, f32) - -#define OPENCV_HAL_IMPL_NEON_INTERLEAVED(_Tpvec, _Tp, suffix) \ -inline void v_load_deinterleave(const _Tp* ptr, v_##_Tpvec& a, v_##_Tpvec& b) \ -{ \ - _Tpvec##x2_t v = vld2q_##suffix(ptr); \ - a.val = v.val[0]; \ - b.val = v.val[1]; \ -} \ -inline void v_load_deinterleave(const _Tp* ptr, v_##_Tpvec& a, v_##_Tpvec& b, v_##_Tpvec& c) \ -{ \ - _Tpvec##x3_t v = vld3q_##suffix(ptr); \ - a.val = v.val[0]; \ - b.val = v.val[1]; \ - c.val = v.val[2]; \ -} \ -inline void v_load_deinterleave(const _Tp* ptr, v_##_Tpvec& a, v_##_Tpvec& b, \ - v_##_Tpvec& c, v_##_Tpvec& d) \ -{ \ - _Tpvec##x4_t v = vld4q_##suffix(ptr); \ - a.val = v.val[0]; \ - b.val = v.val[1]; \ - c.val = v.val[2]; \ - d.val = v.val[3]; \ -} \ -inline void v_store_interleave( _Tp* ptr, const v_##_Tpvec& a, const v_##_Tpvec& b, \ - hal::StoreMode /*mode*/=hal::STORE_UNALIGNED) \ -{ \ - _Tpvec##x2_t v; \ - v.val[0] = a.val; \ - v.val[1] = b.val; \ - vst2q_##suffix(ptr, v); \ -} \ -inline void v_store_interleave( _Tp* ptr, const v_##_Tpvec& a, const v_##_Tpvec& b, \ - const v_##_Tpvec& c, hal::StoreMode /*mode*/=hal::STORE_UNALIGNED) \ -{ \ - _Tpvec##x3_t v; \ - v.val[0] = a.val; \ - v.val[1] = b.val; \ - v.val[2] = c.val; \ - vst3q_##suffix(ptr, v); \ -} \ -inline void v_store_interleave( _Tp* ptr, const v_##_Tpvec& a, const v_##_Tpvec& b, \ - const v_##_Tpvec& c, const v_##_Tpvec& d, \ - hal::StoreMode /*mode*/=hal::STORE_UNALIGNED ) \ -{ \ - _Tpvec##x4_t v; \ - v.val[0] = a.val; \ - v.val[1] = b.val; \ - v.val[2] = c.val; \ - v.val[3] = d.val; \ - vst4q_##suffix(ptr, v); \ -} - -#define OPENCV_HAL_IMPL_NEON_INTERLEAVED_INT64(tp, suffix) \ -inline void v_load_deinterleave( const tp* ptr, v_##tp##x2& a, v_##tp##x2& b ) \ -{ \ - tp##x1_t a0 = vld1_##suffix(ptr); \ - tp##x1_t b0 = vld1_##suffix(ptr + 1); \ - tp##x1_t a1 = vld1_##suffix(ptr + 2); \ - tp##x1_t b1 = vld1_##suffix(ptr + 3); \ - a = v_##tp##x2(vcombine_##suffix(a0, a1)); \ - b = v_##tp##x2(vcombine_##suffix(b0, b1)); \ -} \ - \ -inline void v_load_deinterleave( const tp* ptr, v_##tp##x2& a, \ - v_##tp##x2& b, v_##tp##x2& c ) \ -{ \ - tp##x1_t a0 = vld1_##suffix(ptr); \ - tp##x1_t b0 = vld1_##suffix(ptr + 1); \ - tp##x1_t c0 = vld1_##suffix(ptr + 2); \ - tp##x1_t a1 = vld1_##suffix(ptr + 3); \ - tp##x1_t b1 = vld1_##suffix(ptr + 4); \ - tp##x1_t c1 = vld1_##suffix(ptr + 5); \ - a = v_##tp##x2(vcombine_##suffix(a0, a1)); \ - b = v_##tp##x2(vcombine_##suffix(b0, b1)); \ - c = v_##tp##x2(vcombine_##suffix(c0, c1)); \ -} \ - \ -inline void v_load_deinterleave( const tp* ptr, v_##tp##x2& a, v_##tp##x2& b, \ - v_##tp##x2& c, v_##tp##x2& d ) \ -{ \ - tp##x1_t a0 = vld1_##suffix(ptr); \ - tp##x1_t b0 = vld1_##suffix(ptr + 1); \ - tp##x1_t c0 = vld1_##suffix(ptr + 2); \ - tp##x1_t d0 = vld1_##suffix(ptr + 3); \ - tp##x1_t a1 = vld1_##suffix(ptr + 4); \ - tp##x1_t b1 = vld1_##suffix(ptr + 5); \ - tp##x1_t c1 = vld1_##suffix(ptr + 6); \ - tp##x1_t d1 = vld1_##suffix(ptr + 7); \ - a = v_##tp##x2(vcombine_##suffix(a0, a1)); \ - b = v_##tp##x2(vcombine_##suffix(b0, b1)); \ - c = v_##tp##x2(vcombine_##suffix(c0, c1)); \ - d = v_##tp##x2(vcombine_##suffix(d0, d1)); \ -} \ - \ -inline void v_store_interleave( tp* ptr, const v_##tp##x2& a, const v_##tp##x2& b, \ - hal::StoreMode /*mode*/=hal::STORE_UNALIGNED) \ -{ \ - vst1_##suffix(ptr, vget_low_##suffix(a.val)); \ - vst1_##suffix(ptr + 1, vget_low_##suffix(b.val)); \ - vst1_##suffix(ptr + 2, vget_high_##suffix(a.val)); \ - vst1_##suffix(ptr + 3, vget_high_##suffix(b.val)); \ -} \ - \ -inline void v_store_interleave( tp* ptr, const v_##tp##x2& a, \ - const v_##tp##x2& b, const v_##tp##x2& c, \ - hal::StoreMode /*mode*/=hal::STORE_UNALIGNED) \ -{ \ - vst1_##suffix(ptr, vget_low_##suffix(a.val)); \ - vst1_##suffix(ptr + 1, vget_low_##suffix(b.val)); \ - vst1_##suffix(ptr + 2, vget_low_##suffix(c.val)); \ - vst1_##suffix(ptr + 3, vget_high_##suffix(a.val)); \ - vst1_##suffix(ptr + 4, vget_high_##suffix(b.val)); \ - vst1_##suffix(ptr + 5, vget_high_##suffix(c.val)); \ -} \ - \ -inline void v_store_interleave( tp* ptr, const v_##tp##x2& a, const v_##tp##x2& b, \ - const v_##tp##x2& c, const v_##tp##x2& d, \ - hal::StoreMode /*mode*/=hal::STORE_UNALIGNED) \ -{ \ - vst1_##suffix(ptr, vget_low_##suffix(a.val)); \ - vst1_##suffix(ptr + 1, vget_low_##suffix(b.val)); \ - vst1_##suffix(ptr + 2, vget_low_##suffix(c.val)); \ - vst1_##suffix(ptr + 3, vget_low_##suffix(d.val)); \ - vst1_##suffix(ptr + 4, vget_high_##suffix(a.val)); \ - vst1_##suffix(ptr + 5, vget_high_##suffix(b.val)); \ - vst1_##suffix(ptr + 6, vget_high_##suffix(c.val)); \ - vst1_##suffix(ptr + 7, vget_high_##suffix(d.val)); \ -} - -OPENCV_HAL_IMPL_NEON_INTERLEAVED(uint8x16, uchar, u8) -OPENCV_HAL_IMPL_NEON_INTERLEAVED(int8x16, schar, s8) -OPENCV_HAL_IMPL_NEON_INTERLEAVED(uint16x8, ushort, u16) -OPENCV_HAL_IMPL_NEON_INTERLEAVED(int16x8, short, s16) -OPENCV_HAL_IMPL_NEON_INTERLEAVED(uint32x4, unsigned, u32) -OPENCV_HAL_IMPL_NEON_INTERLEAVED(int32x4, int, s32) -OPENCV_HAL_IMPL_NEON_INTERLEAVED(float32x4, float, f32) -#if CV_SIMD128_64F -OPENCV_HAL_IMPL_NEON_INTERLEAVED(float64x2, double, f64) -#endif - -OPENCV_HAL_IMPL_NEON_INTERLEAVED_INT64(int64, s64) -OPENCV_HAL_IMPL_NEON_INTERLEAVED_INT64(uint64, u64) - -inline v_float32x4 v_cvt_f32(const v_int32x4& a) -{ - return v_float32x4(vcvtq_f32_s32(a.val)); -} - -#if CV_SIMD128_64F -inline v_float32x4 v_cvt_f32(const v_float64x2& a) -{ - float32x2_t zero = vdup_n_f32(0.0f); - return v_float32x4(vcombine_f32(vcvt_f32_f64(a.val), zero)); -} - -inline v_float32x4 v_cvt_f32(const v_float64x2& a, const v_float64x2& b) -{ - return v_float32x4(vcombine_f32(vcvt_f32_f64(a.val), vcvt_f32_f64(b.val))); -} - -inline v_float64x2 v_cvt_f64(const v_int32x4& a) -{ - return v_float64x2(vcvt_f64_f32(vcvt_f32_s32(vget_low_s32(a.val)))); -} - -inline v_float64x2 v_cvt_f64_high(const v_int32x4& a) -{ - return v_float64x2(vcvt_f64_f32(vcvt_f32_s32(vget_high_s32(a.val)))); -} - -inline v_float64x2 v_cvt_f64(const v_float32x4& a) -{ - return v_float64x2(vcvt_f64_f32(vget_low_f32(a.val))); -} - -inline v_float64x2 v_cvt_f64_high(const v_float32x4& a) -{ - return v_float64x2(vcvt_f64_f32(vget_high_f32(a.val))); -} - -inline v_float64x2 v_cvt_f64(const v_int64x2& a) -{ return v_float64x2(vcvtq_f64_s64(a.val)); } - -#endif - -////////////// Lookup table access //////////////////// - -inline v_int8x16 v_lut(const schar* tab, const int* idx) -{ - schar CV_DECL_ALIGNED(32) elems[16] = - { - tab[idx[ 0]], - tab[idx[ 1]], - tab[idx[ 2]], - tab[idx[ 3]], - tab[idx[ 4]], - tab[idx[ 5]], - tab[idx[ 6]], - tab[idx[ 7]], - tab[idx[ 8]], - tab[idx[ 9]], - tab[idx[10]], - tab[idx[11]], - tab[idx[12]], - tab[idx[13]], - tab[idx[14]], - tab[idx[15]] - }; - return v_int8x16(vld1q_s8(elems)); -} -inline v_int8x16 v_lut_pairs(const schar* tab, const int* idx) -{ - schar CV_DECL_ALIGNED(32) elems[16] = - { - tab[idx[0]], - tab[idx[0] + 1], - tab[idx[1]], - tab[idx[1] + 1], - tab[idx[2]], - tab[idx[2] + 1], - tab[idx[3]], - tab[idx[3] + 1], - tab[idx[4]], - tab[idx[4] + 1], - tab[idx[5]], - tab[idx[5] + 1], - tab[idx[6]], - tab[idx[6] + 1], - tab[idx[7]], - tab[idx[7] + 1] - }; - return v_int8x16(vld1q_s8(elems)); -} -inline v_int8x16 v_lut_quads(const schar* tab, const int* idx) -{ - schar CV_DECL_ALIGNED(32) elems[16] = - { - tab[idx[0]], - tab[idx[0] + 1], - tab[idx[0] + 2], - tab[idx[0] + 3], - tab[idx[1]], - tab[idx[1] + 1], - tab[idx[1] + 2], - tab[idx[1] + 3], - tab[idx[2]], - tab[idx[2] + 1], - tab[idx[2] + 2], - tab[idx[2] + 3], - tab[idx[3]], - tab[idx[3] + 1], - tab[idx[3] + 2], - tab[idx[3] + 3] - }; - return v_int8x16(vld1q_s8(elems)); -} -inline v_uint8x16 v_lut(const uchar* tab, const int* idx) { return v_reinterpret_as_u8(v_lut((schar*)tab, idx)); } -inline v_uint8x16 v_lut_pairs(const uchar* tab, const int* idx) { return v_reinterpret_as_u8(v_lut_pairs((schar*)tab, idx)); } -inline v_uint8x16 v_lut_quads(const uchar* tab, const int* idx) { return v_reinterpret_as_u8(v_lut_quads((schar*)tab, idx)); } - -inline v_int16x8 v_lut(const short* tab, const int* idx) -{ - short CV_DECL_ALIGNED(32) elems[8] = - { - tab[idx[0]], - tab[idx[1]], - tab[idx[2]], - tab[idx[3]], - tab[idx[4]], - tab[idx[5]], - tab[idx[6]], - tab[idx[7]] - }; - return v_int16x8(vld1q_s16(elems)); -} -inline v_int16x8 v_lut_pairs(const short* tab, const int* idx) -{ - short CV_DECL_ALIGNED(32) elems[8] = - { - tab[idx[0]], - tab[idx[0] + 1], - tab[idx[1]], - tab[idx[1] + 1], - tab[idx[2]], - tab[idx[2] + 1], - tab[idx[3]], - tab[idx[3] + 1] - }; - return v_int16x8(vld1q_s16(elems)); -} -inline v_int16x8 v_lut_quads(const short* tab, const int* idx) -{ - return v_int16x8(vcombine_s16(vld1_s16(tab + idx[0]), vld1_s16(tab + idx[1]))); -} -inline v_uint16x8 v_lut(const ushort* tab, const int* idx) { return v_reinterpret_as_u16(v_lut((short*)tab, idx)); } -inline v_uint16x8 v_lut_pairs(const ushort* tab, const int* idx) { return v_reinterpret_as_u16(v_lut_pairs((short*)tab, idx)); } -inline v_uint16x8 v_lut_quads(const ushort* tab, const int* idx) { return v_reinterpret_as_u16(v_lut_quads((short*)tab, idx)); } - -inline v_int32x4 v_lut(const int* tab, const int* idx) -{ - int CV_DECL_ALIGNED(32) elems[4] = - { - tab[idx[0]], - tab[idx[1]], - tab[idx[2]], - tab[idx[3]] - }; - return v_int32x4(vld1q_s32(elems)); -} -inline v_int32x4 v_lut_pairs(const int* tab, const int* idx) -{ - return v_int32x4(vcombine_s32(vld1_s32(tab + idx[0]), vld1_s32(tab + idx[1]))); -} -inline v_int32x4 v_lut_quads(const int* tab, const int* idx) -{ - return v_int32x4(vld1q_s32(tab + idx[0])); -} -inline v_uint32x4 v_lut(const unsigned* tab, const int* idx) { return v_reinterpret_as_u32(v_lut((int*)tab, idx)); } -inline v_uint32x4 v_lut_pairs(const unsigned* tab, const int* idx) { return v_reinterpret_as_u32(v_lut_pairs((int*)tab, idx)); } -inline v_uint32x4 v_lut_quads(const unsigned* tab, const int* idx) { return v_reinterpret_as_u32(v_lut_quads((int*)tab, idx)); } - -inline v_int64x2 v_lut(const int64_t* tab, const int* idx) -{ - return v_int64x2(vcombine_s64(vcreate_s64(tab[idx[0]]), vcreate_s64(tab[idx[1]]))); -} -inline v_int64x2 v_lut_pairs(const int64_t* tab, const int* idx) -{ - return v_int64x2(vld1q_s64(tab + idx[0])); -} -inline v_uint64x2 v_lut(const uint64_t* tab, const int* idx) { return v_reinterpret_as_u64(v_lut((const int64_t *)tab, idx)); } -inline v_uint64x2 v_lut_pairs(const uint64_t* tab, const int* idx) { return v_reinterpret_as_u64(v_lut_pairs((const int64_t *)tab, idx)); } - -inline v_float32x4 v_lut(const float* tab, const int* idx) -{ - float CV_DECL_ALIGNED(32) elems[4] = - { - tab[idx[0]], - tab[idx[1]], - tab[idx[2]], - tab[idx[3]] - }; - return v_float32x4(vld1q_f32(elems)); -} -inline v_float32x4 v_lut_pairs(const float* tab, const int* idx) -{ - uint64 CV_DECL_ALIGNED(32) elems[2] = - { - *(uint64*)(tab + idx[0]), - *(uint64*)(tab + idx[1]) - }; - return v_float32x4(vreinterpretq_f32_u64(vld1q_u64(elems))); -} -inline v_float32x4 v_lut_quads(const float* tab, const int* idx) -{ - return v_float32x4(vld1q_f32(tab + idx[0])); -} - -inline v_int32x4 v_lut(const int* tab, const v_int32x4& idxvec) -{ - int CV_DECL_ALIGNED(32) elems[4] = - { - tab[vgetq_lane_s32(idxvec.val, 0)], - tab[vgetq_lane_s32(idxvec.val, 1)], - tab[vgetq_lane_s32(idxvec.val, 2)], - tab[vgetq_lane_s32(idxvec.val, 3)] - }; - return v_int32x4(vld1q_s32(elems)); -} - -inline v_uint32x4 v_lut(const unsigned* tab, const v_int32x4& idxvec) -{ - unsigned CV_DECL_ALIGNED(32) elems[4] = - { - tab[vgetq_lane_s32(idxvec.val, 0)], - tab[vgetq_lane_s32(idxvec.val, 1)], - tab[vgetq_lane_s32(idxvec.val, 2)], - tab[vgetq_lane_s32(idxvec.val, 3)] - }; - return v_uint32x4(vld1q_u32(elems)); -} - -inline v_float32x4 v_lut(const float* tab, const v_int32x4& idxvec) -{ - float CV_DECL_ALIGNED(32) elems[4] = - { - tab[vgetq_lane_s32(idxvec.val, 0)], - tab[vgetq_lane_s32(idxvec.val, 1)], - tab[vgetq_lane_s32(idxvec.val, 2)], - tab[vgetq_lane_s32(idxvec.val, 3)] - }; - return v_float32x4(vld1q_f32(elems)); -} - -inline void v_lut_deinterleave(const float* tab, const v_int32x4& idxvec, v_float32x4& x, v_float32x4& y) -{ - /*int CV_DECL_ALIGNED(32) idx[4]; - v_store(idx, idxvec); - - float32x4_t xy02 = vcombine_f32(vld1_f32(tab + idx[0]), vld1_f32(tab + idx[2])); - float32x4_t xy13 = vcombine_f32(vld1_f32(tab + idx[1]), vld1_f32(tab + idx[3])); - - float32x4x2_t xxyy = vuzpq_f32(xy02, xy13); - x = v_float32x4(xxyy.val[0]); - y = v_float32x4(xxyy.val[1]);*/ - int CV_DECL_ALIGNED(32) idx[4]; - v_store_aligned(idx, idxvec); - - x = v_float32x4(tab[idx[0]], tab[idx[1]], tab[idx[2]], tab[idx[3]]); - y = v_float32x4(tab[idx[0]+1], tab[idx[1]+1], tab[idx[2]+1], tab[idx[3]+1]); -} - -inline v_int8x16 v_interleave_pairs(const v_int8x16& vec) -{ - return v_int8x16(vcombine_s8(vtbl1_s8(vget_low_s8(vec.val), vcreate_s8(0x0705060403010200)), vtbl1_s8(vget_high_s8(vec.val), vcreate_s8(0x0705060403010200)))); -} -inline v_uint8x16 v_interleave_pairs(const v_uint8x16& vec) { return v_reinterpret_as_u8(v_interleave_pairs(v_reinterpret_as_s8(vec))); } -inline v_int8x16 v_interleave_quads(const v_int8x16& vec) -{ - return v_int8x16(vcombine_s8(vtbl1_s8(vget_low_s8(vec.val), vcreate_s8(0x0703060205010400)), vtbl1_s8(vget_high_s8(vec.val), vcreate_s8(0x0703060205010400)))); -} -inline v_uint8x16 v_interleave_quads(const v_uint8x16& vec) { return v_reinterpret_as_u8(v_interleave_quads(v_reinterpret_as_s8(vec))); } - -inline v_int16x8 v_interleave_pairs(const v_int16x8& vec) -{ - return v_int16x8(vreinterpretq_s16_s8(vcombine_s8(vtbl1_s8(vget_low_s8(vreinterpretq_s8_s16(vec.val)), vcreate_s8(0x0706030205040100)), vtbl1_s8(vget_high_s8(vreinterpretq_s8_s16(vec.val)), vcreate_s8(0x0706030205040100))))); -} -inline v_uint16x8 v_interleave_pairs(const v_uint16x8& vec) { return v_reinterpret_as_u16(v_interleave_pairs(v_reinterpret_as_s16(vec))); } -inline v_int16x8 v_interleave_quads(const v_int16x8& vec) -{ - int16x4x2_t res = vzip_s16(vget_low_s16(vec.val), vget_high_s16(vec.val)); - return v_int16x8(vcombine_s16(res.val[0], res.val[1])); -} -inline v_uint16x8 v_interleave_quads(const v_uint16x8& vec) { return v_reinterpret_as_u16(v_interleave_quads(v_reinterpret_as_s16(vec))); } - -inline v_int32x4 v_interleave_pairs(const v_int32x4& vec) -{ - int32x2x2_t res = vzip_s32(vget_low_s32(vec.val), vget_high_s32(vec.val)); - return v_int32x4(vcombine_s32(res.val[0], res.val[1])); -} -inline v_uint32x4 v_interleave_pairs(const v_uint32x4& vec) { return v_reinterpret_as_u32(v_interleave_pairs(v_reinterpret_as_s32(vec))); } -inline v_float32x4 v_interleave_pairs(const v_float32x4& vec) { return v_reinterpret_as_f32(v_interleave_pairs(v_reinterpret_as_s32(vec))); } - -inline v_int8x16 v_pack_triplets(const v_int8x16& vec) -{ - return v_int8x16(vextq_s8(vcombine_s8(vtbl1_s8(vget_low_s8(vec.val), vcreate_s8(0x0605040201000000)), vtbl1_s8(vget_high_s8(vec.val), vcreate_s8(0x0807060504020100))), vdupq_n_s8(0), 2)); -} -inline v_uint8x16 v_pack_triplets(const v_uint8x16& vec) { return v_reinterpret_as_u8(v_pack_triplets(v_reinterpret_as_s8(vec))); } - -inline v_int16x8 v_pack_triplets(const v_int16x8& vec) -{ - return v_int16x8(vreinterpretq_s16_s8(vextq_s8(vcombine_s8(vtbl1_s8(vget_low_s8(vreinterpretq_s8_s16(vec.val)), vcreate_s8(0x0504030201000000)), vget_high_s8(vreinterpretq_s8_s16(vec.val))), vdupq_n_s8(0), 2))); -} -inline v_uint16x8 v_pack_triplets(const v_uint16x8& vec) { return v_reinterpret_as_u16(v_pack_triplets(v_reinterpret_as_s16(vec))); } - -inline v_int32x4 v_pack_triplets(const v_int32x4& vec) { return vec; } -inline v_uint32x4 v_pack_triplets(const v_uint32x4& vec) { return vec; } -inline v_float32x4 v_pack_triplets(const v_float32x4& vec) { return vec; } - -#if CV_SIMD128_64F -inline v_float64x2 v_lut(const double* tab, const int* idx) -{ - double CV_DECL_ALIGNED(32) elems[2] = - { - tab[idx[0]], - tab[idx[1]] - }; - return v_float64x2(vld1q_f64(elems)); -} - -inline v_float64x2 v_lut_pairs(const double* tab, const int* idx) -{ - return v_float64x2(vld1q_f64(tab + idx[0])); -} - -inline v_float64x2 v_lut(const double* tab, const v_int32x4& idxvec) -{ - double CV_DECL_ALIGNED(32) elems[2] = - { - tab[vgetq_lane_s32(idxvec.val, 0)], - tab[vgetq_lane_s32(idxvec.val, 1)], - }; - return v_float64x2(vld1q_f64(elems)); -} - -inline void v_lut_deinterleave(const double* tab, const v_int32x4& idxvec, v_float64x2& x, v_float64x2& y) -{ - int CV_DECL_ALIGNED(32) idx[4]; - v_store_aligned(idx, idxvec); - - x = v_float64x2(tab[idx[0]], tab[idx[1]]); - y = v_float64x2(tab[idx[0]+1], tab[idx[1]+1]); -} -#endif - -////// FP16 support /////// -// Unsupported. Currently disabled. -#if 0 -#if CV_FP16 -inline v_float32x4 v_load_expand(const float16_t* ptr) -{ - float16x4_t v = - #ifndef vld1_f16 // APPLE compiler defines vld1_f16 as macro - (float16x4_t)vld1_s16((const short*)ptr); - #else - vld1_f16((const __fp16*)ptr); - #endif - return v_float32x4(vcvt_f32_f16(v)); -} - -inline void v_pack_store(float16_t* ptr, const v_float32x4& v) -{ - float16x4_t hv = vcvt_f16_f32(v.val); - - #ifndef vst1_f16 // APPLE compiler defines vst1_f16 as macro - vst1_s16((short*)ptr, (int16x4_t)hv); - #else - vst1_f16((__fp16*)ptr, hv); - #endif -} -#else -inline v_float32x4 v_load_expand(const float16_t* ptr) -{ - const int N = 4; - float buf[N]; - for( int i = 0; i < N; i++ ) buf[i] = (float)ptr[i]; - return v_load(buf); -} - -inline void v_pack_store(float16_t* ptr, const v_float32x4& v) -{ - const int N = 4; - float buf[N]; - v_store(buf, v); - for( int i = 0; i < N; i++ ) ptr[i] = float16_t(buf[i]); -} -#endif -#endif - -inline void v_cleanup() {} - -static inline v_int16x8 v_mulhi(const v_int16x8& a, short b) { - v_int16x8 result; - - int16x8_t a1 = a.val; - int16x8_t b1 = vdupq_n_s16(b); - - int16x4_t a3210 = vget_low_s16(a1); - int16x4_t b3210 = vget_low_s16(b1); - int32x4_t ab3210 = vmull_s16(a3210, b3210); /* 3333222211110000 */ - - int16x4_t a7654 = vget_high_s16(a1); - int16x4_t b7654 = vget_high_s16(b1); - int32x4_t ab7654 = vmull_s16(a7654, b7654); /* 7777666655554444 */ - - /* issue with large values because of result saturation if to */ - /* int16x8_t ret = vqdmulhq_s16(a1, b1); */ /* =2*a*b */ - /* return vshrq_n_s16(ret, 1); */ - uint16x8x2_t r = vuzpq_u16(vreinterpretq_u16_s32(ab3210), vreinterpretq_u16_s32(ab7654)); - result.val = vreinterpretq_s16_u16(r.val[1]); - - return result; -} - -inline uint16x8_t v_mulhi_u16(const uint16x8_t& a, uint16x8_t b) { - uint16x4_t a3210 = vget_low_u16(a); - uint16x4_t b3210 = vget_low_u16(b); - uint32x4_t ab3210 = vmull_u16(a3210, b3210); /* 3333222211110000 */ - - uint16x4_t a7654 = vget_high_u16(a); - uint16x4_t b7654 = vget_high_u16(b); - uint32x4_t ab7654 = vmull_u16(a7654, b7654); /* 7777666655554444 */ - - uint16x8x2_t res = vuzpq_u16(vreinterpretq_u16_u32(ab3210), vreinterpretq_u16_u32(ab7654)); - return res.val[1]; -} - -inline v_uint16x8 v_mulhi(const v_uint16x8& a, v_uint16x8 b) { - v_uint16x8 result; - uint16x8_t a1 = a.val; - uint16x8_t b1 = b.val; - - result.val = v_mulhi_u16(a1, b1); - return result; -} - -inline v_uint16x8 v_mulhi(const v_uint16x8& a, uint16_t b) { - v_uint16x8 result; - uint16x8_t a1 = a.val; - uint16x8_t b1 = vdupq_n_u16(b); - - result.val = v_mulhi_u16(a1, b1); - return result; -} - -CV_ALWAYS_INLINE v_int16x8 v_mulhrs(const v_int16x8& a, const v_int16x8& b) -{ - // Multiply - int32x4_t mul_lo = vmull_s16(vget_low_s16(a.val), - vget_low_s16(b.val)); - int32x4_t mul_hi = vmull_s16(vget_high_s16(a.val), - vget_high_s16(b.val)); - - // Rounding narrowing shift right - // narrow = (int16_t)((mul + 16384) >> 15); - int16x4_t narrow_lo = vrshrn_n_s32(mul_lo, 15); - int16x4_t narrow_hi = vrshrn_n_s32(mul_hi, 15); - - // Join together - return v_int16x8(vcombine_s16(narrow_lo, narrow_hi)); -} - -CV_ALWAYS_INLINE v_int16x8 v_mulhrs(const v_int16x8& a, const short b) -{ - return v_mulhrs(a, v_setall_s16(b)); -} - -CV_ALWAYS_INLINE void v_gather_channel(v_uint8x16& vec, const uint8_t tmp[], - const short* index, const int chanNum, - const int c, const int shift) -{ - int32x4_t result = {}; - result = vsetq_lane_s32(*reinterpret_cast(&tmp[4 * (chanNum * (*(index + shift + 0)) + c)]), result, 0); - result = vsetq_lane_s32(*reinterpret_cast(&tmp[4 * (chanNum * (*(index + shift + 1)) + c)]), result, 1); - result = vsetq_lane_s32(*reinterpret_cast(&tmp[4 * (chanNum * (*(index + shift + 0) + 1) + c)]), result, 2); - result = vsetq_lane_s32(*reinterpret_cast(&tmp[4 * (chanNum * (*(index + shift + 1) + 1) + c)]), result, 3); - - vec.val = vreinterpretq_u8_s32(result); -} - -template -CV_ALWAYS_INLINE void v_gather_channel(v_uint8x16& vec, const uchar src[], const int channel) -{ - uint8x16_t result = {}; - result = vsetq_lane_u8(*reinterpret_cast(src + channel), result, 0); - result = vsetq_lane_u8(*reinterpret_cast(src + chanNum + channel), result, 1); - result = vsetq_lane_u8(*reinterpret_cast(src + 2 * chanNum + channel), result, 2); - result = vsetq_lane_u8(*reinterpret_cast(src + 3 * chanNum + channel), result, 3); - result = vsetq_lane_u8(*reinterpret_cast(src + 4 * chanNum + channel), result, 4); - result = vsetq_lane_u8(*reinterpret_cast(src + 5 * chanNum + channel), result, 5); - result = vsetq_lane_u8(*reinterpret_cast(src + 6 * chanNum + channel), result, 6); - result = vsetq_lane_u8(*reinterpret_cast(src + 7 * chanNum + channel), result, 7); - result = vsetq_lane_u8(*reinterpret_cast(src + 8 * chanNum + channel), result, 8); - result = vsetq_lane_u8(*reinterpret_cast(src + 9 * chanNum + channel), result, 9); - result = vsetq_lane_u8(*reinterpret_cast(src + 10 * chanNum + channel), result, 10); - result = vsetq_lane_u8(*reinterpret_cast(src + 11 * chanNum + channel), result, 11); - result = vsetq_lane_u8(*reinterpret_cast(src + 12 * chanNum + channel), result, 12); - result = vsetq_lane_u8(*reinterpret_cast(src + 13 * chanNum + channel), result, 13); - result = vsetq_lane_u8(*reinterpret_cast(src + 14 * chanNum + channel), result, 14); - result = vsetq_lane_u8(*reinterpret_cast(src + 15 * chanNum + channel), result, 15); - - vec.val = result; -} - -namespace { -template -CV_ALWAYS_INLINE void v_gather_channel(v_int16x8& vec, const uchar src[], const short* index, const int channel, const int pos) -{ - int16x8_t result = {}; - result = vsetq_lane_s16(*reinterpret_cast(&src[chanNum * (*index + pos) + channel]), result, 0); - result = vsetq_lane_s16(*reinterpret_cast(&src[chanNum * (*(index + 1) + pos) + channel]), result, 1); - result = vsetq_lane_s16(*reinterpret_cast(&src[chanNum * (*(index + 2) + pos) + channel]), result, 2); - result = vsetq_lane_s16(*reinterpret_cast(&src[chanNum * (*(index + 3) + pos) + channel]), result, 3); - result = vsetq_lane_s16(*reinterpret_cast(&src[chanNum * (*(index + 4) + pos) + channel]), result, 4); - result = vsetq_lane_s16(*reinterpret_cast(&src[chanNum * (*(index + 5) + pos) + channel]), result, 5); - result = vsetq_lane_s16(*reinterpret_cast(&src[chanNum * (*(index + 6) + pos) + channel]), result, 6); - result = vsetq_lane_s16(*reinterpret_cast(&src[chanNum * (*(index + 7) + pos) + channel]), result, 7); - - vec.val = result; -} -} // namespace - -CV_ALWAYS_INLINE v_uint8x16 v_gather_pairs(const uchar src[], const short* mapsx) -{ - int16x8_t result = {}; - result = vsetq_lane_s16(*reinterpret_cast(&src[*(mapsx + 0)]), result, 0); - result = vsetq_lane_s16(*reinterpret_cast(&src[*(mapsx + 1)]), result, 1); - result = vsetq_lane_s16(*reinterpret_cast(&src[*(mapsx + 2)]), result, 2); - result = vsetq_lane_s16(*reinterpret_cast(&src[*(mapsx + 3)]), result, 3); - result = vsetq_lane_s16(*reinterpret_cast(&src[*(mapsx + 4)]), result, 4); - result = vsetq_lane_s16(*reinterpret_cast(&src[*(mapsx + 5)]), result, 5); - result = vsetq_lane_s16(*reinterpret_cast(&src[*(mapsx + 6)]), result, 6); - result = vsetq_lane_s16(*reinterpret_cast(&src[*(mapsx + 7)]), result, 7); - return v_uint8x16(vreinterpretq_u8_s16(result)); -} - -CV_ALWAYS_INLINE v_uint8x16 v_gather_lines(const uchar src[], const short* mapsx) -{ - int32x4_t result = {}; - result = vsetq_lane_s32(*reinterpret_cast(&src[4 * (*(mapsx + 0))]), result, 0); - result = vsetq_lane_s32(*reinterpret_cast(&src[4 * (*(mapsx + 1))]), result, 1); - result = vsetq_lane_s32(*reinterpret_cast(&src[4 * (*(mapsx + 0) + 1)]), result, 2); - result = vsetq_lane_s32(*reinterpret_cast(&src[4 * (*(mapsx + 1) + 1)]), result, 3); - - return v_uint8x16(vreinterpretq_u8_s32(result)); -} - -CV_ALWAYS_INLINE void v_gather_pairs(const float src[], const int mapsx[], const int x, - v_float32x4& low, v_float32x4& high) -{ -#if defined(__aarch64__) - float64x2_t l = {}; - l = vsetq_lane_f64(*reinterpret_cast(&src[mapsx[x]]), l, 0); - l = vsetq_lane_f64(*reinterpret_cast(&src[mapsx[x + 1]]), l, 1); - low.val = vreinterpretq_f32_f64(l); - - float64x2_t h = {}; - h = vsetq_lane_f64(*reinterpret_cast(&src[mapsx[x + 2]]), h, 0); - h = vsetq_lane_f64(*reinterpret_cast(&src[mapsx[x + 3]]), h, 1); - high.val = vreinterpretq_f32_f64(h); -#else - float32x4_t l = {}; - l = vsetq_lane_f32(*reinterpret_cast(&src[mapsx[x]]), l, 0); - l = vsetq_lane_f32(*reinterpret_cast(&src[mapsx[x] + 1]), l, 1); - l = vsetq_lane_f32(*reinterpret_cast(&src[mapsx[x + 1]]), l, 2); - l = vsetq_lane_f32(*reinterpret_cast(&src[mapsx[x + 1] + 1]), l, 3); - low.val = l; - - float32x4_t h = {}; - h = vsetq_lane_f32(*reinterpret_cast(&src[mapsx[x + 2]]), h, 0); - h = vsetq_lane_f32(*reinterpret_cast(&src[mapsx[x + 2] + 1]), h, 1); - h = vsetq_lane_f32(*reinterpret_cast(&src[mapsx[x + 3]]), h, 2); - h = vsetq_lane_f32(*reinterpret_cast(&src[mapsx[x + 3] + 1]), h, 3); - high.val = h; -#endif - - return; -} - -CV_ALWAYS_INLINE v_float32x4 v_fma(const v_float32x4& a, float b, const v_float32x4& c) { - return v_fma(a, v_setall_f32(b), c); -} - -template -CV_ALWAYS_INLINE v_uint8x16 v_blend(const v_uint8x16& a, const v_uint8x16& b) -{ - const uint16_t _mask[8] = { ((imm) & (1 << 0)) ? 0xFFFF : 0x0000, - ((imm) & (1 << 1)) ? 0xFFFF : 0x0000, - ((imm) & (1 << 2)) ? 0xFFFF : 0x0000, - ((imm) & (1 << 3)) ? 0xFFFF : 0x0000, - ((imm) & (1 << 4)) ? 0xFFFF : 0x0000, - ((imm) & (1 << 5)) ? 0xFFFF : 0x0000, - ((imm) & (1 << 6)) ? 0xFFFF : 0x0000, - ((imm) & (1 << 7)) ? 0xFFFF : 0x0000 }; - uint16x8_t _mask_vec = vld1q_u16(_mask); - uint16x8_t _a = vreinterpretq_u16_u8(a.val); - uint16x8_t _b = vreinterpretq_u16_u8(b.val); - return v_uint8x16(vreinterpretq_u8_u16(vbslq_u16(_mask_vec, _b, _a))); -} - -CV_ALWAYS_INLINE v_uint8x16 v_shuffle(const v_uint8x16& a, const v_uint8x16& mask) -{ - - uint8x16_t tbl = a.val; // input a - uint8x16_t idx = mask.val; // input mask - uint8x16_t idx_masked = - vandq_u8(idx, vdupq_n_u8(0x8F)); // avoid using meaningless bits -#if defined(__aarch64__) - return v_uint8x16(vqtbl1q_u8(tbl, idx_masked)); -#elif defined(__GNUC__) - uint8x16_t ret; - // %e and %f represent the even and odd D registers - // respectively. - __asm__ __volatile__( - "vtbl.8 %e[ret], {%e[tbl], %f[tbl]}, %e[idx]\n" - "vtbl.8 %f[ret], {%e[tbl], %f[tbl]}, %f[idx]\n" - : [ret] "=&w"(ret) - : [tbl] "w"(tbl), [idx] "w"(idx_masked)); - - return v_uint8x16(ret); -#else - uint8x8x2_t a_split = { vget_low_u8(tbl), vget_high_u8(tbl) }; - - return v_uint8x16(vcombine_u8( - vtbl2_u8(a_split, vget_low_u8(idx_masked)), - vtbl2_u8(a_split, vget_high_u8(idx_masked)))); - -#endif -} - -CV_ALWAYS_INLINE void v_deinterleave(const v_float32x4& low, const v_float32x4& high, - v_float32x4& even, v_float32x4& odd) { - float32x4x2_t p1 = vzipq_f32(low.val, high.val); - float32x4_t tmp0 = p1.val[0]; - float32x4_t tmp1 = p1.val[1]; - - float32x4x2_t p2 = vzipq_f32(tmp0, tmp1); - even.val = p2.val[0]; - odd.val = p2.val[1]; - return; -} - -CV_ALWAYS_INLINE void v_deinterleave(const v_uint8x16& i0, const v_uint8x16& i1, - const v_uint8x16& i2, const v_uint8x16& i3, - v_uint8x16& res0, v_uint8x16& res1, - v_uint8x16& res2, v_uint8x16& res3) -{ - uint8x16x2_t p1 = vzipq_u8(i0.val, i2.val); - uint8x16x2_t p2 = vzipq_u8(i1.val, i3.val); - - uint8x16_t v0 = p1.val[0]; - uint8x16_t v1 = p1.val[1]; - uint8x16_t v2 = p2.val[0]; - uint8x16_t v3 = p2.val[1]; - - uint8x16x2_t p3 = vzipq_u8(v0, v2); - uint8x16x2_t p4 = vzipq_u8(v1, v3); - - uint8x16_t u0 = p3.val[0]; - uint8x16_t u2 = p3.val[1]; - uint8x16_t u1 = p4.val[0]; - uint8x16_t u3 = p4.val[1]; - - uint8x16x2_t p5 = vzipq_u8(u0, u1); - uint8x16x2_t p6 = vzipq_u8(u2, u3); - - v0 = p5.val[0]; - v2 = p5.val[1]; - v1 = p6.val[0]; - v3 = p6.val[1]; - - uint8x16x2_t p7 = vzipq_u8(v0, v1); - uint8x16x2_t p8 = vzipq_u8(v2, v3); - - res0.val = p7.val[0]; - res1.val = p7.val[1]; - res2.val = p8.val[0]; - res3.val = p8.val[1]; -} - -CV_ALWAYS_INLINE void v_deinterleave_expand(const v_uint8x16& src, - v_int16x8& even, v_int16x8& odd) -{ - constexpr int nlanes = static_cast(v_uint8x16::nlanes); - uchar mask_e[nlanes] = { 0x00, 0xff, 0x02, 0xff, 0x04, 0xff, 0x06, 0xff, - 0x08, 0xff, 0x0a, 0xff, 0x0c, 0xff, 0x0e, 0xff }; - - uchar mask_o[nlanes] = { 0x01, 0xff, 0x03, 0xff, 0x05, 0xff, 0x07, 0xff, - 0x09, 0xff, 0x0b, 0xff, 0x0d, 0xff, 0x0f, 0xff }; - - uint8x16_t mask_even = vld1q_u8(mask_e); - uint8x16_t mask_odd = vld1q_u8(mask_o); - - v_uint8x16 res1 = v_shuffle(src, v_uint8x16(mask_even)); - v_uint8x16 res2 = v_shuffle(src, v_uint8x16(mask_odd)); - even.val = vreinterpretq_s16_u8(res1.val); - odd.val = vreinterpretq_s16_u8(res2.val); -} - -CV_ALWAYS_INLINE v_int16x8 v_interleave_low(const v_int16x8& a, const v_int16x8& b) -{ - int16x8x2_t p = vzipq_s16(a.val, b.val); - int16x8_t v = p.val[0]; - return v_int16x8(v); -} - -CV_ALWAYS_INLINE v_int16x8 v_interleave_high(const v_int16x8& a, const v_int16x8& b) -{ - int16x8x2_t p = vzipq_s16(a.val, b.val); - int16x8_t v = p.val[1]; - return v_int16x8(v); -} - -CV_ALWAYS_INLINE v_uint8x16 v_interleave_low(const v_uint8x16& a, const v_uint8x16& b) -{ - uint8x16x2_t p = vzipq_u8(a.val, b.val); - uint8x16_t v = p.val[0]; - return v_uint8x16(v); -} - -CV_ALWAYS_INLINE v_uint8x16 v_interleave_high(const v_uint8x16& a, const v_uint8x16& b) -{ - uint8x16x2_t p = vzipq_u8(a.val, b.val); - uint8x16_t v = p.val[1]; - return v_uint8x16(v); -} - -CV_ALWAYS_INLINE void v_interleave(const v_int16x8& a, const v_int16x8& b, - v_int16x8& v1, v_int16x8& v2) -{ - int16x8x2_t p = vzipq_s16(a.val, b.val); - v1.val = p.val[0]; - v2.val = p.val[1]; - return; -} - -CV_ALWAYS_INLINE void v_interleave(const v_int32x4& a, const v_int32x4& b, - v_int32x4& v1, v_int32x4& v2) -{ - int32x4x2_t p = vzipq_s32(a.val, b.val); - v1.val = p.val[0]; - v2.val = p.val[1]; - return; -} - -template -CV_ALWAYS_INLINE v_uint8x16 v_slli_si128(const v_uint8x16& a) -{ - assert((shift > 0) && (shift <= 15)); - uint8x16_t ret = vextq_u8(vdupq_n_u8(0), a.val, shift /*16 - (imm)*/); - return v_uint8x16(ret); -} - -template -CV_ALWAYS_INLINE v_uint8x16 v_shift_right(const v_uint8x16& a) -{ - assert((shift > 0) && (shift <= 15)); - uint8x16_t ret = vextq_u8(a.val, vdupq_n_u8(0), shift); - return v_uint8x16(ret); -} - -template -CV_ALWAYS_INLINE v_uint8x16 v_shift_left(const v_uint8x16& a) -{ - return v_slli_si128<16 - shift>(a); -} - -template -CV_ALWAYS_INLINE v_uint8x16 v_insert(v_uint8x16& a, int64_t b) -{ - return v_uint8x16(vreinterpretq_u8_s64(vsetq_lane_s64(b, vreinterpretq_s64_u8(a.val), indx))); -} - -CV_CPU_OPTIMIZATION_HAL_NAMESPACE_END - -//! @endcond - -} - -#endif diff --git a/thirdparty/ocv/opencv_hal_sse.hpp b/thirdparty/ocv/opencv_hal_sse.hpp deleted file mode 100644 index 5be42b688e6..00000000000 --- a/thirdparty/ocv/opencv_hal_sse.hpp +++ /dev/null @@ -1,3098 +0,0 @@ -/*M/////////////////////////////////////////////////////////////////////////////////////// -// -// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. -// -// By downloading, copying, installing or using the software you agree to this license. -// If you do not agree to this license, do not download, install, -// copy or use the software. -// -// -// License Agreement -// For Open Source Computer Vision Library -// -// Copyright (C) 2000-2008, Intel Corporation, all rights reserved. -// Copyright (C) 2009, Willow Garage Inc., all rights reserved. -// Copyright (C) 2013, OpenCV Foundation, all rights reserved. -// Copyright (C) 2015, Itseez Inc., all rights reserved. -// Third party copyrights are property of their respective owners. -// -// Redistribution and use in source and binary forms, with or without modification, -// are permitted provided that the following conditions are met: -// -// * Redistribution's of source code must retain the above copyright notice, -// this list of conditions and the following disclaimer. -// -// * Redistribution's in binary form must reproduce the above copyright notice, -// this list of conditions and the following disclaimer in the documentation -// and/or other materials provided with the distribution. -// -// * The name of the copyright holders may not be used to endorse or promote products -// derived from this software without specific prior written permission. -// -// This software is provided by the copyright holders and contributors "as is" and -// any express or implied warranties, including, but not limited to, the implied -// warranties of merchantability and fitness for a particular purpose are disclaimed. -// In no event shall the Intel Corporation or contributors be liable for any direct, -// indirect, incidental, special, exemplary, or consequential damages -// (including, but not limited to, procurement of substitute goods or services; -// loss of use, data, or profits; or business interruption) however caused -// and on any theory of liability, whether in contract, strict liability, -// or tort (including negligence or otherwise) arising in any way out of -// the use of this software, even if advised of the possibility of such damage. -// -//M*/ - -#ifndef OPENCV_HAL_SSE_HPP -#define OPENCV_HAL_SSE_HPP - -#include - -/********************************** - * YL: removed needless dependency: -#include "opencv2/core/utility.hpp" -**********************************/ - -// DM: declare missing types -using uint64 = uint64_t; -using int64 = int64_t; - -#ifndef CV_StaticAssert -# define CV_StaticAssert static_assert -#endif - -#ifndef CV_DECL_ALIGNED -# ifdef __GNUC__ -# define CV_DECL_ALIGNED(x) __attribute__ ((aligned (x))) -# elif defined _MSC_VER -# define CV_DECL_ALIGNED(x) __declspec(align(x)) -# else -# define CV_DECL_ALIGNED(x) -# endif -#endif // CV_DECL_ALIGNED - - -#define CV_SIMD128 1 -#define CV_SIMD128_64F 1 -#define CV_SIMD128_FP16 0 // no native operations with FP16 type. - -namespace cv -{ - -//! @cond IGNORED - -CV_CPU_OPTIMIZATION_HAL_NAMESPACE_BEGIN - -struct v_uint8x16 -{ - typedef uchar lane_type; - typedef __m128i vector_type; - enum { nlanes = 16 }; - - v_uint8x16() : val(_mm_setzero_si128()) {} - explicit v_uint8x16(__m128i v) : val(v) {} - v_uint8x16(uchar v0, uchar v1, uchar v2, uchar v3, uchar v4, uchar v5, uchar v6, uchar v7, - uchar v8, uchar v9, uchar v10, uchar v11, uchar v12, uchar v13, uchar v14, uchar v15) - { - val = _mm_setr_epi8((char)v0, (char)v1, (char)v2, (char)v3, - (char)v4, (char)v5, (char)v6, (char)v7, - (char)v8, (char)v9, (char)v10, (char)v11, - (char)v12, (char)v13, (char)v14, (char)v15); - } - uchar get0() const - { - return (uchar)_mm_cvtsi128_si32(val); - } - - __m128i val; -}; - -struct v_int8x16 -{ - typedef schar lane_type; - typedef __m128i vector_type; - enum { nlanes = 16 }; - - v_int8x16() : val(_mm_setzero_si128()) {} - explicit v_int8x16(__m128i v) : val(v) {} - v_int8x16(schar v0, schar v1, schar v2, schar v3, schar v4, schar v5, schar v6, schar v7, - schar v8, schar v9, schar v10, schar v11, schar v12, schar v13, schar v14, schar v15) - { - val = _mm_setr_epi8((char)v0, (char)v1, (char)v2, (char)v3, - (char)v4, (char)v5, (char)v6, (char)v7, - (char)v8, (char)v9, (char)v10, (char)v11, - (char)v12, (char)v13, (char)v14, (char)v15); - } - schar get0() const - { - return (schar)_mm_cvtsi128_si32(val); - } - - __m128i val; -}; - -struct v_uint16x8 -{ - typedef ushort lane_type; - typedef __m128i vector_type; - enum { nlanes = 8 }; - - v_uint16x8() : val(_mm_setzero_si128()) {} - explicit v_uint16x8(__m128i v) : val(v) {} - v_uint16x8(ushort v0, ushort v1, ushort v2, ushort v3, ushort v4, ushort v5, ushort v6, ushort v7) - { - val = _mm_setr_epi16((short)v0, (short)v1, (short)v2, (short)v3, - (short)v4, (short)v5, (short)v6, (short)v7); - } - ushort get0() const - { - return (ushort)_mm_cvtsi128_si32(val); - } - - __m128i val; -}; - -struct v_int16x8 -{ - typedef short lane_type; - typedef __m128i vector_type; - enum { nlanes = 8 }; - - v_int16x8() : val(_mm_setzero_si128()) {} - explicit v_int16x8(__m128i v) : val(v) {} - v_int16x8(short v0, short v1, short v2, short v3, short v4, short v5, short v6, short v7) - { - val = _mm_setr_epi16((short)v0, (short)v1, (short)v2, (short)v3, - (short)v4, (short)v5, (short)v6, (short)v7); - } - short get0() const - { - return (short)_mm_cvtsi128_si32(val); - } - - __m128i val; -}; - -struct v_uint32x4 -{ - typedef unsigned lane_type; - typedef __m128i vector_type; - enum { nlanes = 4 }; - - v_uint32x4() : val(_mm_setzero_si128()) {} - explicit v_uint32x4(__m128i v) : val(v) {} - v_uint32x4(unsigned v0, unsigned v1, unsigned v2, unsigned v3) - { - val = _mm_setr_epi32((int)v0, (int)v1, (int)v2, (int)v3); - } - unsigned get0() const - { - return (unsigned)_mm_cvtsi128_si32(val); - } - - __m128i val; -}; - -struct v_int32x4 -{ - typedef int lane_type; - typedef __m128i vector_type; - enum { nlanes = 4 }; - - v_int32x4() : val(_mm_setzero_si128()) {} - explicit v_int32x4(__m128i v) : val(v) {} - v_int32x4(int v0, int v1, int v2, int v3) - { - val = _mm_setr_epi32(v0, v1, v2, v3); - } - int get0() const - { - return _mm_cvtsi128_si32(val); - } - - __m128i val; -}; - -struct v_float32x4 -{ - typedef float lane_type; - typedef __m128 vector_type; - enum { nlanes = 4 }; - - v_float32x4() : val(_mm_setzero_ps()) {} - explicit v_float32x4(__m128 v) : val(v) {} - v_float32x4(float v0, float v1, float v2, float v3) - { - val = _mm_setr_ps(v0, v1, v2, v3); - } - float get0() const - { - return _mm_cvtss_f32(val); - } - - __m128 val; -}; - -struct v_uint64x2 -{ - typedef uint64 lane_type; - typedef __m128i vector_type; - enum { nlanes = 2 }; - - v_uint64x2() : val(_mm_setzero_si128()) {} - explicit v_uint64x2(__m128i v) : val(v) {} - v_uint64x2(uint64 v0, uint64 v1) - { - val = _mm_setr_epi32((int)v0, (int)(v0 >> 32), (int)v1, (int)(v1 >> 32)); - } - uint64 get0() const - { - int a = _mm_cvtsi128_si32(val); - int b = _mm_cvtsi128_si32(_mm_srli_epi64(val, 32)); - return (unsigned)a | ((uint64)(unsigned)b << 32); - } - - __m128i val; -}; - -struct v_int64x2 -{ - typedef int64 lane_type; - typedef __m128i vector_type; - enum { nlanes = 2 }; - - v_int64x2() : val(_mm_setzero_si128()) {} - explicit v_int64x2(__m128i v) : val(v) {} - v_int64x2(int64 v0, int64 v1) - { - val = _mm_setr_epi32((int)v0, (int)(v0 >> 32), (int)v1, (int)(v1 >> 32)); - } - int64 get0() const - { - int a = _mm_cvtsi128_si32(val); - int b = _mm_cvtsi128_si32(_mm_srli_epi64(val, 32)); - return (int64)((unsigned)a | ((uint64)(unsigned)b << 32)); - } - - __m128i val; -}; - -struct v_float64x2 -{ - typedef double lane_type; - typedef __m128d vector_type; - enum { nlanes = 2 }; - - v_float64x2() : val(_mm_setzero_pd()) {} - explicit v_float64x2(__m128d v) : val(v) {} - v_float64x2(double v0, double v1) - { - val = _mm_setr_pd(v0, v1); - } - double get0() const - { - return _mm_cvtsd_f64(val); - } - - __m128d val; -}; - -namespace hal_sse_internal -{ - template - to_sse_type v_sse_reinterpret_as(const from_sse_type& val); - -#define OPENCV_HAL_IMPL_SSE_REINTERPRET_RAW(to_sse_type, from_sse_type, sse_cast_intrin) \ - template<> inline \ - to_sse_type v_sse_reinterpret_as(const from_sse_type& a) \ - { return sse_cast_intrin(a); } - - OPENCV_HAL_IMPL_SSE_REINTERPRET_RAW(__m128i, __m128i, OPENCV_HAL_NOP) - OPENCV_HAL_IMPL_SSE_REINTERPRET_RAW(__m128i, __m128, _mm_castps_si128) - OPENCV_HAL_IMPL_SSE_REINTERPRET_RAW(__m128i, __m128d, _mm_castpd_si128) - OPENCV_HAL_IMPL_SSE_REINTERPRET_RAW(__m128, __m128i, _mm_castsi128_ps) - OPENCV_HAL_IMPL_SSE_REINTERPRET_RAW(__m128, __m128, OPENCV_HAL_NOP) - OPENCV_HAL_IMPL_SSE_REINTERPRET_RAW(__m128, __m128d, _mm_castpd_ps) - OPENCV_HAL_IMPL_SSE_REINTERPRET_RAW(__m128d, __m128i, _mm_castsi128_pd) - OPENCV_HAL_IMPL_SSE_REINTERPRET_RAW(__m128d, __m128, _mm_castps_pd) - OPENCV_HAL_IMPL_SSE_REINTERPRET_RAW(__m128d, __m128d, OPENCV_HAL_NOP) -} // namespace hal_sse_internal - -#define OPENCV_HAL_IMPL_SSE_INITVEC(_Tpvec, _Tp, suffix, zsuffix, ssuffix, _Tps, cast) \ -inline _Tpvec v_setzero_##suffix() { return _Tpvec(_mm_setzero_##zsuffix()); } \ -inline _Tpvec v_setall_##suffix(_Tp v) { return _Tpvec(_mm_set1_##ssuffix((_Tps)v)); } \ -template inline _Tpvec v_reinterpret_as_##suffix(const _Tpvec0& a) \ -{ return _Tpvec(cast(a.val)); } - -OPENCV_HAL_IMPL_SSE_INITVEC(v_uint8x16, uchar, u8, si128, epi8, char, OPENCV_HAL_NOP) -OPENCV_HAL_IMPL_SSE_INITVEC(v_int8x16, schar, s8, si128, epi8, char, OPENCV_HAL_NOP) -OPENCV_HAL_IMPL_SSE_INITVEC(v_uint16x8, ushort, u16, si128, epi16, short, OPENCV_HAL_NOP) -OPENCV_HAL_IMPL_SSE_INITVEC(v_int16x8, short, s16, si128, epi16, short, OPENCV_HAL_NOP) -OPENCV_HAL_IMPL_SSE_INITVEC(v_uint32x4, unsigned, u32, si128, epi32, int, OPENCV_HAL_NOP) -OPENCV_HAL_IMPL_SSE_INITVEC(v_int32x4, int, s32, si128, epi32, int, OPENCV_HAL_NOP) -OPENCV_HAL_IMPL_SSE_INITVEC(v_float32x4, float, f32, ps, ps, float, _mm_castsi128_ps) -OPENCV_HAL_IMPL_SSE_INITVEC(v_float64x2, double, f64, pd, pd, double, _mm_castsi128_pd) - -inline v_uint64x2 v_setzero_u64() { return v_uint64x2(_mm_setzero_si128()); } -inline v_int64x2 v_setzero_s64() { return v_int64x2(_mm_setzero_si128()); } -inline v_uint64x2 v_setall_u64(uint64 val) { return v_uint64x2(val, val); } -inline v_int64x2 v_setall_s64(int64 val) { return v_int64x2(val, val); } - -template inline -v_uint64x2 v_reinterpret_as_u64(const _Tpvec& a) { return v_uint64x2(a.val); } -template inline -v_int64x2 v_reinterpret_as_s64(const _Tpvec& a) { return v_int64x2(a.val); } -inline v_float32x4 v_reinterpret_as_f32(const v_uint64x2& a) -{ return v_float32x4(_mm_castsi128_ps(a.val)); } -inline v_float32x4 v_reinterpret_as_f32(const v_int64x2& a) -{ return v_float32x4(_mm_castsi128_ps(a.val)); } -inline v_float64x2 v_reinterpret_as_f64(const v_uint64x2& a) -{ return v_float64x2(_mm_castsi128_pd(a.val)); } -inline v_float64x2 v_reinterpret_as_f64(const v_int64x2& a) -{ return v_float64x2(_mm_castsi128_pd(a.val)); } - -#define OPENCV_HAL_IMPL_SSE_INIT_FROM_FLT(_Tpvec, suffix) \ -inline _Tpvec v_reinterpret_as_##suffix(const v_float32x4& a) \ -{ return _Tpvec(_mm_castps_si128(a.val)); } \ -inline _Tpvec v_reinterpret_as_##suffix(const v_float64x2& a) \ -{ return _Tpvec(_mm_castpd_si128(a.val)); } - -OPENCV_HAL_IMPL_SSE_INIT_FROM_FLT(v_uint8x16, u8) -OPENCV_HAL_IMPL_SSE_INIT_FROM_FLT(v_int8x16, s8) -OPENCV_HAL_IMPL_SSE_INIT_FROM_FLT(v_uint16x8, u16) -OPENCV_HAL_IMPL_SSE_INIT_FROM_FLT(v_int16x8, s16) -OPENCV_HAL_IMPL_SSE_INIT_FROM_FLT(v_uint32x4, u32) -OPENCV_HAL_IMPL_SSE_INIT_FROM_FLT(v_int32x4, s32) -OPENCV_HAL_IMPL_SSE_INIT_FROM_FLT(v_uint64x2, u64) -OPENCV_HAL_IMPL_SSE_INIT_FROM_FLT(v_int64x2, s64) - -inline v_float32x4 v_reinterpret_as_f32(const v_float32x4& a) {return a; } -inline v_float64x2 v_reinterpret_as_f64(const v_float64x2& a) {return a; } -inline v_float32x4 v_reinterpret_as_f32(const v_float64x2& a) {return v_float32x4(_mm_castpd_ps(a.val)); } -inline v_float64x2 v_reinterpret_as_f64(const v_float32x4& a) {return v_float64x2(_mm_castps_pd(a.val)); } - -//////////////// PACK /////////////// -static inline v_uint8x16 v_packus(const v_int16x8& a, const v_int16x8& b) { - v_uint8x16 res; - res.val = _mm_packus_epi16(a.val, b.val); - return res; -} - -inline v_uint8x16 v_pack(const v_uint16x8& a, const v_uint16x8& b) -{ - __m128i delta = _mm_set1_epi16(255); - return v_uint8x16(_mm_packus_epi16(_mm_subs_epu16(a.val, _mm_subs_epu16(a.val, delta)), - _mm_subs_epu16(b.val, _mm_subs_epu16(b.val, delta)))); -} - -inline void v_pack_store(uchar* ptr, const v_uint16x8& a) -{ - __m128i delta = _mm_set1_epi16(255); - __m128i a1 = _mm_subs_epu16(a.val, _mm_subs_epu16(a.val, delta)); - _mm_storel_epi64((__m128i*)ptr, _mm_packus_epi16(a1, a1)); -} - -inline v_uint8x16 v_pack_u(const v_int16x8& a, const v_int16x8& b) -{ return v_uint8x16(_mm_packus_epi16(a.val, b.val)); } - -inline void v_pack_u_store(uchar* ptr, const v_int16x8& a) -{ _mm_storel_epi64((__m128i*)ptr, _mm_packus_epi16(a.val, a.val)); } - -template inline -v_uint8x16 v_rshr_pack(const v_uint16x8& a, const v_uint16x8& b) -{ - // we assume that n > 0, and so the shifted 16-bit values can be treated as signed numbers. - __m128i delta = _mm_set1_epi16((short)(1 << (n-1))); - return v_uint8x16(_mm_packus_epi16(_mm_srli_epi16(_mm_adds_epu16(a.val, delta), n), - _mm_srli_epi16(_mm_adds_epu16(b.val, delta), n))); -} - -template inline -void v_rshr_pack_store(uchar* ptr, const v_uint16x8& a) -{ - __m128i delta = _mm_set1_epi16((short)(1 << (n-1))); - __m128i a1 = _mm_srli_epi16(_mm_adds_epu16(a.val, delta), n); - _mm_storel_epi64((__m128i*)ptr, _mm_packus_epi16(a1, a1)); -} - -template inline -v_uint8x16 v_rshr_pack_u(const v_int16x8& a, const v_int16x8& b) -{ - __m128i delta = _mm_set1_epi16((short)(1 << (n-1))); - return v_uint8x16(_mm_packus_epi16(_mm_srai_epi16(_mm_adds_epi16(a.val, delta), n), - _mm_srai_epi16(_mm_adds_epi16(b.val, delta), n))); -} - -template inline -void v_rshr_pack_u_store(uchar* ptr, const v_int16x8& a) -{ - __m128i delta = _mm_set1_epi16((short)(1 << (n-1))); - __m128i a1 = _mm_srai_epi16(_mm_adds_epi16(a.val, delta), n); - _mm_storel_epi64((__m128i*)ptr, _mm_packus_epi16(a1, a1)); -} - -inline v_int8x16 v_pack(const v_int16x8& a, const v_int16x8& b) -{ return v_int8x16(_mm_packs_epi16(a.val, b.val)); } - -inline void v_pack_store(schar* ptr, const v_int16x8& a) -{ _mm_storel_epi64((__m128i*)ptr, _mm_packs_epi16(a.val, a.val)); } - -template inline -v_int8x16 v_rshr_pack(const v_int16x8& a, const v_int16x8& b) -{ - // we assume that n > 0, and so the shifted 16-bit values can be treated as signed numbers. - __m128i delta = _mm_set1_epi16((short)(1 << (n-1))); - return v_int8x16(_mm_packs_epi16(_mm_srai_epi16(_mm_adds_epi16(a.val, delta), n), - _mm_srai_epi16(_mm_adds_epi16(b.val, delta), n))); -} -template inline -void v_rshr_pack_store(schar* ptr, const v_int16x8& a) -{ - // we assume that n > 0, and so the shifted 16-bit values can be treated as signed numbers. - __m128i delta = _mm_set1_epi16((short)(1 << (n-1))); - __m128i a1 = _mm_srai_epi16(_mm_adds_epi16(a.val, delta), n); - _mm_storel_epi64((__m128i*)ptr, _mm_packs_epi16(a1, a1)); -} - - -// byte-wise "mask ? a : b" -inline __m128i v_select_si128(__m128i mask, __m128i a, __m128i b) -{ -#if CV_SSE4_1 - return _mm_blendv_epi8(b, a, mask); -#else - return _mm_xor_si128(b, _mm_and_si128(_mm_xor_si128(a, b), mask)); -#endif -} - -inline v_uint16x8 v_pack(const v_uint32x4& a, const v_uint32x4& b) -{ - __m128i z = _mm_setzero_si128(), maxval32 = _mm_set1_epi32(65535), delta32 = _mm_set1_epi32(32768); - __m128i a1 = _mm_sub_epi32(v_select_si128(_mm_cmpgt_epi32(z, a.val), maxval32, a.val), delta32); - __m128i b1 = _mm_sub_epi32(v_select_si128(_mm_cmpgt_epi32(z, b.val), maxval32, b.val), delta32); - __m128i r = _mm_packs_epi32(a1, b1); - return v_uint16x8(_mm_sub_epi16(r, _mm_set1_epi16(-32768))); -} - -inline void v_pack_store(ushort* ptr, const v_uint32x4& a) -{ - __m128i z = _mm_setzero_si128(), maxval32 = _mm_set1_epi32(65535), delta32 = _mm_set1_epi32(32768); - __m128i a1 = _mm_sub_epi32(v_select_si128(_mm_cmpgt_epi32(z, a.val), maxval32, a.val), delta32); - __m128i r = _mm_packs_epi32(a1, a1); - _mm_storel_epi64((__m128i*)ptr, _mm_sub_epi16(r, _mm_set1_epi16(-32768))); -} - -template inline -v_uint16x8 v_rshr_pack(const v_uint32x4& a, const v_uint32x4& b) -{ - __m128i delta = _mm_set1_epi32(1 << (n-1)), delta32 = _mm_set1_epi32(32768); - __m128i a1 = _mm_sub_epi32(_mm_srli_epi32(_mm_add_epi32(a.val, delta), n), delta32); - __m128i b1 = _mm_sub_epi32(_mm_srli_epi32(_mm_add_epi32(b.val, delta), n), delta32); - return v_uint16x8(_mm_sub_epi16(_mm_packs_epi32(a1, b1), _mm_set1_epi16(-32768))); -} - -template inline -void v_rshr_pack_store(ushort* ptr, const v_uint32x4& a) -{ - __m128i delta = _mm_set1_epi32(1 << (n-1)), delta32 = _mm_set1_epi32(32768); - __m128i a1 = _mm_sub_epi32(_mm_srli_epi32(_mm_add_epi32(a.val, delta), n), delta32); - __m128i a2 = _mm_sub_epi16(_mm_packs_epi32(a1, a1), _mm_set1_epi16(-32768)); - _mm_storel_epi64((__m128i*)ptr, a2); -} - -inline v_uint16x8 v_pack_u(const v_int32x4& a, const v_int32x4& b) -{ - __m128i delta32 = _mm_set1_epi32(32768); - - // preliminary saturate negative values to zero - __m128i a1 = _mm_and_si128(a.val, _mm_cmpgt_epi32(a.val, _mm_set1_epi32(0))); - __m128i b1 = _mm_and_si128(b.val, _mm_cmpgt_epi32(b.val, _mm_set1_epi32(0))); - - __m128i r = _mm_packs_epi32(_mm_sub_epi32(a1, delta32), _mm_sub_epi32(b1, delta32)); - return v_uint16x8(_mm_sub_epi16(r, _mm_set1_epi16(-32768))); -} - -inline void v_pack_u_store(ushort* ptr, const v_int32x4& a) -{ - __m128i delta32 = _mm_set1_epi32(32768); - __m128i a1 = _mm_sub_epi32(a.val, delta32); - __m128i r = _mm_sub_epi16(_mm_packs_epi32(a1, a1), _mm_set1_epi16(-32768)); - _mm_storel_epi64((__m128i*)ptr, r); -} - -template inline -v_uint16x8 v_rshr_pack_u(const v_int32x4& a, const v_int32x4& b) -{ - __m128i delta = _mm_set1_epi32(1 << (n-1)), delta32 = _mm_set1_epi32(32768); - __m128i a1 = _mm_sub_epi32(_mm_srai_epi32(_mm_add_epi32(a.val, delta), n), delta32); - __m128i a2 = _mm_sub_epi16(_mm_packs_epi32(a1, a1), _mm_set1_epi16(-32768)); - __m128i b1 = _mm_sub_epi32(_mm_srai_epi32(_mm_add_epi32(b.val, delta), n), delta32); - __m128i b2 = _mm_sub_epi16(_mm_packs_epi32(b1, b1), _mm_set1_epi16(-32768)); - return v_uint16x8(_mm_unpacklo_epi64(a2, b2)); -} - -template inline -void v_rshr_pack_u_store(ushort* ptr, const v_int32x4& a) -{ - __m128i delta = _mm_set1_epi32(1 << (n-1)), delta32 = _mm_set1_epi32(32768); - __m128i a1 = _mm_sub_epi32(_mm_srai_epi32(_mm_add_epi32(a.val, delta), n), delta32); - __m128i a2 = _mm_sub_epi16(_mm_packs_epi32(a1, a1), _mm_set1_epi16(-32768)); - _mm_storel_epi64((__m128i*)ptr, a2); -} - -inline v_int16x8 v_pack(const v_int32x4& a, const v_int32x4& b) -{ return v_int16x8(_mm_packs_epi32(a.val, b.val)); } - -inline void v_pack_store(short* ptr, const v_int32x4& a) -{ - _mm_storel_epi64((__m128i*)ptr, _mm_packs_epi32(a.val, a.val)); -} - -template inline -v_int16x8 v_rshr_pack(const v_int32x4& a, const v_int32x4& b) -{ - __m128i delta = _mm_set1_epi32(1 << (n-1)); - return v_int16x8(_mm_packs_epi32(_mm_srai_epi32(_mm_add_epi32(a.val, delta), n), - _mm_srai_epi32(_mm_add_epi32(b.val, delta), n))); -} - -template inline -void v_rshr_pack_store(short* ptr, const v_int32x4& a) -{ - __m128i delta = _mm_set1_epi32(1 << (n-1)); - __m128i a1 = _mm_srai_epi32(_mm_add_epi32(a.val, delta), n); - _mm_storel_epi64((__m128i*)ptr, _mm_packs_epi32(a1, a1)); -} - - -// [a0 0 | b0 0] [a1 0 | b1 0] -inline v_uint32x4 v_pack(const v_uint64x2& a, const v_uint64x2& b) -{ - __m128i v0 = _mm_unpacklo_epi32(a.val, b.val); // a0 a1 0 0 - __m128i v1 = _mm_unpackhi_epi32(a.val, b.val); // b0 b1 0 0 - return v_uint32x4(_mm_unpacklo_epi32(v0, v1)); -} - -inline void v_pack_store(unsigned* ptr, const v_uint64x2& a) -{ - __m128i a1 = _mm_shuffle_epi32(a.val, _MM_SHUFFLE(0, 2, 2, 0)); - _mm_storel_epi64((__m128i*)ptr, a1); -} - -// [a0 0 | b0 0] [a1 0 | b1 0] -inline v_int32x4 v_pack(const v_int64x2& a, const v_int64x2& b) -{ - __m128i v0 = _mm_unpacklo_epi32(a.val, b.val); // a0 a1 0 0 - __m128i v1 = _mm_unpackhi_epi32(a.val, b.val); // b0 b1 0 0 - return v_int32x4(_mm_unpacklo_epi32(v0, v1)); -} - -inline void v_pack_store(int* ptr, const v_int64x2& a) -{ - __m128i a1 = _mm_shuffle_epi32(a.val, _MM_SHUFFLE(0, 2, 2, 0)); - _mm_storel_epi64((__m128i*)ptr, a1); -} - -template inline -v_uint32x4 v_rshr_pack(const v_uint64x2& a, const v_uint64x2& b) -{ - uint64 delta = (uint64)1 << (n-1); - v_uint64x2 delta2(delta, delta); - __m128i a1 = _mm_srli_epi64(_mm_add_epi64(a.val, delta2.val), n); - __m128i b1 = _mm_srli_epi64(_mm_add_epi64(b.val, delta2.val), n); - __m128i v0 = _mm_unpacklo_epi32(a1, b1); // a0 a1 0 0 - __m128i v1 = _mm_unpackhi_epi32(a1, b1); // b0 b1 0 0 - return v_uint32x4(_mm_unpacklo_epi32(v0, v1)); -} - -template inline -void v_rshr_pack_store(unsigned* ptr, const v_uint64x2& a) -{ - uint64 delta = (uint64)1 << (n-1); - v_uint64x2 delta2(delta, delta); - __m128i a1 = _mm_srli_epi64(_mm_add_epi64(a.val, delta2.val), n); - __m128i a2 = _mm_shuffle_epi32(a1, _MM_SHUFFLE(0, 2, 2, 0)); - _mm_storel_epi64((__m128i*)ptr, a2); -} - -inline __m128i v_sign_epi64(__m128i a) -{ - return _mm_shuffle_epi32(_mm_srai_epi32(a, 31), _MM_SHUFFLE(3, 3, 1, 1)); // x m0 | x m1 -} - -inline __m128i v_srai_epi64(__m128i a, int imm) -{ - __m128i smask = v_sign_epi64(a); - return _mm_xor_si128(_mm_srli_epi64(_mm_xor_si128(a, smask), imm), smask); -} - -template inline -v_int32x4 v_rshr_pack(const v_int64x2& a, const v_int64x2& b) -{ - int64 delta = (int64)1 << (n-1); - v_int64x2 delta2(delta, delta); - __m128i a1 = v_srai_epi64(_mm_add_epi64(a.val, delta2.val), n); - __m128i b1 = v_srai_epi64(_mm_add_epi64(b.val, delta2.val), n); - __m128i v0 = _mm_unpacklo_epi32(a1, b1); // a0 a1 0 0 - __m128i v1 = _mm_unpackhi_epi32(a1, b1); // b0 b1 0 0 - return v_int32x4(_mm_unpacklo_epi32(v0, v1)); -} - -template inline -void v_rshr_pack_store(int* ptr, const v_int64x2& a) -{ - int64 delta = (int64)1 << (n-1); - v_int64x2 delta2(delta, delta); - __m128i a1 = v_srai_epi64(_mm_add_epi64(a.val, delta2.val), n); - __m128i a2 = _mm_shuffle_epi32(a1, _MM_SHUFFLE(0, 2, 2, 0)); - _mm_storel_epi64((__m128i*)ptr, a2); -} - -inline v_float32x4 v_matmul(const v_float32x4& v, const v_float32x4& m0, - const v_float32x4& m1, const v_float32x4& m2, - const v_float32x4& m3) -{ - __m128 v0 = _mm_mul_ps(_mm_shuffle_ps(v.val, v.val, _MM_SHUFFLE(0, 0, 0, 0)), m0.val); - __m128 v1 = _mm_mul_ps(_mm_shuffle_ps(v.val, v.val, _MM_SHUFFLE(1, 1, 1, 1)), m1.val); - __m128 v2 = _mm_mul_ps(_mm_shuffle_ps(v.val, v.val, _MM_SHUFFLE(2, 2, 2, 2)), m2.val); - __m128 v3 = _mm_mul_ps(_mm_shuffle_ps(v.val, v.val, _MM_SHUFFLE(3, 3, 3, 3)), m3.val); - - return v_float32x4(_mm_add_ps(_mm_add_ps(v0, v1), _mm_add_ps(v2, v3))); -} - -inline v_float32x4 v_matmuladd(const v_float32x4& v, const v_float32x4& m0, - const v_float32x4& m1, const v_float32x4& m2, - const v_float32x4& a) -{ - __m128 v0 = _mm_mul_ps(_mm_shuffle_ps(v.val, v.val, _MM_SHUFFLE(0, 0, 0, 0)), m0.val); - __m128 v1 = _mm_mul_ps(_mm_shuffle_ps(v.val, v.val, _MM_SHUFFLE(1, 1, 1, 1)), m1.val); - __m128 v2 = _mm_mul_ps(_mm_shuffle_ps(v.val, v.val, _MM_SHUFFLE(2, 2, 2, 2)), m2.val); - - return v_float32x4(_mm_add_ps(_mm_add_ps(v0, v1), _mm_add_ps(v2, a.val))); -} - -#define OPENCV_HAL_IMPL_SSE_BIN_OP(bin_op, _Tpvec, intrin) \ - inline _Tpvec operator bin_op (const _Tpvec& a, const _Tpvec& b) \ - { \ - return _Tpvec(intrin(a.val, b.val)); \ - } \ - inline _Tpvec& operator bin_op##= (_Tpvec& a, const _Tpvec& b) \ - { \ - a.val = intrin(a.val, b.val); \ - return a; \ - } - -OPENCV_HAL_IMPL_SSE_BIN_OP(+, v_uint8x16, _mm_adds_epu8) -OPENCV_HAL_IMPL_SSE_BIN_OP(-, v_uint8x16, _mm_subs_epu8) -OPENCV_HAL_IMPL_SSE_BIN_OP(+, v_int8x16, _mm_adds_epi8) -OPENCV_HAL_IMPL_SSE_BIN_OP(-, v_int8x16, _mm_subs_epi8) -OPENCV_HAL_IMPL_SSE_BIN_OP(+, v_uint16x8, _mm_adds_epu16) -OPENCV_HAL_IMPL_SSE_BIN_OP(-, v_uint16x8, _mm_subs_epu16) -OPENCV_HAL_IMPL_SSE_BIN_OP(*, v_uint16x8, _mm_mullo_epi16) -OPENCV_HAL_IMPL_SSE_BIN_OP(+, v_int16x8, _mm_adds_epi16) -OPENCV_HAL_IMPL_SSE_BIN_OP(-, v_int16x8, _mm_subs_epi16) -OPENCV_HAL_IMPL_SSE_BIN_OP(*, v_int16x8, _mm_mullo_epi16) -OPENCV_HAL_IMPL_SSE_BIN_OP(+, v_uint32x4, _mm_add_epi32) -OPENCV_HAL_IMPL_SSE_BIN_OP(-, v_uint32x4, _mm_sub_epi32) -OPENCV_HAL_IMPL_SSE_BIN_OP(+, v_int32x4, _mm_add_epi32) -OPENCV_HAL_IMPL_SSE_BIN_OP(-, v_int32x4, _mm_sub_epi32) -OPENCV_HAL_IMPL_SSE_BIN_OP(+, v_float32x4, _mm_add_ps) -OPENCV_HAL_IMPL_SSE_BIN_OP(-, v_float32x4, _mm_sub_ps) -OPENCV_HAL_IMPL_SSE_BIN_OP(*, v_float32x4, _mm_mul_ps) -OPENCV_HAL_IMPL_SSE_BIN_OP(/, v_float32x4, _mm_div_ps) -OPENCV_HAL_IMPL_SSE_BIN_OP(+, v_float64x2, _mm_add_pd) -OPENCV_HAL_IMPL_SSE_BIN_OP(-, v_float64x2, _mm_sub_pd) -OPENCV_HAL_IMPL_SSE_BIN_OP(*, v_float64x2, _mm_mul_pd) -OPENCV_HAL_IMPL_SSE_BIN_OP(/, v_float64x2, _mm_div_pd) -OPENCV_HAL_IMPL_SSE_BIN_OP(+, v_uint64x2, _mm_add_epi64) -OPENCV_HAL_IMPL_SSE_BIN_OP(-, v_uint64x2, _mm_sub_epi64) -OPENCV_HAL_IMPL_SSE_BIN_OP(+, v_int64x2, _mm_add_epi64) -OPENCV_HAL_IMPL_SSE_BIN_OP(-, v_int64x2, _mm_sub_epi64) - -inline v_uint32x4 operator * (const v_uint32x4& a, const v_uint32x4& b) -{ - __m128i c0 = _mm_mul_epu32(a.val, b.val); - __m128i c1 = _mm_mul_epu32(_mm_srli_epi64(a.val, 32), _mm_srli_epi64(b.val, 32)); - __m128i d0 = _mm_unpacklo_epi32(c0, c1); - __m128i d1 = _mm_unpackhi_epi32(c0, c1); - return v_uint32x4(_mm_unpacklo_epi64(d0, d1)); -} -inline v_int32x4 operator * (const v_int32x4& a, const v_int32x4& b) -{ -#if CV_SSE4_1 - return v_int32x4(_mm_mullo_epi32(a.val, b.val)); -#else - __m128i c0 = _mm_mul_epu32(a.val, b.val); - __m128i c1 = _mm_mul_epu32(_mm_srli_epi64(a.val, 32), _mm_srli_epi64(b.val, 32)); - __m128i d0 = _mm_unpacklo_epi32(c0, c1); - __m128i d1 = _mm_unpackhi_epi32(c0, c1); - return v_int32x4(_mm_unpacklo_epi64(d0, d1)); -#endif -} -inline v_uint32x4& operator *= (v_uint32x4& a, const v_uint32x4& b) -{ - a = a * b; - return a; -} -inline v_int32x4& operator *= (v_int32x4& a, const v_int32x4& b) -{ - a = a * b; - return a; -} - -inline void v_mul_expand(const v_int16x8& a, const v_int16x8& b, - v_int32x4& c, v_int32x4& d) -{ - __m128i v0 = _mm_mullo_epi16(a.val, b.val); - __m128i v1 = _mm_mulhi_epi16(a.val, b.val); - c.val = _mm_unpacklo_epi16(v0, v1); - d.val = _mm_unpackhi_epi16(v0, v1); -} - -inline void v_mul_expand(const v_uint16x8& a, const v_uint16x8& b, - v_uint32x4& c, v_uint32x4& d) -{ - __m128i v0 = _mm_mullo_epi16(a.val, b.val); - __m128i v1 = _mm_mulhi_epu16(a.val, b.val); - c.val = _mm_unpacklo_epi16(v0, v1); - d.val = _mm_unpackhi_epi16(v0, v1); -} - -inline void v_mul_expand(const v_uint32x4& a, const v_uint32x4& b, - v_uint64x2& c, v_uint64x2& d) -{ - __m128i c0 = _mm_mul_epu32(a.val, b.val); - __m128i c1 = _mm_mul_epu32(_mm_srli_epi64(a.val, 32), _mm_srli_epi64(b.val, 32)); - c.val = _mm_unpacklo_epi64(c0, c1); - d.val = _mm_unpackhi_epi64(c0, c1); -} - -inline v_int16x8 v_mul_hi(const v_int16x8& a, const v_int16x8& b) { return v_int16x8(_mm_mulhi_epi16(a.val, b.val)); } -inline v_uint16x8 v_mul_hi(const v_uint16x8& a, const v_uint16x8& b) { return v_uint16x8(_mm_mulhi_epu16(a.val, b.val)); } - -inline v_int32x4 v_dotprod(const v_int16x8& a, const v_int16x8& b) -{ - return v_int32x4(_mm_madd_epi16(a.val, b.val)); -} - -inline v_int32x4 v_dotprod(const v_int16x8& a, const v_int16x8& b, const v_int32x4& c) -{ - return v_int32x4(_mm_add_epi32(_mm_madd_epi16(a.val, b.val), c.val)); -} - -#define OPENCV_HAL_IMPL_SSE_LOGIC_OP(_Tpvec, suffix, not_const) \ - OPENCV_HAL_IMPL_SSE_BIN_OP(&, _Tpvec, _mm_and_##suffix) \ - OPENCV_HAL_IMPL_SSE_BIN_OP(|, _Tpvec, _mm_or_##suffix) \ - OPENCV_HAL_IMPL_SSE_BIN_OP(^, _Tpvec, _mm_xor_##suffix) \ - inline _Tpvec operator ~ (const _Tpvec& a) \ - { \ - return _Tpvec(_mm_xor_##suffix(a.val, not_const)); \ - } - -OPENCV_HAL_IMPL_SSE_LOGIC_OP(v_uint8x16, si128, _mm_set1_epi32(-1)) -OPENCV_HAL_IMPL_SSE_LOGIC_OP(v_int8x16, si128, _mm_set1_epi32(-1)) -OPENCV_HAL_IMPL_SSE_LOGIC_OP(v_uint16x8, si128, _mm_set1_epi32(-1)) -OPENCV_HAL_IMPL_SSE_LOGIC_OP(v_int16x8, si128, _mm_set1_epi32(-1)) -OPENCV_HAL_IMPL_SSE_LOGIC_OP(v_uint32x4, si128, _mm_set1_epi32(-1)) -OPENCV_HAL_IMPL_SSE_LOGIC_OP(v_int32x4, si128, _mm_set1_epi32(-1)) -OPENCV_HAL_IMPL_SSE_LOGIC_OP(v_uint64x2, si128, _mm_set1_epi32(-1)) -OPENCV_HAL_IMPL_SSE_LOGIC_OP(v_int64x2, si128, _mm_set1_epi32(-1)) -OPENCV_HAL_IMPL_SSE_LOGIC_OP(v_float32x4, ps, _mm_castsi128_ps(_mm_set1_epi32(-1))) -OPENCV_HAL_IMPL_SSE_LOGIC_OP(v_float64x2, pd, _mm_castsi128_pd(_mm_set1_epi32(-1))) - -inline v_float32x4 v_sqrt(const v_float32x4& x) -{ return v_float32x4(_mm_sqrt_ps(x.val)); } - -inline v_float32x4 v_invsqrt(const v_float32x4& x) -{ - const __m128 _0_5 = _mm_set1_ps(0.5f), _1_5 = _mm_set1_ps(1.5f); - __m128 t = x.val; - __m128 h = _mm_mul_ps(t, _0_5); - t = _mm_rsqrt_ps(t); - t = _mm_mul_ps(t, _mm_sub_ps(_1_5, _mm_mul_ps(_mm_mul_ps(t, t), h))); - return v_float32x4(t); -} - -inline v_float64x2 v_sqrt(const v_float64x2& x) -{ return v_float64x2(_mm_sqrt_pd(x.val)); } - -inline v_float64x2 v_invsqrt(const v_float64x2& x) -{ - const __m128d v_1 = _mm_set1_pd(1.); - return v_float64x2(_mm_div_pd(v_1, _mm_sqrt_pd(x.val))); -} - -#define OPENCV_HAL_IMPL_SSE_ABS_INT_FUNC(_Tpuvec, _Tpsvec, func, suffix, subWidth) \ -inline _Tpuvec v_abs(const _Tpsvec& x) \ -{ return _Tpuvec(_mm_##func##_ep##suffix(x.val, _mm_sub_ep##subWidth(_mm_setzero_si128(), x.val))); } - -OPENCV_HAL_IMPL_SSE_ABS_INT_FUNC(v_uint8x16, v_int8x16, min, u8, i8) -OPENCV_HAL_IMPL_SSE_ABS_INT_FUNC(v_uint16x8, v_int16x8, max, i16, i16) -inline v_uint32x4 v_abs(const v_int32x4& x) -{ - __m128i s = _mm_srli_epi32(x.val, 31); - __m128i f = _mm_srai_epi32(x.val, 31); - return v_uint32x4(_mm_add_epi32(_mm_xor_si128(x.val, f), s)); -} -inline v_float32x4 v_abs(const v_float32x4& x) -{ return v_float32x4(_mm_and_ps(x.val, _mm_castsi128_ps(_mm_set1_epi32(0x7fffffff)))); } -inline v_float64x2 v_abs(const v_float64x2& x) -{ - return v_float64x2(_mm_and_pd(x.val, - _mm_castsi128_pd(_mm_srli_epi64(_mm_set1_epi32(-1), 1)))); -} - -// TODO: exp, log, sin, cos - -#define OPENCV_HAL_IMPL_SSE_BIN_FUNC(_Tpvec, func, intrin) \ -inline _Tpvec func(const _Tpvec& a, const _Tpvec& b) \ -{ \ - return _Tpvec(intrin(a.val, b.val)); \ -} - -OPENCV_HAL_IMPL_SSE_BIN_FUNC(v_uint8x16, v_min, _mm_min_epu8) -OPENCV_HAL_IMPL_SSE_BIN_FUNC(v_uint8x16, v_max, _mm_max_epu8) -OPENCV_HAL_IMPL_SSE_BIN_FUNC(v_int16x8, v_min, _mm_min_epi16) -OPENCV_HAL_IMPL_SSE_BIN_FUNC(v_int16x8, v_max, _mm_max_epi16) -OPENCV_HAL_IMPL_SSE_BIN_FUNC(v_float32x4, v_min, _mm_min_ps) -OPENCV_HAL_IMPL_SSE_BIN_FUNC(v_float32x4, v_max, _mm_max_ps) -OPENCV_HAL_IMPL_SSE_BIN_FUNC(v_float64x2, v_min, _mm_min_pd) -OPENCV_HAL_IMPL_SSE_BIN_FUNC(v_float64x2, v_max, _mm_max_pd) - -inline v_int8x16 v_min(const v_int8x16& a, const v_int8x16& b) -{ -#if CV_SSE4_1 - return v_int8x16(_mm_min_epi8(a.val, b.val)); -#else - __m128i delta = _mm_set1_epi8((char)-128); - return v_int8x16(_mm_xor_si128(delta, _mm_min_epu8(_mm_xor_si128(a.val, delta), - _mm_xor_si128(b.val, delta)))); -#endif -} -inline v_int8x16 v_max(const v_int8x16& a, const v_int8x16& b) -{ -#if CV_SSE4_1 - return v_int8x16(_mm_max_epi8(a.val, b.val)); -#else - __m128i delta = _mm_set1_epi8((char)-128); - return v_int8x16(_mm_xor_si128(delta, _mm_max_epu8(_mm_xor_si128(a.val, delta), - _mm_xor_si128(b.val, delta)))); -#endif -} -inline v_uint16x8 v_min(const v_uint16x8& a, const v_uint16x8& b) -{ -#if CV_SSE4_1 - return v_uint16x8(_mm_min_epu16(a.val, b.val)); -#else - return v_uint16x8(_mm_subs_epu16(a.val, _mm_subs_epu16(a.val, b.val))); -#endif -} -inline v_uint16x8 v_max(const v_uint16x8& a, const v_uint16x8& b) -{ -#if CV_SSE4_1 - return v_uint16x8(_mm_max_epu16(a.val, b.val)); -#else - return v_uint16x8(_mm_adds_epu16(_mm_subs_epu16(a.val, b.val), b.val)); -#endif -} -inline v_uint32x4 v_min(const v_uint32x4& a, const v_uint32x4& b) -{ -#if CV_SSE4_1 - return v_uint32x4(_mm_min_epu32(a.val, b.val)); -#else - __m128i delta = _mm_set1_epi32((int)0x80000000); - __m128i mask = _mm_cmpgt_epi32(_mm_xor_si128(a.val, delta), _mm_xor_si128(b.val, delta)); - return v_uint32x4(v_select_si128(mask, b.val, a.val)); -#endif -} -inline v_uint32x4 v_max(const v_uint32x4& a, const v_uint32x4& b) -{ -#if CV_SSE4_1 - return v_uint32x4(_mm_max_epu32(a.val, b.val)); -#else - __m128i delta = _mm_set1_epi32((int)0x80000000); - __m128i mask = _mm_cmpgt_epi32(_mm_xor_si128(a.val, delta), _mm_xor_si128(b.val, delta)); - return v_uint32x4(v_select_si128(mask, a.val, b.val)); -#endif -} -inline v_int32x4 v_min(const v_int32x4& a, const v_int32x4& b) -{ -#if CV_SSE4_1 - return v_int32x4(_mm_min_epi32(a.val, b.val)); -#else - return v_int32x4(v_select_si128(_mm_cmpgt_epi32(a.val, b.val), b.val, a.val)); -#endif -} -inline v_int32x4 v_max(const v_int32x4& a, const v_int32x4& b) -{ -#if CV_SSE4_1 - return v_int32x4(_mm_max_epi32(a.val, b.val)); -#else - return v_int32x4(v_select_si128(_mm_cmpgt_epi32(a.val, b.val), a.val, b.val)); -#endif -} - -#define OPENCV_HAL_IMPL_SSE_INT_CMP_OP(_Tpuvec, _Tpsvec, suffix, sbit) \ -inline _Tpuvec operator == (const _Tpuvec& a, const _Tpuvec& b) \ -{ return _Tpuvec(_mm_cmpeq_##suffix(a.val, b.val)); } \ -inline _Tpuvec operator != (const _Tpuvec& a, const _Tpuvec& b) \ -{ \ - __m128i not_mask = _mm_set1_epi32(-1); \ - return _Tpuvec(_mm_xor_si128(_mm_cmpeq_##suffix(a.val, b.val), not_mask)); \ -} \ -inline _Tpsvec operator == (const _Tpsvec& a, const _Tpsvec& b) \ -{ return _Tpsvec(_mm_cmpeq_##suffix(a.val, b.val)); } \ -inline _Tpsvec operator != (const _Tpsvec& a, const _Tpsvec& b) \ -{ \ - __m128i not_mask = _mm_set1_epi32(-1); \ - return _Tpsvec(_mm_xor_si128(_mm_cmpeq_##suffix(a.val, b.val), not_mask)); \ -} \ -inline _Tpuvec operator < (const _Tpuvec& a, const _Tpuvec& b) \ -{ \ - __m128i smask = _mm_set1_##suffix(sbit); \ - return _Tpuvec(_mm_cmpgt_##suffix(_mm_xor_si128(b.val, smask), _mm_xor_si128(a.val, smask))); \ -} \ -inline _Tpuvec operator > (const _Tpuvec& a, const _Tpuvec& b) \ -{ \ - __m128i smask = _mm_set1_##suffix(sbit); \ - return _Tpuvec(_mm_cmpgt_##suffix(_mm_xor_si128(a.val, smask), _mm_xor_si128(b.val, smask))); \ -} \ -inline _Tpuvec operator <= (const _Tpuvec& a, const _Tpuvec& b) \ -{ \ - __m128i smask = _mm_set1_##suffix(sbit); \ - __m128i not_mask = _mm_set1_epi32(-1); \ - __m128i res = _mm_cmpgt_##suffix(_mm_xor_si128(a.val, smask), _mm_xor_si128(b.val, smask)); \ - return _Tpuvec(_mm_xor_si128(res, not_mask)); \ -} \ -inline _Tpuvec operator >= (const _Tpuvec& a, const _Tpuvec& b) \ -{ \ - __m128i smask = _mm_set1_##suffix(sbit); \ - __m128i not_mask = _mm_set1_epi32(-1); \ - __m128i res = _mm_cmpgt_##suffix(_mm_xor_si128(b.val, smask), _mm_xor_si128(a.val, smask)); \ - return _Tpuvec(_mm_xor_si128(res, not_mask)); \ -} \ -inline _Tpsvec operator < (const _Tpsvec& a, const _Tpsvec& b) \ -{ \ - return _Tpsvec(_mm_cmpgt_##suffix(b.val, a.val)); \ -} \ -inline _Tpsvec operator > (const _Tpsvec& a, const _Tpsvec& b) \ -{ \ - return _Tpsvec(_mm_cmpgt_##suffix(a.val, b.val)); \ -} \ -inline _Tpsvec operator <= (const _Tpsvec& a, const _Tpsvec& b) \ -{ \ - __m128i not_mask = _mm_set1_epi32(-1); \ - return _Tpsvec(_mm_xor_si128(_mm_cmpgt_##suffix(a.val, b.val), not_mask)); \ -} \ -inline _Tpsvec operator >= (const _Tpsvec& a, const _Tpsvec& b) \ -{ \ - __m128i not_mask = _mm_set1_epi32(-1); \ - return _Tpsvec(_mm_xor_si128(_mm_cmpgt_##suffix(b.val, a.val), not_mask)); \ -} - -OPENCV_HAL_IMPL_SSE_INT_CMP_OP(v_uint8x16, v_int8x16, epi8, (char)-128) -OPENCV_HAL_IMPL_SSE_INT_CMP_OP(v_uint16x8, v_int16x8, epi16, (short)-32768) -OPENCV_HAL_IMPL_SSE_INT_CMP_OP(v_uint32x4, v_int32x4, epi32, (int)0x80000000) - -#define OPENCV_HAL_IMPL_SSE_FLT_CMP_OP(_Tpvec, suffix) \ -inline _Tpvec operator == (const _Tpvec& a, const _Tpvec& b) \ -{ return _Tpvec(_mm_cmpeq_##suffix(a.val, b.val)); } \ -inline _Tpvec operator != (const _Tpvec& a, const _Tpvec& b) \ -{ return _Tpvec(_mm_cmpneq_##suffix(a.val, b.val)); } \ -inline _Tpvec operator < (const _Tpvec& a, const _Tpvec& b) \ -{ return _Tpvec(_mm_cmplt_##suffix(a.val, b.val)); } \ -inline _Tpvec operator > (const _Tpvec& a, const _Tpvec& b) \ -{ return _Tpvec(_mm_cmpgt_##suffix(a.val, b.val)); } \ -inline _Tpvec operator <= (const _Tpvec& a, const _Tpvec& b) \ -{ return _Tpvec(_mm_cmple_##suffix(a.val, b.val)); } \ -inline _Tpvec operator >= (const _Tpvec& a, const _Tpvec& b) \ -{ return _Tpvec(_mm_cmpge_##suffix(a.val, b.val)); } - -OPENCV_HAL_IMPL_SSE_FLT_CMP_OP(v_float32x4, ps) -OPENCV_HAL_IMPL_SSE_FLT_CMP_OP(v_float64x2, pd) - -#define OPENCV_HAL_IMPL_SSE_64BIT_CMP_OP(_Tpvec, cast) \ -inline _Tpvec operator == (const _Tpvec& a, const _Tpvec& b) \ -{ return cast(v_reinterpret_as_f64(a) == v_reinterpret_as_f64(b)); } \ -inline _Tpvec operator != (const _Tpvec& a, const _Tpvec& b) \ -{ return cast(v_reinterpret_as_f64(a) != v_reinterpret_as_f64(b)); } - -OPENCV_HAL_IMPL_SSE_64BIT_CMP_OP(v_uint64x2, v_reinterpret_as_u64) -OPENCV_HAL_IMPL_SSE_64BIT_CMP_OP(v_int64x2, v_reinterpret_as_s64) - -OPENCV_HAL_IMPL_SSE_BIN_FUNC(v_uint8x16, v_add_wrap, _mm_add_epi8) -OPENCV_HAL_IMPL_SSE_BIN_FUNC(v_int8x16, v_add_wrap, _mm_add_epi8) -OPENCV_HAL_IMPL_SSE_BIN_FUNC(v_uint16x8, v_add_wrap, _mm_add_epi16) -OPENCV_HAL_IMPL_SSE_BIN_FUNC(v_int16x8, v_add_wrap, _mm_add_epi16) -OPENCV_HAL_IMPL_SSE_BIN_FUNC(v_uint8x16, v_sub_wrap, _mm_sub_epi8) -OPENCV_HAL_IMPL_SSE_BIN_FUNC(v_int8x16, v_sub_wrap, _mm_sub_epi8) -OPENCV_HAL_IMPL_SSE_BIN_FUNC(v_uint16x8, v_sub_wrap, _mm_sub_epi16) -OPENCV_HAL_IMPL_SSE_BIN_FUNC(v_int16x8, v_sub_wrap, _mm_sub_epi16) - -#define OPENCV_HAL_IMPL_SSE_ABSDIFF_8_16(_Tpuvec, _Tpsvec, bits, smask32) \ -inline _Tpuvec v_absdiff(const _Tpuvec& a, const _Tpuvec& b) \ -{ \ - return _Tpuvec(_mm_add_epi##bits(_mm_subs_epu##bits(a.val, b.val), _mm_subs_epu##bits(b.val, a.val))); \ -} \ -inline _Tpuvec v_absdiff(const _Tpsvec& a, const _Tpsvec& b) \ -{ \ - __m128i smask = _mm_set1_epi32(smask32); \ - __m128i a1 = _mm_xor_si128(a.val, smask); \ - __m128i b1 = _mm_xor_si128(b.val, smask); \ - return _Tpuvec(_mm_add_epi##bits(_mm_subs_epu##bits(a1, b1), _mm_subs_epu##bits(b1, a1))); \ -} - -OPENCV_HAL_IMPL_SSE_ABSDIFF_8_16(v_uint8x16, v_int8x16, 8, (int)0x80808080) -OPENCV_HAL_IMPL_SSE_ABSDIFF_8_16(v_uint16x8, v_int16x8, 16, (int)0x80008000) - -inline v_uint32x4 v_absdiff(const v_uint32x4& a, const v_uint32x4& b) -{ - return v_max(a, b) - v_min(a, b); -} - -inline v_uint32x4 v_absdiff(const v_int32x4& a, const v_int32x4& b) -{ - __m128i d = _mm_sub_epi32(a.val, b.val); - __m128i m = _mm_cmpgt_epi32(b.val, a.val); - return v_uint32x4(_mm_sub_epi32(_mm_xor_si128(d, m), m)); -} - -inline v_int32x4 v_fma(const v_int32x4& a, const v_int32x4& b, const v_int32x4& c) -{ - return a * b + c; -} - -inline v_int32x4 v_muladd(const v_int32x4& a, const v_int32x4& b, const v_int32x4& c) -{ - return v_fma(a, b, c); -} - -inline v_float32x4 v_fma(const v_float32x4& a, const v_float32x4& b, const v_float32x4& c) -{ -#if CV_FMA3 - return v_float32x4(_mm_fmadd_ps(a.val, b.val, c.val)); -#else - return v_float32x4(_mm_add_ps(_mm_mul_ps(a.val, b.val), c.val)); -#endif -} - -inline v_float64x2 v_fma(const v_float64x2& a, const v_float64x2& b, const v_float64x2& c) -{ -#if CV_FMA3 - return v_float64x2(_mm_fmadd_pd(a.val, b.val, c.val)); -#else - return v_float64x2(_mm_add_pd(_mm_mul_pd(a.val, b.val), c.val)); -#endif -} - -#define OPENCV_HAL_IMPL_SSE_MISC_FLT_OP(_Tpvec, _Tp, _Tpreg, suffix, absmask_vec) \ -inline _Tpvec v_absdiff(const _Tpvec& a, const _Tpvec& b) \ -{ \ - _Tpreg absmask = _mm_castsi128_##suffix(absmask_vec); \ - return _Tpvec(_mm_and_##suffix(_mm_sub_##suffix(a.val, b.val), absmask)); \ -} \ -inline _Tpvec v_magnitude(const _Tpvec& a, const _Tpvec& b) \ -{ \ - _Tpvec res = v_fma(a, a, b*b); \ - return _Tpvec(_mm_sqrt_##suffix(res.val)); \ -} \ -inline _Tpvec v_sqr_magnitude(const _Tpvec& a, const _Tpvec& b) \ -{ \ - return v_fma(a, a, b*b); \ -} \ -inline _Tpvec v_muladd(const _Tpvec& a, const _Tpvec& b, const _Tpvec& c) \ -{ \ - return v_fma(a, b, c); \ -} - -OPENCV_HAL_IMPL_SSE_MISC_FLT_OP(v_float32x4, float, __m128, ps, _mm_set1_epi32((int)0x7fffffff)) -OPENCV_HAL_IMPL_SSE_MISC_FLT_OP(v_float64x2, double, __m128d, pd, _mm_srli_epi64(_mm_set1_epi32(-1), 1)) - -#define OPENCV_HAL_IMPL_SSE_SHIFT_OP(_Tpuvec, _Tpsvec, suffix, srai) \ -inline _Tpuvec operator << (const _Tpuvec& a, int imm) \ -{ \ - return _Tpuvec(_mm_slli_##suffix(a.val, imm)); \ -} \ -inline _Tpsvec operator << (const _Tpsvec& a, int imm) \ -{ \ - return _Tpsvec(_mm_slli_##suffix(a.val, imm)); \ -} \ -inline _Tpuvec operator >> (const _Tpuvec& a, int imm) \ -{ \ - return _Tpuvec(_mm_srli_##suffix(a.val, imm)); \ -} \ -inline _Tpsvec operator >> (const _Tpsvec& a, int imm) \ -{ \ - return _Tpsvec(srai(a.val, imm)); \ -} \ -template \ -inline _Tpuvec v_shl(const _Tpuvec& a) \ -{ \ - return _Tpuvec(_mm_slli_##suffix(a.val, imm)); \ -} \ -template \ -inline _Tpsvec v_shl(const _Tpsvec& a) \ -{ \ - return _Tpsvec(_mm_slli_##suffix(a.val, imm)); \ -} \ -template \ -inline _Tpuvec v_shr(const _Tpuvec& a) \ -{ \ - return _Tpuvec(_mm_srli_##suffix(a.val, imm)); \ -} \ -template \ -inline _Tpsvec v_shr(const _Tpsvec& a) \ -{ \ - return _Tpsvec(srai(a.val, imm)); \ -} - -OPENCV_HAL_IMPL_SSE_SHIFT_OP(v_uint16x8, v_int16x8, epi16, _mm_srai_epi16) -OPENCV_HAL_IMPL_SSE_SHIFT_OP(v_uint32x4, v_int32x4, epi32, _mm_srai_epi32) -OPENCV_HAL_IMPL_SSE_SHIFT_OP(v_uint64x2, v_int64x2, epi64, v_srai_epi64) - -namespace hal_sse_internal -{ - template 16)), - bool is_first = (imm == 0), - bool is_half = (imm == 8), - bool is_second = (imm == 16), - bool is_other = (((imm > 0) && (imm < 8)) || ((imm > 8) && (imm < 16)))> - class v_sse_palignr_u8_class; - - template - class v_sse_palignr_u8_class; - - template - class v_sse_palignr_u8_class - { - public: - inline __m128i operator()(const __m128i& a, const __m128i&) const - { - return a; - } - }; - - template - class v_sse_palignr_u8_class - { - public: - inline __m128i operator()(const __m128i& a, const __m128i& b) const - { - return _mm_unpacklo_epi64(_mm_unpackhi_epi64(a, a), b); - } - }; - - template - class v_sse_palignr_u8_class - { - public: - inline __m128i operator()(const __m128i&, const __m128i& b) const - { - return b; - } - }; - - template - class v_sse_palignr_u8_class - { -#if CV_SSSE3 - public: - inline __m128i operator()(const __m128i& a, const __m128i& b) const - { - return _mm_alignr_epi8(b, a, imm); - } -#else - public: - inline __m128i operator()(const __m128i& a, const __m128i& b) const - { - enum { imm2 = (sizeof(__m128i) - imm) }; - return _mm_or_si128(_mm_srli_si128(a, imm), _mm_slli_si128(b, imm2)); - } -#endif - }; - - template - inline __m128i v_sse_palignr_u8(const __m128i& a, const __m128i& b) - { - CV_StaticAssert((imm >= 0) && (imm <= 16), "Invalid imm for v_sse_palignr_u8."); - return v_sse_palignr_u8_class()(a, b); - } -} // namespace hal_sse_internal - -template -inline _Tpvec v_rotate_right(const _Tpvec &a) -{ - using namespace hal_sse_internal; - enum { imm2 = (imm * sizeof(typename _Tpvec::lane_type)) }; - return _Tpvec(v_sse_reinterpret_as( - _mm_srli_si128( - v_sse_reinterpret_as<__m128i>(a.val), imm2))); -} - -template -inline _Tpvec v_rotate_left(const _Tpvec &a) -{ - using namespace hal_sse_internal; - enum { imm2 = (imm * sizeof(typename _Tpvec::lane_type)) }; - return _Tpvec(v_sse_reinterpret_as( - _mm_slli_si128( - v_sse_reinterpret_as<__m128i>(a.val), imm2))); -} - -template -inline _Tpvec v_rotate_right(const _Tpvec &a, const _Tpvec &b) -{ - using namespace hal_sse_internal; - enum { imm2 = (imm * sizeof(typename _Tpvec::lane_type)) }; - return _Tpvec(v_sse_reinterpret_as( - v_sse_palignr_u8( - v_sse_reinterpret_as<__m128i>(a.val), - v_sse_reinterpret_as<__m128i>(b.val)))); -} - -template -inline _Tpvec v_rotate_left(const _Tpvec &a, const _Tpvec &b) -{ - using namespace hal_sse_internal; - enum { imm2 = ((_Tpvec::nlanes - imm) * sizeof(typename _Tpvec::lane_type)) }; - return _Tpvec(v_sse_reinterpret_as( - v_sse_palignr_u8( - v_sse_reinterpret_as<__m128i>(b.val), - v_sse_reinterpret_as<__m128i>(a.val)))); -} - -#define OPENCV_HAL_IMPL_SSE_LOADSTORE_INT_OP(_Tpvec, _Tp) \ -inline _Tpvec v_load(const _Tp* ptr) \ -{ return _Tpvec(_mm_loadu_si128((const __m128i*)ptr)); } \ -inline _Tpvec v_load_aligned(const _Tp* ptr) \ -{ return _Tpvec(_mm_load_si128((const __m128i*)ptr)); } \ -inline _Tpvec v_load_low(const _Tp* ptr) \ -{ return _Tpvec(_mm_loadl_epi64((const __m128i*)ptr)); } \ -inline _Tpvec v_load_halves(const _Tp* ptr0, const _Tp* ptr1) \ -{ \ - return _Tpvec(_mm_unpacklo_epi64(_mm_loadl_epi64((const __m128i*)ptr0), \ - _mm_loadl_epi64((const __m128i*)ptr1))); \ -} \ -inline void v_store(_Tp* ptr, const _Tpvec& a) \ -{ _mm_storeu_si128((__m128i*)ptr, a.val); } \ -inline void v_store_aligned(_Tp* ptr, const _Tpvec& a) \ -{ _mm_store_si128((__m128i*)ptr, a.val); } \ -inline void v_store_aligned_nocache(_Tp* ptr, const _Tpvec& a) \ -{ _mm_stream_si128((__m128i*)ptr, a.val); } \ -inline void v_store(_Tp* ptr, const _Tpvec& a, hal::StoreMode mode) \ -{ \ - if ( mode == hal::STORE_UNALIGNED ) \ - _mm_storeu_si128((__m128i*)ptr, a.val); \ - else if ( mode == hal::STORE_ALIGNED_NOCACHE ) \ - _mm_stream_si128((__m128i*)ptr, a.val); \ - else \ - _mm_store_si128((__m128i*)ptr, a.val); \ -} \ -inline void v_store_low(_Tp* ptr, const _Tpvec& a) \ -{ _mm_storel_epi64((__m128i*)ptr, a.val); } \ -inline void v_store_high(_Tp* ptr, const _Tpvec& a) \ -{ _mm_storel_epi64((__m128i*)ptr, _mm_unpackhi_epi64(a.val, a.val)); } - -OPENCV_HAL_IMPL_SSE_LOADSTORE_INT_OP(v_uint8x16, uchar) -OPENCV_HAL_IMPL_SSE_LOADSTORE_INT_OP(v_int8x16, schar) -OPENCV_HAL_IMPL_SSE_LOADSTORE_INT_OP(v_uint16x8, ushort) -OPENCV_HAL_IMPL_SSE_LOADSTORE_INT_OP(v_int16x8, short) -OPENCV_HAL_IMPL_SSE_LOADSTORE_INT_OP(v_uint32x4, unsigned) -OPENCV_HAL_IMPL_SSE_LOADSTORE_INT_OP(v_int32x4, int) -OPENCV_HAL_IMPL_SSE_LOADSTORE_INT_OP(v_uint64x2, uint64) -OPENCV_HAL_IMPL_SSE_LOADSTORE_INT_OP(v_int64x2, int64) - -#define OPENCV_HAL_IMPL_SSE_LOADSTORE_FLT_OP(_Tpvec, _Tp, suffix) \ -inline _Tpvec v_load(const _Tp* ptr) \ -{ return _Tpvec(_mm_loadu_##suffix(ptr)); } \ -inline _Tpvec v_load_aligned(const _Tp* ptr) \ -{ return _Tpvec(_mm_load_##suffix(ptr)); } \ -inline _Tpvec v_load_low(const _Tp* ptr) \ -{ return _Tpvec(_mm_castsi128_##suffix(_mm_loadl_epi64((const __m128i*)ptr))); } \ -inline _Tpvec v_load_halves(const _Tp* ptr0, const _Tp* ptr1) \ -{ \ - return _Tpvec(_mm_castsi128_##suffix( \ - _mm_unpacklo_epi64(_mm_loadl_epi64((const __m128i*)ptr0), \ - _mm_loadl_epi64((const __m128i*)ptr1)))); \ -} \ -inline void v_store(_Tp* ptr, const _Tpvec& a) \ -{ _mm_storeu_##suffix(ptr, a.val); } \ -inline void v_store_aligned(_Tp* ptr, const _Tpvec& a) \ -{ _mm_store_##suffix(ptr, a.val); } \ -inline void v_store_aligned_nocache(_Tp* ptr, const _Tpvec& a) \ -{ _mm_stream_##suffix(ptr, a.val); } \ -inline void v_store(_Tp* ptr, const _Tpvec& a, hal::StoreMode mode) \ -{ \ - if ( mode == hal::STORE_UNALIGNED ) \ - _mm_storeu_##suffix(ptr, a.val); \ - else if ( mode == hal::STORE_ALIGNED_NOCACHE ) \ - _mm_stream_##suffix(ptr, a.val); \ - else \ - _mm_store_##suffix(ptr, a.val); \ -} \ -inline void v_store_low(_Tp* ptr, const _Tpvec& a) \ -{ _mm_storel_epi64((__m128i*)ptr, _mm_cast##suffix##_si128(a.val)); } \ -inline void v_store_high(_Tp* ptr, const _Tpvec& a) \ -{ \ - __m128i a1 = _mm_cast##suffix##_si128(a.val); \ - _mm_storel_epi64((__m128i*)ptr, _mm_unpackhi_epi64(a1, a1)); \ -} - -OPENCV_HAL_IMPL_SSE_LOADSTORE_FLT_OP(v_float32x4, float, ps) -OPENCV_HAL_IMPL_SSE_LOADSTORE_FLT_OP(v_float64x2, double, pd) - -#define OPENCV_HAL_IMPL_SSE_REDUCE_OP_8(_Tpvec, scalartype, func, suffix, sbit) \ -inline scalartype v_reduce_##func(const v_##_Tpvec& a) \ -{ \ - __m128i val = a.val; \ - val = _mm_##func##_##suffix(val, _mm_srli_si128(val,8)); \ - val = _mm_##func##_##suffix(val, _mm_srli_si128(val,4)); \ - val = _mm_##func##_##suffix(val, _mm_srli_si128(val,2)); \ - return (scalartype)_mm_cvtsi128_si32(val); \ -} \ -inline unsigned scalartype v_reduce_##func(const v_u##_Tpvec& a) \ -{ \ - __m128i val = a.val; \ - __m128i smask = _mm_set1_epi16(sbit); \ - val = _mm_xor_si128(val, smask); \ - val = _mm_##func##_##suffix(val, _mm_srli_si128(val,8)); \ - val = _mm_##func##_##suffix(val, _mm_srli_si128(val,4)); \ - val = _mm_##func##_##suffix(val, _mm_srli_si128(val,2)); \ - return (unsigned scalartype)(_mm_cvtsi128_si32(val) ^ sbit); \ -} -#define OPENCV_HAL_IMPL_SSE_REDUCE_OP_8_SUM(_Tpvec, scalartype, suffix) \ -inline scalartype v_reduce_sum(const v_##_Tpvec& a) \ -{ \ - __m128i val = a.val; \ - val = _mm_adds_epi##suffix(val, _mm_srli_si128(val, 8)); \ - val = _mm_adds_epi##suffix(val, _mm_srli_si128(val, 4)); \ - val = _mm_adds_epi##suffix(val, _mm_srli_si128(val, 2)); \ - return (scalartype)_mm_cvtsi128_si32(val); \ -} \ -inline unsigned scalartype v_reduce_sum(const v_u##_Tpvec& a) \ -{ \ - __m128i val = a.val; \ - val = _mm_adds_epu##suffix(val, _mm_srli_si128(val, 8)); \ - val = _mm_adds_epu##suffix(val, _mm_srli_si128(val, 4)); \ - val = _mm_adds_epu##suffix(val, _mm_srli_si128(val, 2)); \ - return (unsigned scalartype)_mm_cvtsi128_si32(val); \ -} -OPENCV_HAL_IMPL_SSE_REDUCE_OP_8(int16x8, short, max, epi16, (short)-32768) -OPENCV_HAL_IMPL_SSE_REDUCE_OP_8(int16x8, short, min, epi16, (short)-32768) -OPENCV_HAL_IMPL_SSE_REDUCE_OP_8_SUM(int16x8, short, 16) - -#define OPENCV_HAL_IMPL_SSE_REDUCE_OP_4_SUM(_Tpvec, scalartype, regtype, suffix, cast_from, cast_to, extract) \ -inline scalartype v_reduce_sum(const _Tpvec& a) \ -{ \ - regtype val = a.val; \ - val = _mm_add_##suffix(val, cast_to(_mm_srli_si128(cast_from(val), 8))); \ - val = _mm_add_##suffix(val, cast_to(_mm_srli_si128(cast_from(val), 4))); \ - return (scalartype)_mm_cvt##extract(val); \ -} - -#define OPENCV_HAL_IMPL_SSE_REDUCE_OP_4(_Tpvec, scalartype, func, scalar_func) \ -inline scalartype v_reduce_##func(const _Tpvec& a) \ -{ \ - scalartype CV_DECL_ALIGNED(16) buf[4]; \ - v_store_aligned(buf, a); \ - scalartype s0 = scalar_func(buf[0], buf[1]); \ - scalartype s1 = scalar_func(buf[2], buf[3]); \ - return scalar_func(s0, s1); \ -} - -OPENCV_HAL_IMPL_SSE_REDUCE_OP_4_SUM(v_uint32x4, unsigned, __m128i, epi32, OPENCV_HAL_NOP, OPENCV_HAL_NOP, si128_si32) -OPENCV_HAL_IMPL_SSE_REDUCE_OP_4_SUM(v_int32x4, int, __m128i, epi32, OPENCV_HAL_NOP, OPENCV_HAL_NOP, si128_si32) -OPENCV_HAL_IMPL_SSE_REDUCE_OP_4_SUM(v_float32x4, float, __m128, ps, _mm_castps_si128, _mm_castsi128_ps, ss_f32) - -inline v_float32x4 v_reduce_sum4(const v_float32x4& a, const v_float32x4& b, - const v_float32x4& c, const v_float32x4& d) -{ -#if CV_SSE3 - __m128 ab = _mm_hadd_ps(a.val, b.val); - __m128 cd = _mm_hadd_ps(c.val, d.val); - return v_float32x4(_mm_hadd_ps(ab, cd)); -#else - __m128 ac = _mm_add_ps(_mm_unpacklo_ps(a.val, c.val), _mm_unpackhi_ps(a.val, c.val)); - __m128 bd = _mm_add_ps(_mm_unpacklo_ps(b.val, d.val), _mm_unpackhi_ps(b.val, d.val)); - return v_float32x4(_mm_add_ps(_mm_unpacklo_ps(ac, bd), _mm_unpackhi_ps(ac, bd))); -#endif -} - -OPENCV_HAL_IMPL_SSE_REDUCE_OP_4(v_uint32x4, unsigned, max, std::max) -OPENCV_HAL_IMPL_SSE_REDUCE_OP_4(v_uint32x4, unsigned, min, std::min) -OPENCV_HAL_IMPL_SSE_REDUCE_OP_4(v_int32x4, int, max, std::max) -OPENCV_HAL_IMPL_SSE_REDUCE_OP_4(v_int32x4, int, min, std::min) -OPENCV_HAL_IMPL_SSE_REDUCE_OP_4(v_float32x4, float, max, std::max) -OPENCV_HAL_IMPL_SSE_REDUCE_OP_4(v_float32x4, float, min, std::min) - -#define OPENCV_HAL_IMPL_SSE_POPCOUNT(_Tpvec) \ -inline v_uint32x4 v_popcount(const _Tpvec& a) \ -{ \ - __m128i m1 = _mm_set1_epi32(0x55555555); \ - __m128i m2 = _mm_set1_epi32(0x33333333); \ - __m128i m4 = _mm_set1_epi32(0x0f0f0f0f); \ - __m128i p = a.val; \ - p = _mm_add_epi32(_mm_and_si128(_mm_srli_epi32(p, 1), m1), _mm_and_si128(p, m1)); \ - p = _mm_add_epi32(_mm_and_si128(_mm_srli_epi32(p, 2), m2), _mm_and_si128(p, m2)); \ - p = _mm_add_epi32(_mm_and_si128(_mm_srli_epi32(p, 4), m4), _mm_and_si128(p, m4)); \ - p = _mm_adds_epi8(p, _mm_srli_si128(p, 1)); \ - p = _mm_adds_epi8(p, _mm_srli_si128(p, 2)); \ - return v_uint32x4(_mm_and_si128(p, _mm_set1_epi32(0x000000ff))); \ -} - -OPENCV_HAL_IMPL_SSE_POPCOUNT(v_uint8x16) -OPENCV_HAL_IMPL_SSE_POPCOUNT(v_uint16x8) -OPENCV_HAL_IMPL_SSE_POPCOUNT(v_uint32x4) -OPENCV_HAL_IMPL_SSE_POPCOUNT(v_int8x16) -OPENCV_HAL_IMPL_SSE_POPCOUNT(v_int16x8) -OPENCV_HAL_IMPL_SSE_POPCOUNT(v_int32x4) - -#define OPENCV_HAL_IMPL_SSE_CHECK_SIGNS(_Tpvec, suffix, pack_op, and_op, signmask, allmask) \ -inline int v_signmask(const _Tpvec& a) \ -{ \ - return and_op(_mm_movemask_##suffix(pack_op(a.val)), signmask); \ -} \ -inline bool v_check_all(const _Tpvec& a) \ -{ return and_op(_mm_movemask_##suffix(a.val), allmask) == allmask; } \ -inline bool v_check_any(const _Tpvec& a) \ -{ return and_op(_mm_movemask_##suffix(a.val), allmask) != 0; } - -#define OPENCV_HAL_PACKS(a) _mm_packs_epi16(a, a) -inline __m128i v_packq_epi32(__m128i a) -{ - __m128i b = _mm_packs_epi32(a, a); - return _mm_packs_epi16(b, b); -} - -OPENCV_HAL_IMPL_SSE_CHECK_SIGNS(v_uint8x16, epi8, OPENCV_HAL_NOP, OPENCV_HAL_1ST, 65535, 65535) -OPENCV_HAL_IMPL_SSE_CHECK_SIGNS(v_int8x16, epi8, OPENCV_HAL_NOP, OPENCV_HAL_1ST, 65535, 65535) -OPENCV_HAL_IMPL_SSE_CHECK_SIGNS(v_uint16x8, epi8, OPENCV_HAL_PACKS, OPENCV_HAL_AND, 255, (int)0xaaaa) -OPENCV_HAL_IMPL_SSE_CHECK_SIGNS(v_int16x8, epi8, OPENCV_HAL_PACKS, OPENCV_HAL_AND, 255, (int)0xaaaa) -OPENCV_HAL_IMPL_SSE_CHECK_SIGNS(v_uint32x4, epi8, v_packq_epi32, OPENCV_HAL_AND, 15, (int)0x8888) -OPENCV_HAL_IMPL_SSE_CHECK_SIGNS(v_int32x4, epi8, v_packq_epi32, OPENCV_HAL_AND, 15, (int)0x8888) -OPENCV_HAL_IMPL_SSE_CHECK_SIGNS(v_float32x4, ps, OPENCV_HAL_NOP, OPENCV_HAL_1ST, 15, 15) -OPENCV_HAL_IMPL_SSE_CHECK_SIGNS(v_float64x2, pd, OPENCV_HAL_NOP, OPENCV_HAL_1ST, 3, 3) - -#if CV_SSE4_1 -#define OPENCV_HAL_IMPL_SSE_SELECT(_Tpvec, cast_ret, cast, suffix) \ -inline _Tpvec v_select(const _Tpvec& mask, const _Tpvec& a, const _Tpvec& b) \ -{ \ - return _Tpvec(cast_ret(_mm_blendv_##suffix(cast(b.val), cast(a.val), cast(mask.val)))); \ -} - -OPENCV_HAL_IMPL_SSE_SELECT(v_uint8x16, OPENCV_HAL_NOP, OPENCV_HAL_NOP, epi8) -OPENCV_HAL_IMPL_SSE_SELECT(v_int8x16, OPENCV_HAL_NOP, OPENCV_HAL_NOP, epi8) -OPENCV_HAL_IMPL_SSE_SELECT(v_uint16x8, OPENCV_HAL_NOP, OPENCV_HAL_NOP, epi8) -OPENCV_HAL_IMPL_SSE_SELECT(v_int16x8, OPENCV_HAL_NOP, OPENCV_HAL_NOP, epi8) -OPENCV_HAL_IMPL_SSE_SELECT(v_uint32x4, _mm_castps_si128, _mm_castsi128_ps, ps) -OPENCV_HAL_IMPL_SSE_SELECT(v_int32x4, _mm_castps_si128, _mm_castsi128_ps, ps) -// OPENCV_HAL_IMPL_SSE_SELECT(v_uint64x2, TBD, TBD, pd) -// OPENCV_HAL_IMPL_SSE_SELECT(v_int64x2, TBD, TBD, ps) -OPENCV_HAL_IMPL_SSE_SELECT(v_float32x4, OPENCV_HAL_NOP, OPENCV_HAL_NOP, ps) -OPENCV_HAL_IMPL_SSE_SELECT(v_float64x2, OPENCV_HAL_NOP, OPENCV_HAL_NOP, pd) - -#else // CV_SSE4_1 - -#define OPENCV_HAL_IMPL_SSE_SELECT(_Tpvec, suffix) \ -inline _Tpvec v_select(const _Tpvec& mask, const _Tpvec& a, const _Tpvec& b) \ -{ \ - return _Tpvec(_mm_xor_##suffix(b.val, _mm_and_##suffix(_mm_xor_##suffix(b.val, a.val), mask.val))); \ -} - -OPENCV_HAL_IMPL_SSE_SELECT(v_uint8x16, si128) -OPENCV_HAL_IMPL_SSE_SELECT(v_int8x16, si128) -OPENCV_HAL_IMPL_SSE_SELECT(v_uint16x8, si128) -OPENCV_HAL_IMPL_SSE_SELECT(v_int16x8, si128) -OPENCV_HAL_IMPL_SSE_SELECT(v_uint32x4, si128) -OPENCV_HAL_IMPL_SSE_SELECT(v_int32x4, si128) -// OPENCV_HAL_IMPL_SSE_SELECT(v_uint64x2, si128) -// OPENCV_HAL_IMPL_SSE_SELECT(v_int64x2, si128) -OPENCV_HAL_IMPL_SSE_SELECT(v_float32x4, ps) -OPENCV_HAL_IMPL_SSE_SELECT(v_float64x2, pd) -#endif - -#define OPENCV_HAL_IMPL_SSE_EXPAND(_Tpuvec, _Tpwuvec, _Tpu, _Tpsvec, _Tpwsvec, _Tps, suffix, wsuffix, shift) \ -inline void v_expand(const _Tpuvec& a, _Tpwuvec& b0, _Tpwuvec& b1) \ -{ \ - __m128i z = _mm_setzero_si128(); \ - b0.val = _mm_unpacklo_##suffix(a.val, z); \ - b1.val = _mm_unpackhi_##suffix(a.val, z); \ -} \ -inline _Tpwuvec v_load_expand(const _Tpu* ptr) \ -{ \ - __m128i z = _mm_setzero_si128(); \ - return _Tpwuvec(_mm_unpacklo_##suffix(_mm_loadl_epi64((const __m128i*)ptr), z)); \ -} \ -inline void v_expand(const _Tpsvec& a, _Tpwsvec& b0, _Tpwsvec& b1) \ -{ \ - b0.val = _mm_srai_##wsuffix(_mm_unpacklo_##suffix(a.val, a.val), shift); \ - b1.val = _mm_srai_##wsuffix(_mm_unpackhi_##suffix(a.val, a.val), shift); \ -} \ -inline _Tpwsvec v_load_expand(const _Tps* ptr) \ -{ \ - __m128i a = _mm_loadl_epi64((const __m128i*)ptr); \ - return _Tpwsvec(_mm_srai_##wsuffix(_mm_unpacklo_##suffix(a, a), shift)); \ -}\ -inline _Tpwuvec v_expand_low(const _Tpuvec& a) { \ - _Tpwuvec res; \ - res.val = _mm_cvtepu8_epi16(a.val); \ - return res; \ -} \ -inline _Tpwuvec v_expand_high(const _Tpuvec& a) { \ - _Tpwuvec res; \ - res.val = _mm_unpackhi_epi8(a.val, _mm_setzero_si128()); \ - return res; \ -} \ - -OPENCV_HAL_IMPL_SSE_EXPAND(v_uint8x16, v_uint16x8, uchar, v_int8x16, v_int16x8, schar, epi8, epi16, 8) -OPENCV_HAL_IMPL_SSE_EXPAND(v_uint16x8, v_uint32x4, ushort, v_int16x8, v_int32x4, short, epi16, epi32, 16) - -inline void v_expand(const v_uint32x4& a, v_uint64x2& b0, v_uint64x2& b1) -{ - __m128i z = _mm_setzero_si128(); - b0.val = _mm_unpacklo_epi32(a.val, z); - b1.val = _mm_unpackhi_epi32(a.val, z); -} -inline v_uint64x2 v_load_expand(const unsigned* ptr) -{ - __m128i z = _mm_setzero_si128(); - return v_uint64x2(_mm_unpacklo_epi32(_mm_loadl_epi64((const __m128i*)ptr), z)); -} -inline void v_expand(const v_int32x4& a, v_int64x2& b0, v_int64x2& b1) -{ - __m128i s = _mm_srai_epi32(a.val, 31); - b0.val = _mm_unpacklo_epi32(a.val, s); - b1.val = _mm_unpackhi_epi32(a.val, s); -} -inline v_int64x2 v_load_expand(const int* ptr) -{ - __m128i a = _mm_loadl_epi64((const __m128i*)ptr); - __m128i s = _mm_srai_epi32(a, 31); - return v_int64x2(_mm_unpacklo_epi32(a, s)); -} - -inline v_uint32x4 v_load_expand_q(const uchar* ptr) -{ - __m128i z = _mm_setzero_si128(); - __m128i a = _mm_cvtsi32_si128(*(const int*)ptr); - return v_uint32x4(_mm_unpacklo_epi16(_mm_unpacklo_epi8(a, z), z)); -} - -inline v_int32x4 v_load_expand_q(const schar* ptr) -{ - __m128i a = _mm_cvtsi32_si128(*(const int*)ptr); - a = _mm_unpacklo_epi8(a, a); - a = _mm_unpacklo_epi8(a, a); - return v_int32x4(_mm_srai_epi32(a, 24)); -} - -#define OPENCV_HAL_IMPL_SSE_UNPACKS(_Tpvec, suffix, cast_from, cast_to) \ -inline void v_zip(const _Tpvec& a0, const _Tpvec& a1, _Tpvec& b0, _Tpvec& b1) \ -{ \ - b0.val = _mm_unpacklo_##suffix(a0.val, a1.val); \ - b1.val = _mm_unpackhi_##suffix(a0.val, a1.val); \ -} \ -inline _Tpvec v_combine_low(const _Tpvec& a, const _Tpvec& b) \ -{ \ - __m128i a1 = cast_from(a.val), b1 = cast_from(b.val); \ - return _Tpvec(cast_to(_mm_unpacklo_epi64(a1, b1))); \ -} \ -inline _Tpvec v_combine_high(const _Tpvec& a, const _Tpvec& b) \ -{ \ - __m128i a1 = cast_from(a.val), b1 = cast_from(b.val); \ - return _Tpvec(cast_to(_mm_unpackhi_epi64(a1, b1))); \ -} \ -inline void v_recombine(const _Tpvec& a, const _Tpvec& b, _Tpvec& c, _Tpvec& d) \ -{ \ - __m128i a1 = cast_from(a.val), b1 = cast_from(b.val); \ - c.val = cast_to(_mm_unpacklo_epi64(a1, b1)); \ - d.val = cast_to(_mm_unpackhi_epi64(a1, b1)); \ -} - -OPENCV_HAL_IMPL_SSE_UNPACKS(v_uint8x16, epi8, OPENCV_HAL_NOP, OPENCV_HAL_NOP) -OPENCV_HAL_IMPL_SSE_UNPACKS(v_int8x16, epi8, OPENCV_HAL_NOP, OPENCV_HAL_NOP) -OPENCV_HAL_IMPL_SSE_UNPACKS(v_uint16x8, epi16, OPENCV_HAL_NOP, OPENCV_HAL_NOP) -OPENCV_HAL_IMPL_SSE_UNPACKS(v_int16x8, epi16, OPENCV_HAL_NOP, OPENCV_HAL_NOP) -OPENCV_HAL_IMPL_SSE_UNPACKS(v_uint32x4, epi32, OPENCV_HAL_NOP, OPENCV_HAL_NOP) -OPENCV_HAL_IMPL_SSE_UNPACKS(v_int32x4, epi32, OPENCV_HAL_NOP, OPENCV_HAL_NOP) -OPENCV_HAL_IMPL_SSE_UNPACKS(v_float32x4, ps, _mm_castps_si128, _mm_castsi128_ps) -OPENCV_HAL_IMPL_SSE_UNPACKS(v_float64x2, pd, _mm_castpd_si128, _mm_castsi128_pd) - -template -inline _Tpvec v_extract(const _Tpvec& a, const _Tpvec& b) -{ - return v_rotate_right(a, b); -} - -inline v_int32x4 v_round(const v_float32x4& a) -{ return v_int32x4(_mm_cvtps_epi32(a.val)); } - -inline v_int32x4 v_floor(const v_float32x4& a) -{ - __m128i a1 = _mm_cvtps_epi32(a.val); - __m128i mask = _mm_castps_si128(_mm_cmpgt_ps(_mm_cvtepi32_ps(a1), a.val)); - return v_int32x4(_mm_add_epi32(a1, mask)); -} - -inline v_int32x4 v_ceil(const v_float32x4& a) -{ - __m128i a1 = _mm_cvtps_epi32(a.val); - __m128i mask = _mm_castps_si128(_mm_cmpgt_ps(a.val, _mm_cvtepi32_ps(a1))); - return v_int32x4(_mm_sub_epi32(a1, mask)); -} - -inline v_int32x4 v_trunc(const v_float32x4& a) -{ return v_int32x4(_mm_cvttps_epi32(a.val)); } - -inline v_int32x4 v_round(const v_float64x2& a) -{ return v_int32x4(_mm_cvtpd_epi32(a.val)); } - -inline v_int32x4 v_floor(const v_float64x2& a) -{ - __m128i a1 = _mm_cvtpd_epi32(a.val); - __m128i mask = _mm_castpd_si128(_mm_cmpgt_pd(_mm_cvtepi32_pd(a1), a.val)); - mask = _mm_srli_si128(_mm_slli_si128(mask, 4), 8); // m0 m0 m1 m1 => m0 m1 0 0 - return v_int32x4(_mm_add_epi32(a1, mask)); -} - -inline v_int32x4 v_ceil(const v_float64x2& a) -{ - __m128i a1 = _mm_cvtpd_epi32(a.val); - __m128i mask = _mm_castpd_si128(_mm_cmpgt_pd(a.val, _mm_cvtepi32_pd(a1))); - mask = _mm_srli_si128(_mm_slli_si128(mask, 4), 8); // m0 m0 m1 m1 => m0 m1 0 0 - return v_int32x4(_mm_sub_epi32(a1, mask)); -} - -inline v_int32x4 v_trunc(const v_float64x2& a) -{ return v_int32x4(_mm_cvttpd_epi32(a.val)); } - -#define OPENCV_HAL_IMPL_SSE_TRANSPOSE4x4(_Tpvec, suffix, cast_from, cast_to) \ -inline void v_transpose4x4(const _Tpvec& a0, const _Tpvec& a1, \ - const _Tpvec& a2, const _Tpvec& a3, \ - _Tpvec& b0, _Tpvec& b1, \ - _Tpvec& b2, _Tpvec& b3) \ -{ \ - __m128i t0 = cast_from(_mm_unpacklo_##suffix(a0.val, a1.val)); \ - __m128i t1 = cast_from(_mm_unpacklo_##suffix(a2.val, a3.val)); \ - __m128i t2 = cast_from(_mm_unpackhi_##suffix(a0.val, a1.val)); \ - __m128i t3 = cast_from(_mm_unpackhi_##suffix(a2.val, a3.val)); \ -\ - b0.val = cast_to(_mm_unpacklo_epi64(t0, t1)); \ - b1.val = cast_to(_mm_unpackhi_epi64(t0, t1)); \ - b2.val = cast_to(_mm_unpacklo_epi64(t2, t3)); \ - b3.val = cast_to(_mm_unpackhi_epi64(t2, t3)); \ -} - -OPENCV_HAL_IMPL_SSE_TRANSPOSE4x4(v_uint32x4, epi32, OPENCV_HAL_NOP, OPENCV_HAL_NOP) -OPENCV_HAL_IMPL_SSE_TRANSPOSE4x4(v_int32x4, epi32, OPENCV_HAL_NOP, OPENCV_HAL_NOP) -OPENCV_HAL_IMPL_SSE_TRANSPOSE4x4(v_float32x4, ps, _mm_castps_si128, _mm_castsi128_ps) - -// load deinterleave -inline void v_load_deinterleave(const uchar* ptr, v_uint8x16& a, v_uint8x16& b) -{ - __m128i t00 = _mm_loadu_si128((const __m128i*)ptr); - __m128i t01 = _mm_loadu_si128((const __m128i*)(ptr + 16)); - - __m128i t10 = _mm_unpacklo_epi8(t00, t01); - __m128i t11 = _mm_unpackhi_epi8(t00, t01); - - __m128i t20 = _mm_unpacklo_epi8(t10, t11); - __m128i t21 = _mm_unpackhi_epi8(t10, t11); - - __m128i t30 = _mm_unpacklo_epi8(t20, t21); - __m128i t31 = _mm_unpackhi_epi8(t20, t21); - - a.val = _mm_unpacklo_epi8(t30, t31); - b.val = _mm_unpackhi_epi8(t30, t31); -} - -inline void v_load_deinterleave(const uchar* ptr, v_uint8x16& a, v_uint8x16& b, v_uint8x16& c) -{ -#if CV_SSE4_1 - const __m128i m0 = _mm_setr_epi8(0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0); - const __m128i m1 = _mm_setr_epi8(0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0); - __m128i s0 = _mm_loadu_si128((const __m128i*)ptr); - __m128i s1 = _mm_loadu_si128((const __m128i*)(ptr + 16)); - __m128i s2 = _mm_loadu_si128((const __m128i*)(ptr + 32)); - __m128i a0 = _mm_blendv_epi8(_mm_blendv_epi8(s0, s1, m0), s2, m1); - __m128i b0 = _mm_blendv_epi8(_mm_blendv_epi8(s1, s2, m0), s0, m1); - __m128i c0 = _mm_blendv_epi8(_mm_blendv_epi8(s2, s0, m0), s1, m1); - const __m128i sh_b = _mm_setr_epi8(0, 3, 6, 9, 12, 15, 2, 5, 8, 11, 14, 1, 4, 7, 10, 13); - const __m128i sh_g = _mm_setr_epi8(1, 4, 7, 10, 13, 0, 3, 6, 9, 12, 15, 2, 5, 8, 11, 14); - const __m128i sh_r = _mm_setr_epi8(2, 5, 8, 11, 14, 1, 4, 7, 10, 13, 0, 3, 6, 9, 12, 15); - a0 = _mm_shuffle_epi8(a0, sh_b); - b0 = _mm_shuffle_epi8(b0, sh_g); - c0 = _mm_shuffle_epi8(c0, sh_r); - a.val = a0; - b.val = b0; - c.val = c0; -#elif CV_SSSE3 - const __m128i m0 = _mm_setr_epi8(0, 3, 6, 9, 12, 15, 1, 4, 7, 10, 13, 2, 5, 8, 11, 14); - const __m128i m1 = _mm_alignr_epi8(m0, m0, 11); - const __m128i m2 = _mm_alignr_epi8(m0, m0, 6); - - __m128i t0 = _mm_loadu_si128((const __m128i*)ptr); - __m128i t1 = _mm_loadu_si128((const __m128i*)(ptr + 16)); - __m128i t2 = _mm_loadu_si128((const __m128i*)(ptr + 32)); - - __m128i s0 = _mm_shuffle_epi8(t0, m0); - __m128i s1 = _mm_shuffle_epi8(t1, m1); - __m128i s2 = _mm_shuffle_epi8(t2, m2); - - t0 = _mm_alignr_epi8(s1, _mm_slli_si128(s0, 10), 5); - a.val = _mm_alignr_epi8(s2, t0, 5); - - t1 = _mm_alignr_epi8(_mm_srli_si128(s1, 5), _mm_slli_si128(s0, 5), 6); - b.val = _mm_alignr_epi8(_mm_srli_si128(s2, 5), t1, 5); - - t2 = _mm_alignr_epi8(_mm_srli_si128(s2, 10), s1, 11); - c.val = _mm_alignr_epi8(t2, s0, 11); -#else - __m128i t00 = _mm_loadu_si128((const __m128i*)ptr); - __m128i t01 = _mm_loadu_si128((const __m128i*)(ptr + 16)); - __m128i t02 = _mm_loadu_si128((const __m128i*)(ptr + 32)); - - __m128i t10 = _mm_unpacklo_epi8(t00, _mm_unpackhi_epi64(t01, t01)); - __m128i t11 = _mm_unpacklo_epi8(_mm_unpackhi_epi64(t00, t00), t02); - __m128i t12 = _mm_unpacklo_epi8(t01, _mm_unpackhi_epi64(t02, t02)); - - __m128i t20 = _mm_unpacklo_epi8(t10, _mm_unpackhi_epi64(t11, t11)); - __m128i t21 = _mm_unpacklo_epi8(_mm_unpackhi_epi64(t10, t10), t12); - __m128i t22 = _mm_unpacklo_epi8(t11, _mm_unpackhi_epi64(t12, t12)); - - __m128i t30 = _mm_unpacklo_epi8(t20, _mm_unpackhi_epi64(t21, t21)); - __m128i t31 = _mm_unpacklo_epi8(_mm_unpackhi_epi64(t20, t20), t22); - __m128i t32 = _mm_unpacklo_epi8(t21, _mm_unpackhi_epi64(t22, t22)); - - a.val = _mm_unpacklo_epi8(t30, _mm_unpackhi_epi64(t31, t31)); - b.val = _mm_unpacklo_epi8(_mm_unpackhi_epi64(t30, t30), t32); - c.val = _mm_unpacklo_epi8(t31, _mm_unpackhi_epi64(t32, t32)); -#endif -} - -inline void v_load_deinterleave(const uchar* ptr, v_uint8x16& a, v_uint8x16& b, v_uint8x16& c, v_uint8x16& d) -{ - __m128i u0 = _mm_loadu_si128((const __m128i*)ptr); // a0 b0 c0 d0 a1 b1 c1 d1 ... - __m128i u1 = _mm_loadu_si128((const __m128i*)(ptr + 16)); // a4 b4 c4 d4 ... - __m128i u2 = _mm_loadu_si128((const __m128i*)(ptr + 32)); // a8 b8 c8 d8 ... - __m128i u3 = _mm_loadu_si128((const __m128i*)(ptr + 48)); // a12 b12 c12 d12 ... - - __m128i v0 = _mm_unpacklo_epi8(u0, u2); // a0 a8 b0 b8 ... - __m128i v1 = _mm_unpackhi_epi8(u0, u2); // a2 a10 b2 b10 ... - __m128i v2 = _mm_unpacklo_epi8(u1, u3); // a4 a12 b4 b12 ... - __m128i v3 = _mm_unpackhi_epi8(u1, u3); // a6 a14 b6 b14 ... - - u0 = _mm_unpacklo_epi8(v0, v2); // a0 a4 a8 a12 ... - u1 = _mm_unpacklo_epi8(v1, v3); // a2 a6 a10 a14 ... - u2 = _mm_unpackhi_epi8(v0, v2); // a1 a5 a9 a13 ... - u3 = _mm_unpackhi_epi8(v1, v3); // a3 a7 a11 a15 ... - - v0 = _mm_unpacklo_epi8(u0, u1); // a0 a2 a4 a6 ... - v1 = _mm_unpacklo_epi8(u2, u3); // a1 a3 a5 a7 ... - v2 = _mm_unpackhi_epi8(u0, u1); // c0 c2 c4 c6 ... - v3 = _mm_unpackhi_epi8(u2, u3); // c1 c3 c5 c7 ... - - a.val = _mm_unpacklo_epi8(v0, v1); - b.val = _mm_unpackhi_epi8(v0, v1); - c.val = _mm_unpacklo_epi8(v2, v3); - d.val = _mm_unpackhi_epi8(v2, v3); -} - -inline void v_load_deinterleave(const ushort* ptr, v_uint16x8& a, v_uint16x8& b) -{ - __m128i v0 = _mm_loadu_si128((__m128i*)(ptr)); // a0 b0 a1 b1 a2 b2 a3 b3 - __m128i v1 = _mm_loadu_si128((__m128i*)(ptr + 8)); // a4 b4 a5 b5 a6 b6 a7 b7 - - __m128i v2 = _mm_unpacklo_epi16(v0, v1); // a0 a4 b0 b4 a1 a5 b1 b5 - __m128i v3 = _mm_unpackhi_epi16(v0, v1); // a2 a6 b2 b6 a3 a7 b3 b7 - __m128i v4 = _mm_unpacklo_epi16(v2, v3); // a0 a2 a4 a6 b0 b2 b4 b6 - __m128i v5 = _mm_unpackhi_epi16(v2, v3); // a1 a3 a5 a7 b1 b3 b5 b7 - - a.val = _mm_unpacklo_epi16(v4, v5); // a0 a1 a2 a3 a4 a5 a6 a7 - b.val = _mm_unpackhi_epi16(v4, v5); // b0 b1 ab b3 b4 b5 b6 b7 -} - -inline void v_load_deinterleave(const ushort* ptr, v_uint16x8& a, v_uint16x8& b, v_uint16x8& c) -{ -#if CV_SSE4_1 - __m128i v0 = _mm_loadu_si128((__m128i*)(ptr)); - __m128i v1 = _mm_loadu_si128((__m128i*)(ptr + 8)); - __m128i v2 = _mm_loadu_si128((__m128i*)(ptr + 16)); - __m128i a0 = _mm_blend_epi16(_mm_blend_epi16(v0, v1, 0x92), v2, 0x24); - __m128i b0 = _mm_blend_epi16(_mm_blend_epi16(v2, v0, 0x92), v1, 0x24); - __m128i c0 = _mm_blend_epi16(_mm_blend_epi16(v1, v2, 0x92), v0, 0x24); - - const __m128i sh_a = _mm_setr_epi8(0, 1, 6, 7, 12, 13, 2, 3, 8, 9, 14, 15, 4, 5, 10, 11); - const __m128i sh_b = _mm_setr_epi8(2, 3, 8, 9, 14, 15, 4, 5, 10, 11, 0, 1, 6, 7, 12, 13); - const __m128i sh_c = _mm_setr_epi8(4, 5, 10, 11, 0, 1, 6, 7, 12, 13, 2, 3, 8, 9, 14, 15); - a0 = _mm_shuffle_epi8(a0, sh_a); - b0 = _mm_shuffle_epi8(b0, sh_b); - c0 = _mm_shuffle_epi8(c0, sh_c); - - a.val = a0; - b.val = b0; - c.val = c0; -#else - __m128i t00 = _mm_loadu_si128((const __m128i*)ptr); - __m128i t01 = _mm_loadu_si128((const __m128i*)(ptr + 8)); - __m128i t02 = _mm_loadu_si128((const __m128i*)(ptr + 16)); - - __m128i t10 = _mm_unpacklo_epi16(t00, _mm_unpackhi_epi64(t01, t01)); - __m128i t11 = _mm_unpacklo_epi16(_mm_unpackhi_epi64(t00, t00), t02); - __m128i t12 = _mm_unpacklo_epi16(t01, _mm_unpackhi_epi64(t02, t02)); - - __m128i t20 = _mm_unpacklo_epi16(t10, _mm_unpackhi_epi64(t11, t11)); - __m128i t21 = _mm_unpacklo_epi16(_mm_unpackhi_epi64(t10, t10), t12); - __m128i t22 = _mm_unpacklo_epi16(t11, _mm_unpackhi_epi64(t12, t12)); - - a.val = _mm_unpacklo_epi16(t20, _mm_unpackhi_epi64(t21, t21)); - b.val = _mm_unpacklo_epi16(_mm_unpackhi_epi64(t20, t20), t22); - c.val = _mm_unpacklo_epi16(t21, _mm_unpackhi_epi64(t22, t22)); -#endif -} - -inline void v_load_deinterleave(const ushort* ptr, v_uint16x8& a, v_uint16x8& b, v_uint16x8& c, v_uint16x8& d) -{ - __m128i u0 = _mm_loadu_si128((const __m128i*)ptr); // a0 b0 c0 d0 a1 b1 c1 d1 - __m128i u1 = _mm_loadu_si128((const __m128i*)(ptr + 8)); // a2 b2 c2 d2 ... - __m128i u2 = _mm_loadu_si128((const __m128i*)(ptr + 16)); // a4 b4 c4 d4 ... - __m128i u3 = _mm_loadu_si128((const __m128i*)(ptr + 24)); // a6 b6 c6 d6 ... - - __m128i v0 = _mm_unpacklo_epi16(u0, u2); // a0 a4 b0 b4 ... - __m128i v1 = _mm_unpackhi_epi16(u0, u2); // a1 a5 b1 b5 ... - __m128i v2 = _mm_unpacklo_epi16(u1, u3); // a2 a6 b2 b6 ... - __m128i v3 = _mm_unpackhi_epi16(u1, u3); // a3 a7 b3 b7 ... - - u0 = _mm_unpacklo_epi16(v0, v2); // a0 a2 a4 a6 ... - u1 = _mm_unpacklo_epi16(v1, v3); // a1 a3 a5 a7 ... - u2 = _mm_unpackhi_epi16(v0, v2); // c0 c2 c4 c6 ... - u3 = _mm_unpackhi_epi16(v1, v3); // c1 c3 c5 c7 ... - - a.val = _mm_unpacklo_epi16(u0, u1); - b.val = _mm_unpackhi_epi16(u0, u1); - c.val = _mm_unpacklo_epi16(u2, u3); - d.val = _mm_unpackhi_epi16(u2, u3); -} - -inline void v_load_deinterleave(const unsigned* ptr, v_uint32x4& a, v_uint32x4& b) -{ - __m128i v0 = _mm_loadu_si128((__m128i*)(ptr)); // a0 b0 a1 b1 - __m128i v1 = _mm_loadu_si128((__m128i*)(ptr + 4)); // a2 b2 a3 b3 - - __m128i v2 = _mm_unpacklo_epi32(v0, v1); // a0 a2 b0 b2 - __m128i v3 = _mm_unpackhi_epi32(v0, v1); // a1 a3 b1 b3 - - a.val = _mm_unpacklo_epi32(v2, v3); // a0 a1 a2 a3 - b.val = _mm_unpackhi_epi32(v2, v3); // b0 b1 ab b3 -} - -inline void v_load_deinterleave(const unsigned* ptr, v_uint32x4& a, v_uint32x4& b, v_uint32x4& c) -{ - __m128i t00 = _mm_loadu_si128((const __m128i*)ptr); - __m128i t01 = _mm_loadu_si128((const __m128i*)(ptr + 4)); - __m128i t02 = _mm_loadu_si128((const __m128i*)(ptr + 8)); - - __m128i t10 = _mm_unpacklo_epi32(t00, _mm_unpackhi_epi64(t01, t01)); - __m128i t11 = _mm_unpacklo_epi32(_mm_unpackhi_epi64(t00, t00), t02); - __m128i t12 = _mm_unpacklo_epi32(t01, _mm_unpackhi_epi64(t02, t02)); - - a.val = _mm_unpacklo_epi32(t10, _mm_unpackhi_epi64(t11, t11)); - b.val = _mm_unpacklo_epi32(_mm_unpackhi_epi64(t10, t10), t12); - c.val = _mm_unpacklo_epi32(t11, _mm_unpackhi_epi64(t12, t12)); -} - -inline void v_load_deinterleave(const unsigned* ptr, v_uint32x4& a, v_uint32x4& b, v_uint32x4& c, v_uint32x4& d) -{ - v_uint32x4 s0(_mm_loadu_si128((const __m128i*)ptr)); // a0 b0 c0 d0 - v_uint32x4 s1(_mm_loadu_si128((const __m128i*)(ptr + 4))); // a1 b1 c1 d1 - v_uint32x4 s2(_mm_loadu_si128((const __m128i*)(ptr + 8))); // a2 b2 c2 d2 - v_uint32x4 s3(_mm_loadu_si128((const __m128i*)(ptr + 12))); // a3 b3 c3 d3 - - v_transpose4x4(s0, s1, s2, s3, a, b, c, d); -} - -inline void v_load_deinterleave(const float* ptr, v_float32x4& a, v_float32x4& b) -{ - const int mask_lo = _MM_SHUFFLE(2, 0, 2, 0), mask_hi = _MM_SHUFFLE(3, 1, 3, 1); - - __m128 u0 = _mm_loadu_ps(ptr); // a0 b0 a1 b1 - __m128 u1 = _mm_loadu_ps((ptr + 4)); // a2 b2 a3 b3 - - a.val = _mm_shuffle_ps(u0, u1, mask_lo); // a0 a1 a2 a3 - b.val = _mm_shuffle_ps(u0, u1, mask_hi); // b0 b1 ab b3 -} - -inline void v_load_deinterleave(const float* ptr, v_float32x4& a, v_float32x4& b, v_float32x4& c) -{ - __m128 t0 = _mm_loadu_ps(ptr + 0); - __m128 t1 = _mm_loadu_ps(ptr + 4); - __m128 t2 = _mm_loadu_ps(ptr + 8); - - __m128 at12 = _mm_shuffle_ps(t1, t2, _MM_SHUFFLE(0, 1, 0, 2)); - a.val = _mm_shuffle_ps(t0, at12, _MM_SHUFFLE(2, 0, 3, 0)); - - __m128 bt01 = _mm_shuffle_ps(t0, t1, _MM_SHUFFLE(0, 0, 0, 1)); - __m128 bt12 = _mm_shuffle_ps(t1, t2, _MM_SHUFFLE(0, 2, 0, 3)); - b.val = _mm_shuffle_ps(bt01, bt12, _MM_SHUFFLE(2, 0, 2, 0)); - - __m128 ct01 = _mm_shuffle_ps(t0, t1, _MM_SHUFFLE(0, 1, 0, 2)); - c.val = _mm_shuffle_ps(ct01, t2, _MM_SHUFFLE(3, 0, 2, 0)); -} - -inline void v_load_deinterleave(const float* ptr, v_float32x4& a, v_float32x4& b, v_float32x4& c, v_float32x4& d) -{ - __m128 t0 = _mm_loadu_ps(ptr + 0); - __m128 t1 = _mm_loadu_ps(ptr + 4); - __m128 t2 = _mm_loadu_ps(ptr + 8); - __m128 t3 = _mm_loadu_ps(ptr + 12); - __m128 t02lo = _mm_unpacklo_ps(t0, t2); - __m128 t13lo = _mm_unpacklo_ps(t1, t3); - __m128 t02hi = _mm_unpackhi_ps(t0, t2); - __m128 t13hi = _mm_unpackhi_ps(t1, t3); - a.val = _mm_unpacklo_ps(t02lo, t13lo); - b.val = _mm_unpackhi_ps(t02lo, t13lo); - c.val = _mm_unpacklo_ps(t02hi, t13hi); - d.val = _mm_unpackhi_ps(t02hi, t13hi); -} - -inline void v_load_deinterleave(const uint64 *ptr, v_uint64x2& a, v_uint64x2& b) -{ - __m128i t0 = _mm_loadu_si128((const __m128i*)ptr); - __m128i t1 = _mm_loadu_si128((const __m128i*)(ptr + 2)); - - a = v_uint64x2(_mm_unpacklo_epi64(t0, t1)); - b = v_uint64x2(_mm_unpackhi_epi64(t0, t1)); -} - -inline void v_load_deinterleave(const uint64 *ptr, v_uint64x2& a, v_uint64x2& b, v_uint64x2& c) -{ - __m128i t0 = _mm_loadu_si128((const __m128i*)ptr); // a0, b0 - __m128i t1 = _mm_loadu_si128((const __m128i*)(ptr + 2)); // c0, a1 - __m128i t2 = _mm_loadu_si128((const __m128i*)(ptr + 4)); // b1, c1 - - t1 = _mm_shuffle_epi32(t1, 0x4e); // a1, c0 - - a = v_uint64x2(_mm_unpacklo_epi64(t0, t1)); - b = v_uint64x2(_mm_unpacklo_epi64(_mm_unpackhi_epi64(t0, t0), t2)); - c = v_uint64x2(_mm_unpackhi_epi64(t1, t2)); -} - -inline void v_load_deinterleave(const uint64 *ptr, v_uint64x2& a, - v_uint64x2& b, v_uint64x2& c, v_uint64x2& d) -{ - __m128i t0 = _mm_loadu_si128((const __m128i*)ptr); // a0 b0 - __m128i t1 = _mm_loadu_si128((const __m128i*)(ptr + 2)); // c0 d0 - __m128i t2 = _mm_loadu_si128((const __m128i*)(ptr + 4)); // a1 b1 - __m128i t3 = _mm_loadu_si128((const __m128i*)(ptr + 6)); // c1 d1 - - a = v_uint64x2(_mm_unpacklo_epi64(t0, t2)); - b = v_uint64x2(_mm_unpackhi_epi64(t0, t2)); - c = v_uint64x2(_mm_unpacklo_epi64(t1, t3)); - d = v_uint64x2(_mm_unpackhi_epi64(t1, t3)); -} - -// store interleave - -inline void v_store_interleave( uchar* ptr, const v_uint8x16& a, const v_uint8x16& b, - hal::StoreMode mode = hal::STORE_UNALIGNED) -{ - __m128i v0 = _mm_unpacklo_epi8(a.val, b.val); - __m128i v1 = _mm_unpackhi_epi8(a.val, b.val); - - if ( mode == hal::STORE_ALIGNED_NOCACHE ) - { - _mm_stream_si128((__m128i*)(ptr), v0); - _mm_stream_si128((__m128i*)(ptr + 16), v1); - } - else if ( mode == hal::STORE_ALIGNED ) - { - _mm_store_si128((__m128i*)(ptr), v0); - _mm_store_si128((__m128i*)(ptr + 16), v1); - } - else - { - _mm_storeu_si128((__m128i*)(ptr), v0); - _mm_storeu_si128((__m128i*)(ptr + 16), v1); - } -} - -inline void v_store_interleave( uchar* ptr, const v_uint8x16& a, const v_uint8x16& b, - const v_uint8x16& c, hal::StoreMode mode = hal::STORE_UNALIGNED) -{ -#if CV_SSE4_1 - const __m128i sh_a = _mm_setr_epi8(0, 11, 6, 1, 12, 7, 2, 13, 8, 3, 14, 9, 4, 15, 10, 5); - const __m128i sh_b = _mm_setr_epi8(5, 0, 11, 6, 1, 12, 7, 2, 13, 8, 3, 14, 9, 4, 15, 10); - const __m128i sh_c = _mm_setr_epi8(10, 5, 0, 11, 6, 1, 12, 7, 2, 13, 8, 3, 14, 9, 4, 15); - __m128i a0 = _mm_shuffle_epi8(a.val, sh_a); - __m128i b0 = _mm_shuffle_epi8(b.val, sh_b); - __m128i c0 = _mm_shuffle_epi8(c.val, sh_c); - - const __m128i m0 = _mm_setr_epi8(0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0); - const __m128i m1 = _mm_setr_epi8(0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0); - __m128i v0 = _mm_blendv_epi8(_mm_blendv_epi8(a0, b0, m1), c0, m0); - __m128i v1 = _mm_blendv_epi8(_mm_blendv_epi8(b0, c0, m1), a0, m0); - __m128i v2 = _mm_blendv_epi8(_mm_blendv_epi8(c0, a0, m1), b0, m0); -#elif CV_SSSE3 - const __m128i m0 = _mm_setr_epi8(0, 6, 11, 1, 7, 12, 2, 8, 13, 3, 9, 14, 4, 10, 15, 5); - const __m128i m1 = _mm_setr_epi8(5, 11, 0, 6, 12, 1, 7, 13, 2, 8, 14, 3, 9, 15, 4, 10); - const __m128i m2 = _mm_setr_epi8(10, 0, 5, 11, 1, 6, 12, 2, 7, 13, 3, 8, 14, 4, 9, 15); - - __m128i t0 = _mm_alignr_epi8(b.val, _mm_slli_si128(a.val, 10), 5); - t0 = _mm_alignr_epi8(c.val, t0, 5); - __m128i v0 = _mm_shuffle_epi8(t0, m0); - - __m128i t1 = _mm_alignr_epi8(_mm_srli_si128(b.val, 5), _mm_slli_si128(a.val, 5), 6); - t1 = _mm_alignr_epi8(_mm_srli_si128(c.val, 5), t1, 5); - __m128i v1 = _mm_shuffle_epi8(t1, m1); - - __m128i t2 = _mm_alignr_epi8(_mm_srli_si128(c.val, 10), b.val, 11); - t2 = _mm_alignr_epi8(t2, a.val, 11); - __m128i v2 = _mm_shuffle_epi8(t2, m2); -#else - __m128i z = _mm_setzero_si128(); - __m128i ab0 = _mm_unpacklo_epi8(a.val, b.val); - __m128i ab1 = _mm_unpackhi_epi8(a.val, b.val); - __m128i c0 = _mm_unpacklo_epi8(c.val, z); - __m128i c1 = _mm_unpackhi_epi8(c.val, z); - - __m128i p00 = _mm_unpacklo_epi16(ab0, c0); - __m128i p01 = _mm_unpackhi_epi16(ab0, c0); - __m128i p02 = _mm_unpacklo_epi16(ab1, c1); - __m128i p03 = _mm_unpackhi_epi16(ab1, c1); - - __m128i p10 = _mm_unpacklo_epi32(p00, p01); - __m128i p11 = _mm_unpackhi_epi32(p00, p01); - __m128i p12 = _mm_unpacklo_epi32(p02, p03); - __m128i p13 = _mm_unpackhi_epi32(p02, p03); - - __m128i p20 = _mm_unpacklo_epi64(p10, p11); - __m128i p21 = _mm_unpackhi_epi64(p10, p11); - __m128i p22 = _mm_unpacklo_epi64(p12, p13); - __m128i p23 = _mm_unpackhi_epi64(p12, p13); - - p20 = _mm_slli_si128(p20, 1); - p22 = _mm_slli_si128(p22, 1); - - __m128i p30 = _mm_slli_epi64(_mm_unpacklo_epi32(p20, p21), 8); - __m128i p31 = _mm_srli_epi64(_mm_unpackhi_epi32(p20, p21), 8); - __m128i p32 = _mm_slli_epi64(_mm_unpacklo_epi32(p22, p23), 8); - __m128i p33 = _mm_srli_epi64(_mm_unpackhi_epi32(p22, p23), 8); - - __m128i p40 = _mm_unpacklo_epi64(p30, p31); - __m128i p41 = _mm_unpackhi_epi64(p30, p31); - __m128i p42 = _mm_unpacklo_epi64(p32, p33); - __m128i p43 = _mm_unpackhi_epi64(p32, p33); - - __m128i v0 = _mm_or_si128(_mm_srli_si128(p40, 2), _mm_slli_si128(p41, 10)); - __m128i v1 = _mm_or_si128(_mm_srli_si128(p41, 6), _mm_slli_si128(p42, 6)); - __m128i v2 = _mm_or_si128(_mm_srli_si128(p42, 10), _mm_slli_si128(p43, 2)); -#endif - - if ( mode == hal::STORE_ALIGNED_NOCACHE ) - { - _mm_stream_si128((__m128i*)(ptr), v0); - _mm_stream_si128((__m128i*)(ptr + 16), v1); - _mm_stream_si128((__m128i*)(ptr + 32), v2); - } - else if ( mode == hal::STORE_ALIGNED ) - { - _mm_store_si128((__m128i*)(ptr), v0); - _mm_store_si128((__m128i*)(ptr + 16), v1); - _mm_store_si128((__m128i*)(ptr + 32), v2); - } - else - { - _mm_storeu_si128((__m128i*)(ptr), v0); - _mm_storeu_si128((__m128i*)(ptr + 16), v1); - _mm_storeu_si128((__m128i*)(ptr + 32), v2); - } -} - -inline void v_store_interleave( uchar* ptr, const v_uint8x16& a, const v_uint8x16& b, - const v_uint8x16& c, const v_uint8x16& d, - hal::StoreMode mode = hal::STORE_UNALIGNED) -{ - // a0 a1 a2 a3 .... - // b0 b1 b2 b3 .... - // c0 c1 c2 c3 .... - // d0 d1 d2 d3 .... - __m128i u0 = _mm_unpacklo_epi8(a.val, c.val); // a0 c0 a1 c1 ... - __m128i u1 = _mm_unpackhi_epi8(a.val, c.val); // a8 c8 a9 c9 ... - __m128i u2 = _mm_unpacklo_epi8(b.val, d.val); // b0 d0 b1 d1 ... - __m128i u3 = _mm_unpackhi_epi8(b.val, d.val); // b8 d8 b9 d9 ... - - __m128i v0 = _mm_unpacklo_epi8(u0, u2); // a0 b0 c0 d0 ... - __m128i v1 = _mm_unpackhi_epi8(u0, u2); // a4 b4 c4 d4 ... - __m128i v2 = _mm_unpacklo_epi8(u1, u3); // a8 b8 c8 d8 ... - __m128i v3 = _mm_unpackhi_epi8(u1, u3); // a12 b12 c12 d12 ... - - if ( mode == hal::STORE_ALIGNED_NOCACHE ) - { - _mm_stream_si128((__m128i*)(ptr), v0); - _mm_stream_si128((__m128i*)(ptr + 16), v1); - _mm_stream_si128((__m128i*)(ptr + 32), v2); - _mm_stream_si128((__m128i*)(ptr + 48), v3); - } - else if ( mode == hal::STORE_ALIGNED ) - { - _mm_store_si128((__m128i*)(ptr), v0); - _mm_store_si128((__m128i*)(ptr + 16), v1); - _mm_store_si128((__m128i*)(ptr + 32), v2); - _mm_store_si128((__m128i*)(ptr + 48), v3); - } - else - { - _mm_storeu_si128((__m128i*)(ptr), v0); - _mm_storeu_si128((__m128i*)(ptr + 16), v1); - _mm_storeu_si128((__m128i*)(ptr + 32), v2); - _mm_storeu_si128((__m128i*)(ptr + 48), v3); - } -} - -inline void v_store_interleave( ushort* ptr, const v_uint16x8& a, const v_uint16x8& b, - hal::StoreMode mode = hal::STORE_UNALIGNED) -{ - __m128i v0 = _mm_unpacklo_epi16(a.val, b.val); - __m128i v1 = _mm_unpackhi_epi16(a.val, b.val); - - if ( mode == hal::STORE_ALIGNED_NOCACHE ) - { - _mm_stream_si128((__m128i*)(ptr), v0); - _mm_stream_si128((__m128i*)(ptr + 8), v1); - } - else if ( mode == hal::STORE_ALIGNED ) - { - _mm_store_si128((__m128i*)(ptr), v0); - _mm_store_si128((__m128i*)(ptr + 8), v1); - } - else - { - _mm_storeu_si128((__m128i*)(ptr), v0); - _mm_storeu_si128((__m128i*)(ptr + 8), v1); - } -} - -inline void v_store_interleave( ushort* ptr, const v_uint16x8& a, - const v_uint16x8& b, const v_uint16x8& c, - hal::StoreMode mode = hal::STORE_UNALIGNED) -{ -#if CV_SSE4_1 - const __m128i sh_a = _mm_setr_epi8(0, 1, 6, 7, 12, 13, 2, 3, 8, 9, 14, 15, 4, 5, 10, 11); - const __m128i sh_b = _mm_setr_epi8(10, 11, 0, 1, 6, 7, 12, 13, 2, 3, 8, 9, 14, 15, 4, 5); - const __m128i sh_c = _mm_setr_epi8(4, 5, 10, 11, 0, 1, 6, 7, 12, 13, 2, 3, 8, 9, 14, 15); - __m128i a0 = _mm_shuffle_epi8(a.val, sh_a); - __m128i b0 = _mm_shuffle_epi8(b.val, sh_b); - __m128i c0 = _mm_shuffle_epi8(c.val, sh_c); - - __m128i v0 = _mm_blend_epi16(_mm_blend_epi16(a0, b0, 0x92), c0, 0x24); - __m128i v1 = _mm_blend_epi16(_mm_blend_epi16(c0, a0, 0x92), b0, 0x24); - __m128i v2 = _mm_blend_epi16(_mm_blend_epi16(b0, c0, 0x92), a0, 0x24); -#else - __m128i z = _mm_setzero_si128(); - __m128i ab0 = _mm_unpacklo_epi16(a.val, b.val); - __m128i ab1 = _mm_unpackhi_epi16(a.val, b.val); - __m128i c0 = _mm_unpacklo_epi16(c.val, z); - __m128i c1 = _mm_unpackhi_epi16(c.val, z); - - __m128i p10 = _mm_unpacklo_epi32(ab0, c0); - __m128i p11 = _mm_unpackhi_epi32(ab0, c0); - __m128i p12 = _mm_unpacklo_epi32(ab1, c1); - __m128i p13 = _mm_unpackhi_epi32(ab1, c1); - - __m128i p20 = _mm_unpacklo_epi64(p10, p11); - __m128i p21 = _mm_unpackhi_epi64(p10, p11); - __m128i p22 = _mm_unpacklo_epi64(p12, p13); - __m128i p23 = _mm_unpackhi_epi64(p12, p13); - - p20 = _mm_slli_si128(p20, 2); - p22 = _mm_slli_si128(p22, 2); - - __m128i p30 = _mm_unpacklo_epi64(p20, p21); - __m128i p31 = _mm_unpackhi_epi64(p20, p21); - __m128i p32 = _mm_unpacklo_epi64(p22, p23); - __m128i p33 = _mm_unpackhi_epi64(p22, p23); - - __m128i v0 = _mm_or_si128(_mm_srli_si128(p30, 2), _mm_slli_si128(p31, 10)); - __m128i v1 = _mm_or_si128(_mm_srli_si128(p31, 6), _mm_slli_si128(p32, 6)); - __m128i v2 = _mm_or_si128(_mm_srli_si128(p32, 10), _mm_slli_si128(p33, 2)); -#endif - if ( mode == hal::STORE_ALIGNED_NOCACHE ) - { - _mm_stream_si128((__m128i*)(ptr), v0); - _mm_stream_si128((__m128i*)(ptr + 8), v1); - _mm_stream_si128((__m128i*)(ptr + 16), v2); - } - else if ( mode == hal::STORE_ALIGNED ) - { - _mm_store_si128((__m128i*)(ptr), v0); - _mm_store_si128((__m128i*)(ptr + 8), v1); - _mm_store_si128((__m128i*)(ptr + 16), v2); - } - else - { - _mm_storeu_si128((__m128i*)(ptr), v0); - _mm_storeu_si128((__m128i*)(ptr + 8), v1); - _mm_storeu_si128((__m128i*)(ptr + 16), v2); - } -} - -inline void v_store_interleave( ushort* ptr, const v_uint16x8& a, const v_uint16x8& b, - const v_uint16x8& c, const v_uint16x8& d, - hal::StoreMode mode = hal::STORE_UNALIGNED) -{ - // a0 a1 a2 a3 .... - // b0 b1 b2 b3 .... - // c0 c1 c2 c3 .... - // d0 d1 d2 d3 .... - __m128i u0 = _mm_unpacklo_epi16(a.val, c.val); // a0 c0 a1 c1 ... - __m128i u1 = _mm_unpackhi_epi16(a.val, c.val); // a4 c4 a5 c5 ... - __m128i u2 = _mm_unpacklo_epi16(b.val, d.val); // b0 d0 b1 d1 ... - __m128i u3 = _mm_unpackhi_epi16(b.val, d.val); // b4 d4 b5 d5 ... - - __m128i v0 = _mm_unpacklo_epi16(u0, u2); // a0 b0 c0 d0 ... - __m128i v1 = _mm_unpackhi_epi16(u0, u2); // a2 b2 c2 d2 ... - __m128i v2 = _mm_unpacklo_epi16(u1, u3); // a4 b4 c4 d4 ... - __m128i v3 = _mm_unpackhi_epi16(u1, u3); // a6 b6 c6 d6 ... - - if ( mode == hal::STORE_ALIGNED_NOCACHE ) - { - _mm_stream_si128((__m128i*)(ptr), v0); - _mm_stream_si128((__m128i*)(ptr + 8), v1); - _mm_stream_si128((__m128i*)(ptr + 16), v2); - _mm_stream_si128((__m128i*)(ptr + 24), v3); - } - else if ( mode == hal::STORE_ALIGNED ) - { - _mm_store_si128((__m128i*)(ptr), v0); - _mm_store_si128((__m128i*)(ptr + 8), v1); - _mm_store_si128((__m128i*)(ptr + 16), v2); - _mm_store_si128((__m128i*)(ptr + 24), v3); - } - else - { - _mm_storeu_si128((__m128i*)(ptr), v0); - _mm_storeu_si128((__m128i*)(ptr + 8), v1); - _mm_storeu_si128((__m128i*)(ptr + 16), v2); - _mm_storeu_si128((__m128i*)(ptr + 24), v3); - } -} - -inline void v_store_interleave( unsigned* ptr, const v_uint32x4& a, const v_uint32x4& b, - hal::StoreMode mode = hal::STORE_UNALIGNED) -{ - __m128i v0 = _mm_unpacklo_epi32(a.val, b.val); - __m128i v1 = _mm_unpackhi_epi32(a.val, b.val); - - if ( mode == hal::STORE_ALIGNED_NOCACHE ) - { - _mm_stream_si128((__m128i*)(ptr), v0); - _mm_stream_si128((__m128i*)(ptr + 4), v1); - } - else if ( mode == hal::STORE_ALIGNED ) - { - _mm_store_si128((__m128i*)(ptr), v0); - _mm_store_si128((__m128i*)(ptr + 4), v1); - } - else - { - _mm_storeu_si128((__m128i*)(ptr), v0); - _mm_storeu_si128((__m128i*)(ptr + 4), v1); - } -} - -inline void v_store_interleave( unsigned* ptr, const v_uint32x4& a, const v_uint32x4& b, - const v_uint32x4& c, hal::StoreMode mode = hal::STORE_UNALIGNED) -{ - v_uint32x4 z = v_setzero_u32(), u0, u1, u2, u3; - v_transpose4x4(a, b, c, z, u0, u1, u2, u3); - - __m128i v0 = _mm_or_si128(u0.val, _mm_slli_si128(u1.val, 12)); - __m128i v1 = _mm_or_si128(_mm_srli_si128(u1.val, 4), _mm_slli_si128(u2.val, 8)); - __m128i v2 = _mm_or_si128(_mm_srli_si128(u2.val, 8), _mm_slli_si128(u3.val, 4)); - - if ( mode == hal::STORE_ALIGNED_NOCACHE ) - { - _mm_stream_si128((__m128i*)(ptr), v0); - _mm_stream_si128((__m128i*)(ptr + 4), v1); - _mm_stream_si128((__m128i*)(ptr + 8), v2); - } - else if ( mode == hal::STORE_ALIGNED ) - { - _mm_store_si128((__m128i*)(ptr), v0); - _mm_store_si128((__m128i*)(ptr + 4), v1); - _mm_store_si128((__m128i*)(ptr + 8), v2); - } - else - { - _mm_storeu_si128((__m128i*)(ptr), v0); - _mm_storeu_si128((__m128i*)(ptr + 4), v1); - _mm_storeu_si128((__m128i*)(ptr + 8), v2); - } -} - -inline void v_store_interleave(unsigned* ptr, const v_uint32x4& a, const v_uint32x4& b, - const v_uint32x4& c, const v_uint32x4& d, - hal::StoreMode mode = hal::STORE_UNALIGNED) -{ - v_uint32x4 v0, v1, v2, v3; - v_transpose4x4(a, b, c, d, v0, v1, v2, v3); - - if ( mode == hal::STORE_ALIGNED_NOCACHE ) - { - _mm_stream_si128((__m128i*)(ptr), v0.val); - _mm_stream_si128((__m128i*)(ptr + 4), v1.val); - _mm_stream_si128((__m128i*)(ptr + 8), v2.val); - _mm_stream_si128((__m128i*)(ptr + 12), v3.val); - } - else if ( mode == hal::STORE_ALIGNED ) - { - _mm_store_si128((__m128i*)(ptr), v0.val); - _mm_store_si128((__m128i*)(ptr + 4), v1.val); - _mm_store_si128((__m128i*)(ptr + 8), v2.val); - _mm_store_si128((__m128i*)(ptr + 12), v3.val); - } - else - { - _mm_storeu_si128((__m128i*)(ptr), v0.val); - _mm_storeu_si128((__m128i*)(ptr + 4), v1.val); - _mm_storeu_si128((__m128i*)(ptr + 8), v2.val); - _mm_storeu_si128((__m128i*)(ptr + 12), v3.val); - } -} - -// 2-channel, float only -inline void v_store_interleave(float* ptr, const v_float32x4& a, const v_float32x4& b, - hal::StoreMode mode = hal::STORE_UNALIGNED) -{ - __m128 v0 = _mm_unpacklo_ps(a.val, b.val); // a0 b0 a1 b1 - __m128 v1 = _mm_unpackhi_ps(a.val, b.val); // a2 b2 a3 b3 - - if ( mode == hal::STORE_ALIGNED_NOCACHE ) - { - _mm_stream_ps(ptr, v0); - _mm_stream_ps(ptr + 4, v1); - } - else if ( mode == hal::STORE_ALIGNED ) - { - _mm_store_ps(ptr, v0); - _mm_store_ps(ptr + 4, v1); - } - else - { - _mm_storeu_ps(ptr, v0); - _mm_storeu_ps(ptr + 4, v1); - } -} - -inline void v_store_interleave(float* ptr, const v_float32x4& a, const v_float32x4& b, - const v_float32x4& c, hal::StoreMode mode = hal::STORE_UNALIGNED) -{ - __m128 u0 = _mm_shuffle_ps(a.val, b.val, _MM_SHUFFLE(0, 0, 0, 0)); - __m128 u1 = _mm_shuffle_ps(c.val, a.val, _MM_SHUFFLE(1, 1, 0, 0)); - __m128 v0 = _mm_shuffle_ps(u0, u1, _MM_SHUFFLE(2, 0, 2, 0)); - __m128 u2 = _mm_shuffle_ps(b.val, c.val, _MM_SHUFFLE(1, 1, 1, 1)); - __m128 u3 = _mm_shuffle_ps(a.val, b.val, _MM_SHUFFLE(2, 2, 2, 2)); - __m128 v1 = _mm_shuffle_ps(u2, u3, _MM_SHUFFLE(2, 0, 2, 0)); - __m128 u4 = _mm_shuffle_ps(c.val, a.val, _MM_SHUFFLE(3, 3, 2, 2)); - __m128 u5 = _mm_shuffle_ps(b.val, c.val, _MM_SHUFFLE(3, 3, 3, 3)); - __m128 v2 = _mm_shuffle_ps(u4, u5, _MM_SHUFFLE(2, 0, 2, 0)); - - if ( mode == hal::STORE_ALIGNED_NOCACHE ) - { - _mm_stream_ps(ptr, v0); - _mm_stream_ps(ptr + 4, v1); - _mm_stream_ps(ptr + 8, v2); - } - else if ( mode == hal::STORE_ALIGNED ) - { - _mm_store_ps(ptr, v0); - _mm_store_ps(ptr + 4, v1); - _mm_store_ps(ptr + 8, v2); - } - else - { - _mm_storeu_ps(ptr, v0); - _mm_storeu_ps(ptr + 4, v1); - _mm_storeu_ps(ptr + 8, v2); - } -} - -inline void v_store_interleave(float* ptr, const v_float32x4& a, const v_float32x4& b, - const v_float32x4& c, const v_float32x4& d, - hal::StoreMode mode = hal::STORE_UNALIGNED) -{ - __m128 u0 = _mm_unpacklo_ps(a.val, c.val); - __m128 u1 = _mm_unpacklo_ps(b.val, d.val); - __m128 u2 = _mm_unpackhi_ps(a.val, c.val); - __m128 u3 = _mm_unpackhi_ps(b.val, d.val); - __m128 v0 = _mm_unpacklo_ps(u0, u1); - __m128 v2 = _mm_unpacklo_ps(u2, u3); - __m128 v1 = _mm_unpackhi_ps(u0, u1); - __m128 v3 = _mm_unpackhi_ps(u2, u3); - - if ( mode == hal::STORE_ALIGNED_NOCACHE ) - { - _mm_stream_ps(ptr, v0); - _mm_stream_ps(ptr + 4, v1); - _mm_stream_ps(ptr + 8, v2); - _mm_stream_ps(ptr + 12, v3); - } - else if ( mode == hal::STORE_ALIGNED ) - { - _mm_store_ps(ptr, v0); - _mm_store_ps(ptr + 4, v1); - _mm_store_ps(ptr + 8, v2); - _mm_store_ps(ptr + 12, v3); - } - else - { - _mm_storeu_ps(ptr, v0); - _mm_storeu_ps(ptr + 4, v1); - _mm_storeu_ps(ptr + 8, v2); - _mm_storeu_ps(ptr + 12, v3); - } -} - -inline void v_store_interleave(uint64 *ptr, const v_uint64x2& a, const v_uint64x2& b, - hal::StoreMode mode = hal::STORE_UNALIGNED) -{ - __m128i v0 = _mm_unpacklo_epi64(a.val, b.val); - __m128i v1 = _mm_unpackhi_epi64(a.val, b.val); - - if ( mode == hal::STORE_ALIGNED_NOCACHE ) - { - _mm_stream_si128((__m128i*)(ptr), v0); - _mm_stream_si128((__m128i*)(ptr + 2), v1); - } - else if ( mode == hal::STORE_ALIGNED ) - { - _mm_store_si128((__m128i*)(ptr), v0); - _mm_store_si128((__m128i*)(ptr + 2), v1); - } - else - { - _mm_storeu_si128((__m128i*)(ptr), v0); - _mm_storeu_si128((__m128i*)(ptr + 2), v1); - } -} - -inline void v_store_interleave(uint64 *ptr, const v_uint64x2& a, const v_uint64x2& b, - const v_uint64x2& c, hal::StoreMode mode = hal::STORE_UNALIGNED) -{ - __m128i v0 = _mm_unpacklo_epi64(a.val, b.val); - __m128i v1 = _mm_unpacklo_epi64(c.val, _mm_unpackhi_epi64(a.val, a.val)); - __m128i v2 = _mm_unpackhi_epi64(b.val, c.val); - - if ( mode == hal::STORE_ALIGNED_NOCACHE ) - { - _mm_stream_si128((__m128i*)(ptr), v0); - _mm_stream_si128((__m128i*)(ptr + 2), v1); - _mm_stream_si128((__m128i*)(ptr + 4), v2); - } - else if ( mode == hal::STORE_ALIGNED ) - { - _mm_store_si128((__m128i*)(ptr), v0); - _mm_store_si128((__m128i*)(ptr + 2), v1); - _mm_store_si128((__m128i*)(ptr + 4), v2); - } - else - { - _mm_storeu_si128((__m128i*)(ptr), v0); - _mm_storeu_si128((__m128i*)(ptr + 2), v1); - _mm_storeu_si128((__m128i*)(ptr + 4), v2); - } -} - -inline void v_store_interleave(uint64 *ptr, const v_uint64x2& a, const v_uint64x2& b, - const v_uint64x2& c, const v_uint64x2& d, - hal::StoreMode mode = hal::STORE_UNALIGNED) -{ - __m128i v0 = _mm_unpacklo_epi64(a.val, b.val); - __m128i v1 = _mm_unpacklo_epi64(c.val, d.val); - __m128i v2 = _mm_unpackhi_epi64(a.val, b.val); - __m128i v3 = _mm_unpackhi_epi64(c.val, d.val); - - if ( mode == hal::STORE_ALIGNED_NOCACHE ) - { - _mm_stream_si128((__m128i*)(ptr), v0); - _mm_stream_si128((__m128i*)(ptr + 2), v1); - _mm_stream_si128((__m128i*)(ptr + 4), v2); - _mm_stream_si128((__m128i*)(ptr + 6), v3); - } - else if ( mode == hal::STORE_ALIGNED ) - { - _mm_store_si128((__m128i*)(ptr), v0); - _mm_store_si128((__m128i*)(ptr + 2), v1); - _mm_store_si128((__m128i*)(ptr + 4), v2); - _mm_store_si128((__m128i*)(ptr + 6), v3); - } - else - { - _mm_storeu_si128((__m128i*)(ptr), v0); - _mm_storeu_si128((__m128i*)(ptr + 2), v1); - _mm_storeu_si128((__m128i*)(ptr + 4), v2); - _mm_storeu_si128((__m128i*)(ptr + 6), v3); - } -} - -#define OPENCV_HAL_IMPL_SSE_LOADSTORE_INTERLEAVE(_Tpvec0, _Tp0, suffix0, _Tpvec1, _Tp1, suffix1) \ -inline void v_load_deinterleave( const _Tp0* ptr, _Tpvec0& a0, _Tpvec0& b0 ) \ -{ \ - _Tpvec1 a1, b1; \ - v_load_deinterleave((const _Tp1*)ptr, a1, b1); \ - a0 = v_reinterpret_as_##suffix0(a1); \ - b0 = v_reinterpret_as_##suffix0(b1); \ -} \ -inline void v_load_deinterleave( const _Tp0* ptr, _Tpvec0& a0, _Tpvec0& b0, _Tpvec0& c0 ) \ -{ \ - _Tpvec1 a1, b1, c1; \ - v_load_deinterleave((const _Tp1*)ptr, a1, b1, c1); \ - a0 = v_reinterpret_as_##suffix0(a1); \ - b0 = v_reinterpret_as_##suffix0(b1); \ - c0 = v_reinterpret_as_##suffix0(c1); \ -} \ -inline void v_load_deinterleave( const _Tp0* ptr, _Tpvec0& a0, _Tpvec0& b0, _Tpvec0& c0, _Tpvec0& d0 ) \ -{ \ - _Tpvec1 a1, b1, c1, d1; \ - v_load_deinterleave((const _Tp1*)ptr, a1, b1, c1, d1); \ - a0 = v_reinterpret_as_##suffix0(a1); \ - b0 = v_reinterpret_as_##suffix0(b1); \ - c0 = v_reinterpret_as_##suffix0(c1); \ - d0 = v_reinterpret_as_##suffix0(d1); \ -} \ -inline void v_store_interleave( _Tp0* ptr, const _Tpvec0& a0, const _Tpvec0& b0, \ - hal::StoreMode mode = hal::STORE_UNALIGNED ) \ -{ \ - _Tpvec1 a1 = v_reinterpret_as_##suffix1(a0); \ - _Tpvec1 b1 = v_reinterpret_as_##suffix1(b0); \ - v_store_interleave((_Tp1*)ptr, a1, b1, mode); \ -} \ -inline void v_store_interleave( _Tp0* ptr, const _Tpvec0& a0, const _Tpvec0& b0, \ - const _Tpvec0& c0, hal::StoreMode mode = hal::STORE_UNALIGNED ) \ -{ \ - _Tpvec1 a1 = v_reinterpret_as_##suffix1(a0); \ - _Tpvec1 b1 = v_reinterpret_as_##suffix1(b0); \ - _Tpvec1 c1 = v_reinterpret_as_##suffix1(c0); \ - v_store_interleave((_Tp1*)ptr, a1, b1, c1, mode); \ -} \ -inline void v_store_interleave( _Tp0* ptr, const _Tpvec0& a0, const _Tpvec0& b0, \ - const _Tpvec0& c0, const _Tpvec0& d0, \ - hal::StoreMode mode = hal::STORE_UNALIGNED ) \ -{ \ - _Tpvec1 a1 = v_reinterpret_as_##suffix1(a0); \ - _Tpvec1 b1 = v_reinterpret_as_##suffix1(b0); \ - _Tpvec1 c1 = v_reinterpret_as_##suffix1(c0); \ - _Tpvec1 d1 = v_reinterpret_as_##suffix1(d0); \ - v_store_interleave((_Tp1*)ptr, a1, b1, c1, d1, mode); \ -} - -OPENCV_HAL_IMPL_SSE_LOADSTORE_INTERLEAVE(v_int8x16, schar, s8, v_uint8x16, uchar, u8) -OPENCV_HAL_IMPL_SSE_LOADSTORE_INTERLEAVE(v_int16x8, short, s16, v_uint16x8, ushort, u16) -OPENCV_HAL_IMPL_SSE_LOADSTORE_INTERLEAVE(v_int32x4, int, s32, v_uint32x4, unsigned, u32) -OPENCV_HAL_IMPL_SSE_LOADSTORE_INTERLEAVE(v_int64x2, int64, s64, v_uint64x2, uint64, u64) -OPENCV_HAL_IMPL_SSE_LOADSTORE_INTERLEAVE(v_float64x2, double, f64, v_uint64x2, uint64, u64) - -inline v_float32x4 v_cvt_f32(const v_int32x4& a) -{ - return v_float32x4(_mm_cvtepi32_ps(a.val)); -} - -inline v_float32x4 v_cvt_f32(const v_float64x2& a) -{ - return v_float32x4(_mm_cvtpd_ps(a.val)); -} - -inline v_float32x4 v_cvt_f32(const v_float64x2& a, const v_float64x2& b) -{ - return v_float32x4(_mm_movelh_ps(_mm_cvtpd_ps(a.val), _mm_cvtpd_ps(b.val))); -} - -inline v_float64x2 v_cvt_f64(const v_int32x4& a) -{ - return v_float64x2(_mm_cvtepi32_pd(a.val)); -} - -inline v_float64x2 v_cvt_f64_high(const v_int32x4& a) -{ - return v_float64x2(_mm_cvtepi32_pd(_mm_srli_si128(a.val,8))); -} - -inline v_float64x2 v_cvt_f64(const v_float32x4& a) -{ - return v_float64x2(_mm_cvtps_pd(a.val)); -} - -inline v_float64x2 v_cvt_f64_high(const v_float32x4& a) -{ - return v_float64x2(_mm_cvtps_pd(_mm_movehl_ps(a.val, a.val))); -} - -#if CV_FP16 -inline v_float32x4 v128_load_fp16_f32(const short* ptr) -{ - return v_float32x4(_mm_cvtph_ps(_mm_loadu_si128((const __m128i*)ptr))); -} - -inline void v_store_fp16(short* ptr, const v_float32x4& a) -{ - __m128i fp16_value = _mm_cvtps_ph(a.val, 0); - _mm_storel_epi64((__m128i*)ptr, fp16_value); -} -#endif - -////////////// Lookup table access //////////////////// -inline v_int8x16 v_lut(const schar* tab, const int* idx) -{ -#if defined(_MSC_VER) - return v_int8x16(_mm_setr_epi8(tab[idx[0]], tab[idx[1]], tab[idx[ 2]], tab[idx[ 3]], tab[idx[ 4]], tab[idx[ 5]], tab[idx[ 6]], tab[idx[ 7]], - tab[idx[8]], tab[idx[9]], tab[idx[10]], tab[idx[11]], tab[idx[12]], tab[idx[13]], tab[idx[14]], tab[idx[15]])); -#else - return v_int8x16(_mm_setr_epi64( - _mm_setr_pi8(tab[idx[0]], tab[idx[1]], tab[idx[ 2]], tab[idx[ 3]], tab[idx[ 4]], tab[idx[ 5]], tab[idx[ 6]], tab[idx[ 7]]), - _mm_setr_pi8(tab[idx[8]], tab[idx[9]], tab[idx[10]], tab[idx[11]], tab[idx[12]], tab[idx[13]], tab[idx[14]], tab[idx[15]]) - )); -#endif -} - -inline v_int8x16 v_lut_pairs(const schar* tab, const int* idx) -{ -#if defined(_MSC_VER) - return v_int8x16(_mm_setr_epi16(*(const short*)(tab + idx[0]), *(const short*)(tab + idx[1]), *(const short*)(tab + idx[2]), *(const short*)(tab + idx[3]), - *(const short*)(tab + idx[4]), *(const short*)(tab + idx[5]), *(const short*)(tab + idx[6]), *(const short*)(tab + idx[7]))); -#else - return v_int8x16(_mm_setr_epi64( - _mm_setr_pi16(*(const short*)(tab + idx[0]), *(const short*)(tab + idx[1]), *(const short*)(tab + idx[2]), *(const short*)(tab + idx[3])), - _mm_setr_pi16(*(const short*)(tab + idx[4]), *(const short*)(tab + idx[5]), *(const short*)(tab + idx[6]), *(const short*)(tab + idx[7])) - )); -#endif -} -inline v_int8x16 v_lut_quads(const schar* tab, const int* idx) -{ -#if defined(_MSC_VER) - return v_int8x16(_mm_setr_epi32(*(const int*)(tab + idx[0]), *(const int*)(tab + idx[1]), - *(const int*)(tab + idx[2]), *(const int*)(tab + idx[3]))); -#else - return v_int8x16(_mm_setr_epi64( - _mm_setr_pi32(*(const int*)(tab + idx[0]), *(const int*)(tab + idx[1])), - _mm_setr_pi32(*(const int*)(tab + idx[2]), *(const int*)(tab + idx[3])) - )); -#endif -} -inline v_uint8x16 v_lut(const uchar* tab, const int* idx) { return v_reinterpret_as_u8(v_lut((const schar *)tab, idx)); } -inline v_uint8x16 v_lut_pairs(const uchar* tab, const int* idx) { return v_reinterpret_as_u8(v_lut_pairs((const schar *)tab, idx)); } -inline v_uint8x16 v_lut_quads(const uchar* tab, const int* idx) { return v_reinterpret_as_u8(v_lut_quads((const schar *)tab, idx)); } - -inline v_int16x8 v_lut(const short* tab, const int* idx) -{ -#if defined(_MSC_VER) - return v_int16x8(_mm_setr_epi16(tab[idx[0]], tab[idx[1]], tab[idx[2]], tab[idx[3]], - tab[idx[4]], tab[idx[5]], tab[idx[6]], tab[idx[7]])); -#else - return v_int16x8(_mm_setr_epi64( - _mm_setr_pi16(tab[idx[0]], tab[idx[1]], tab[idx[2]], tab[idx[3]]), - _mm_setr_pi16(tab[idx[4]], tab[idx[5]], tab[idx[6]], tab[idx[7]]) - )); -#endif -} -inline v_int16x8 v_lut_pairs(const short* tab, const int* idx) -{ -#if defined(_MSC_VER) - return v_int16x8(_mm_setr_epi32(*(const int*)(tab + idx[0]), *(const int*)(tab + idx[1]), - *(const int*)(tab + idx[2]), *(const int*)(tab + idx[3]))); -#else - return v_int16x8(_mm_setr_epi64( - _mm_setr_pi32(*(const int*)(tab + idx[0]), *(const int*)(tab + idx[1])), - _mm_setr_pi32(*(const int*)(tab + idx[2]), *(const int*)(tab + idx[3])) - )); -#endif -} -inline v_int16x8 v_lut_quads(const short* tab, const int* idx) -{ - return v_int16x8(_mm_set_epi64x(*(const int64_t*)(tab + idx[1]), *(const int64_t*)(tab + idx[0]))); -} -inline v_uint16x8 v_lut(const ushort* tab, const int* idx) { return v_reinterpret_as_u16(v_lut((const short *)tab, idx)); } -inline v_uint16x8 v_lut_pairs(const ushort* tab, const int* idx) { return v_reinterpret_as_u16(v_lut_pairs((const short *)tab, idx)); } -inline v_uint16x8 v_lut_quads(const ushort* tab, const int* idx) { return v_reinterpret_as_u16(v_lut_quads((const short *)tab, idx)); } - -inline v_int32x4 v_lut(const int* tab, const int* idx) -{ -#if defined(_MSC_VER) - return v_int32x4(_mm_setr_epi32(tab[idx[0]], tab[idx[1]], - tab[idx[2]], tab[idx[3]])); -#else - return v_int32x4(_mm_setr_epi64( - _mm_setr_pi32(tab[idx[0]], tab[idx[1]]), - _mm_setr_pi32(tab[idx[2]], tab[idx[3]]) - )); -#endif -} -inline v_int32x4 v_lut_pairs(const int* tab, const int* idx) -{ - return v_int32x4(_mm_set_epi64x(*(const int64_t*)(tab + idx[1]), *(const int64_t*)(tab + idx[0]))); -} -inline v_int32x4 v_lut_quads(const int* tab, const int* idx) -{ - return v_int32x4(_mm_loadu_si128((const __m128i*)(tab + idx[0]))); -} -inline v_uint32x4 v_lut(const unsigned* tab, const int* idx) { return v_reinterpret_as_u32(v_lut((const int *)tab, idx)); } -inline v_uint32x4 v_lut_pairs(const unsigned* tab, const int* idx) { return v_reinterpret_as_u32(v_lut_pairs((const int *)tab, idx)); } -inline v_uint32x4 v_lut_quads(const unsigned* tab, const int* idx) { return v_reinterpret_as_u32(v_lut_quads((const int *)tab, idx)); } - -inline v_float32x4 v_lut(const float* tab, const int* idx) -{ - return v_float32x4(_mm_setr_ps(tab[idx[0]], tab[idx[1]], tab[idx[2]], tab[idx[3]])); -} -inline v_float32x4 v_lut_pairs(const float* tab, const int* idx) { return v_reinterpret_as_f32(v_lut_pairs((const int *)tab, idx)); } -inline v_float32x4 v_lut_quads(const float* tab, const int* idx) { return v_reinterpret_as_f32(v_lut_quads((const int *)tab, idx)); } - -inline v_float64x2 v_lut(const double* tab, const int* idx) -{ - return v_float64x2(_mm_setr_pd(tab[idx[0]], tab[idx[1]])); -} -inline v_float64x2 v_lut_pairs(const double* tab, const int* idx) { return v_float64x2(_mm_castsi128_pd(_mm_loadu_si128((const __m128i*)(tab + idx[0])))); } - -inline v_int64x2 v_lut(const int64_t* tab, const int* idx) -{ - return v_int64x2(_mm_set_epi64x(tab[idx[1]], tab[idx[0]])); -} -inline v_int64x2 v_lut_pairs(const int64_t* tab, const int* idx) -{ - return v_int64x2(_mm_loadu_si128((const __m128i*)(tab + idx[0]))); -} -inline v_uint64x2 v_lut(const uint64_t* tab, const int* idx) { return v_reinterpret_as_u64(v_lut((const int64_t *)tab, idx)); } -inline v_uint64x2 v_lut_pairs(const uint64_t* tab, const int* idx) { return v_reinterpret_as_u64(v_lut_pairs((const int64_t *)tab, idx)); } - -inline v_int32x4 v_lut(const int* tab, const v_int32x4& idxvec) -{ - int CV_DECL_ALIGNED(32) idx[4]; - v_store_aligned(idx, idxvec); - return v_int32x4(_mm_setr_epi32(tab[idx[0]], tab[idx[1]], tab[idx[2]], tab[idx[3]])); -} - -inline v_float32x4 v_lut(const float* tab, const v_int32x4& idxvec) -{ - int CV_DECL_ALIGNED(32) idx[4]; - v_store_aligned(idx, idxvec); - return v_float32x4(_mm_setr_ps(tab[idx[0]], tab[idx[1]], tab[idx[2]], tab[idx[3]])); -} - -inline v_float64x2 v_lut(const double* tab, const v_int32x4& idxvec) -{ - int idx[2]; - v_store_low(idx, idxvec); - return v_float64x2(_mm_setr_pd(tab[idx[0]], tab[idx[1]])); -} - -// loads pairs from the table and deinterleaves them, e.g. returns: -// x = (tab[idxvec[0], tab[idxvec[1]], tab[idxvec[2]], tab[idxvec[3]]), -// y = (tab[idxvec[0]+1], tab[idxvec[1]+1], tab[idxvec[2]+1], tab[idxvec[3]+1]) -// note that the indices are float's indices, not the float-pair indices. -// in theory, this function can be used to implement bilinear interpolation, -// when idxvec are the offsets within the image. -inline void v_lut_deinterleave(const float* tab, const v_int32x4& idxvec, v_float32x4& x, v_float32x4& y) -{ - int CV_DECL_ALIGNED(32) idx[4]; - v_store_aligned(idx, idxvec); - __m128 z = _mm_setzero_ps(); - __m128 xy01 = _mm_loadl_pi(z, (__m64*)(tab + idx[0])); - __m128 xy23 = _mm_loadl_pi(z, (__m64*)(tab + idx[2])); - xy01 = _mm_loadh_pi(xy01, (__m64*)(tab + idx[1])); - xy23 = _mm_loadh_pi(xy23, (__m64*)(tab + idx[3])); - __m128 xxyy02 = _mm_unpacklo_ps(xy01, xy23); - __m128 xxyy13 = _mm_unpackhi_ps(xy01, xy23); - x = v_float32x4(_mm_unpacklo_ps(xxyy02, xxyy13)); - y = v_float32x4(_mm_unpackhi_ps(xxyy02, xxyy13)); -} - -inline void v_lut_deinterleave(const double* tab, const v_int32x4& idxvec, v_float64x2& x, v_float64x2& y) -{ - int idx[2]; - v_store_low(idx, idxvec); - __m128d xy0 = _mm_loadu_pd(tab + idx[0]); - __m128d xy1 = _mm_loadu_pd(tab + idx[1]); - x = v_float64x2(_mm_unpacklo_pd(xy0, xy1)); - y = v_float64x2(_mm_unpackhi_pd(xy0, xy1)); -} - - -////////////// FP16 support /////////////////////////// -// Currently disabled (DM) -#if 0 -inline v_float32x4 v_load_expand(const float16_t* ptr) -{ - const __m128i z = _mm_setzero_si128(), delta = _mm_set1_epi32(0x38000000); - const __m128i signmask = _mm_set1_epi32(0x80000000), maxexp = _mm_set1_epi32(0x7c000000); - const __m128 deltaf = _mm_castsi128_ps(_mm_set1_epi32(0x38800000)); - __m128i bits = _mm_unpacklo_epi16(z, _mm_loadl_epi64((const __m128i*)ptr)); // h << 16 - __m128i e = _mm_and_si128(bits, maxexp), sign = _mm_and_si128(bits, signmask); - __m128i t = _mm_add_epi32(_mm_srli_epi32(_mm_xor_si128(bits, sign), 3), delta); // ((h & 0x7fff) << 13) + delta - __m128i zt = _mm_castps_si128(_mm_sub_ps(_mm_castsi128_ps(_mm_add_epi32(t, _mm_set1_epi32(1 << 23))), deltaf)); - - t = _mm_add_epi32(t, _mm_and_si128(delta, _mm_cmpeq_epi32(maxexp, e))); - __m128i zmask = _mm_cmpeq_epi32(e, z); - __m128i ft = v_select_si128(zmask, zt, t); - return v_float32x4(_mm_castsi128_ps(_mm_or_si128(ft, sign))); -} - -inline void v_pack_store(float16_t* ptr, const v_float32x4& v) -{ - const __m128i signmask = _mm_set1_epi32(0x80000000); - const __m128i rval = _mm_set1_epi32(0x3f000000); - - __m128i t = _mm_castps_si128(v.val); - __m128i sign = _mm_srai_epi32(_mm_and_si128(t, signmask), 16); - t = _mm_andnot_si128(signmask, t); - - __m128i finitemask = _mm_cmpgt_epi32(_mm_set1_epi32(0x47800000), t); - __m128i isnan = _mm_cmpgt_epi32(t, _mm_set1_epi32(0x7f800000)); - __m128i naninf = v_select_si128(isnan, _mm_set1_epi32(0x7e00), _mm_set1_epi32(0x7c00)); - __m128i tinymask = _mm_cmpgt_epi32(_mm_set1_epi32(0x38800000), t); - __m128i tt = _mm_castps_si128(_mm_add_ps(_mm_castsi128_ps(t), _mm_castsi128_ps(rval))); - tt = _mm_sub_epi32(tt, rval); - __m128i odd = _mm_and_si128(_mm_srli_epi32(t, 13), _mm_set1_epi32(1)); - __m128i nt = _mm_add_epi32(t, _mm_set1_epi32(0xc8000fff)); - nt = _mm_srli_epi32(_mm_add_epi32(nt, odd), 13); - t = v_select_si128(tinymask, tt, nt); - t = v_select_si128(finitemask, t, naninf); - t = _mm_or_si128(t, sign); - t = _mm_packs_epi32(t, t); - _mm_storel_epi64((__m128i*)ptr, t); -} -#endif - -inline void v_cleanup() {} - -//! @name Check SIMD support -//! @{ -//! @brief Check CPU capability of SIMD operation -static inline bool hasSIMD128() -{ - return (CV_CPU_HAS_SUPPORT_SSE2) ? true : false; -} - - -static inline void v_deinterleave(const v_float32x4& low, const v_float32x4& high, - v_float32x4& even, v_float32x4& odd) { - __m128 tmp0 = _mm_unpacklo_ps(low.val, high.val); - __m128 tmp1 = _mm_unpackhi_ps(low.val, high.val); - even.val = _mm_unpacklo_ps(tmp0, tmp1); - odd .val = _mm_unpackhi_ps(tmp0, tmp1); -} - -static inline void v_deinterleave(const v_uint8x16& i0, const v_uint8x16& i1, - const v_uint8x16& i2, const v_uint8x16& i3, - v_uint8x16& o0, v_uint8x16& o1, - v_uint8x16& o2, v_uint8x16& o3) { - __m128i u0 = i0.val; // a0 b0 c0 d0 a1 b1 c1 d1 ... - __m128i u1 = i1.val; // a4 b4 c4 d4 ... - __m128i u2 = i2.val; // a8 b8 c8 d8 ... - __m128i u3 = i3.val; // a12 b12 c12 d12 ... - - __m128i v0 = _mm_unpacklo_epi8(u0, u2); // a0 a8 b0 b8 ... - __m128i v1 = _mm_unpackhi_epi8(u0, u2); // a2 a10 b2 b10 ... - __m128i v2 = _mm_unpacklo_epi8(u1, u3); // a4 a12 b4 b12 ... - __m128i v3 = _mm_unpackhi_epi8(u1, u3); // a6 a14 b6 b14 ... - - u0 = _mm_unpacklo_epi8(v0, v2); // a0 a4 a8 a12 ... - u1 = _mm_unpacklo_epi8(v1, v3); // a2 a6 a10 a14 ... - u2 = _mm_unpackhi_epi8(v0, v2); // a1 a5 a9 a13 ... - u3 = _mm_unpackhi_epi8(v1, v3); // a3 a7 a11 a15 ... - - v0 = _mm_unpacklo_epi8(u0, u1); // a0 a2 a4 a6 ... - v1 = _mm_unpacklo_epi8(u2, u3); // a1 a3 a5 a7 ... - v2 = _mm_unpackhi_epi8(u0, u1); // c0 c2 c4 c6 ... - v3 = _mm_unpackhi_epi8(u2, u3); // c1 c3 c5 c7 ... - - o0.val = _mm_unpacklo_epi8(v0, v1); // a0 a1 a2 a3 ... - o1.val = _mm_unpackhi_epi8(v0, v1); // b0 b1 b2 b3 ... - o2.val = _mm_unpacklo_epi8(v2, v3); // c0 c1 c2 c3 ... - o3.val = _mm_unpackhi_epi8(v2, v3); // d0 d1 d2 d3 ... -} - -static inline v_uint8x16 v_interleave_low(const v_uint8x16& a, const v_uint8x16& b) { - return v_uint8x16(_mm_unpacklo_epi8(a.val, b.val)); -} - -static inline v_uint8x16 v_interleave_high(const v_uint8x16& a, const v_uint8x16& b) { - return v_uint8x16(_mm_unpackhi_epi8(a.val, b.val)); -} - -static inline v_int16x8 v_interleave_low(const v_int16x8& a, const v_int16x8& b) { - return v_int16x8(_mm_unpacklo_epi16(a.val, b.val)); -} - -static inline v_int16x8 v_interleave_high(const v_int16x8& a, const v_int16x8& b) { - return v_int16x8(_mm_unpackhi_epi16(a.val, b.val)); -} - -static inline v_uint8x16 v_saturate_u8(const v_int16x8& a) { - v_uint8x16 r; - r.val = _mm_packus_epi16(a.val, _mm_setzero_si128()); - return r; -} - -static inline v_int16x8 v_saturate_s16(const v_int32x4& a) { - v_int16x8 r; - r.val = _mm_packs_epi32(a.val, _mm_setzero_si128()); - return r; -} - -static inline v_uint8x16 v_packus_s16(const v_int16x8& a, const v_int16x8& b) { - v_uint8x16 r; - r.val = _mm_packus_epi16(a.val, b.val); - return r; -} - -// for each j=index[k], load two chars src[j] and src[j+1] -static inline v_uint8x16 v_gather_pairs(const uchar src[], const v_int16x8& index) { - v_uint8x16 r; - r.val = _mm_insert_epi16(r.val, *reinterpret_cast(&src[_mm_extract_epi16(index.val, 0)]), 0); - r.val = _mm_insert_epi16(r.val, *reinterpret_cast(&src[_mm_extract_epi16(index.val, 1)]), 1); - r.val = _mm_insert_epi16(r.val, *reinterpret_cast(&src[_mm_extract_epi16(index.val, 2)]), 2); - r.val = _mm_insert_epi16(r.val, *reinterpret_cast(&src[_mm_extract_epi16(index.val, 3)]), 3); - r.val = _mm_insert_epi16(r.val, *reinterpret_cast(&src[_mm_extract_epi16(index.val, 4)]), 4); - r.val = _mm_insert_epi16(r.val, *reinterpret_cast(&src[_mm_extract_epi16(index.val, 5)]), 5); - r.val = _mm_insert_epi16(r.val, *reinterpret_cast(&src[_mm_extract_epi16(index.val, 6)]), 6); - r.val = _mm_insert_epi16(r.val, *reinterpret_cast(&src[_mm_extract_epi16(index.val, 7)]), 7); - return r; -} - -namespace { - template - static inline v_int16x8 v_gather_chan(const uchar src[], const v_int16x8& index, int channel, int pos) { - v_int16x8 r; - r.val = _mm_insert_epi16(r.val, *reinterpret_cast(&src[chanNum*(_mm_extract_epi16(index.val, 0) + pos) + channel]), 0); - r.val = _mm_insert_epi16(r.val, *reinterpret_cast(&src[chanNum*(_mm_extract_epi16(index.val, 1) + pos) + channel]), 1); - r.val = _mm_insert_epi16(r.val, *reinterpret_cast(&src[chanNum*(_mm_extract_epi16(index.val, 2) + pos) + channel]), 2); - r.val = _mm_insert_epi16(r.val, *reinterpret_cast(&src[chanNum*(_mm_extract_epi16(index.val, 3) + pos) + channel]), 3); - r.val = _mm_insert_epi16(r.val, *reinterpret_cast(&src[chanNum*(_mm_extract_epi16(index.val, 4) + pos) + channel]), 4); - r.val = _mm_insert_epi16(r.val, *reinterpret_cast(&src[chanNum*(_mm_extract_epi16(index.val, 5) + pos) + channel]), 5); - r.val = _mm_insert_epi16(r.val, *reinterpret_cast(&src[chanNum*(_mm_extract_epi16(index.val, 6) + pos) + channel]), 6); - r.val = _mm_insert_epi16(r.val, *reinterpret_cast(&src[chanNum*(_mm_extract_epi16(index.val, 7) + pos) + channel]), 7); - return r; - } -} // namespace - -static inline void v_gather_pairs(const float src[], const int mapsx[], int x, - v_float32x4& low, v_float32x4& high) { - __m128 l = _mm_setzero_ps(); - l = _mm_loadl_pi(l, (const __m64*)&src[mapsx[x + 0]]); // pair of floats - l = _mm_loadh_pi(l, (const __m64*)&src[mapsx[x + 1]]); - low.val = l; - - __m128 h = _mm_setzero_ps(); - h = _mm_loadl_pi(h, (const __m64*)&src[mapsx[x + 2]]); - h = _mm_loadh_pi(h, (const __m64*)&src[mapsx[x + 3]]); - high.val = h; -} - -static inline v_int32x4 v_madd(const v_int16x8& a, const v_int16x8& b) { - v_int32x4 r; - r.val = _mm_madd_epi16(a.val, b.val); - return r; -} - -static inline v_int16x8 v_mulhi(const v_int16x8& a, short b) { - v_int16x8 r; - r.val = _mm_mulhi_epi16(a.val, _mm_set1_epi16(b)); - return r; -} - -static inline v_uint16x8 v_mulhi(const v_uint16x8& a, v_uint16x8 b) { - v_uint16x8 r; - r.val = _mm_mulhi_epu16(a.val, b.val); - return r; -} - -static inline v_uint16x8 v_mulhi(const v_uint16x8& a, uint16_t b) { - v_uint16x8 r; - r.val = _mm_mulhi_epu16(a.val, _mm_set1_epi16(b)); - return r; -} - -static inline v_int16x8 v_mulhrs(const v_int16x8& a, const v_int16x8& b) { - v_int16x8 r; - r.val = _mm_mulhrs_epi16(a.val, b.val); - return r; -} - -static inline v_int16x8 v_mulhrs(const v_int16x8& a, short b) { - return v_mulhrs(a, v_setall_s16(b)); -} - -#ifdef CV_SSE3 -static inline void v_deinterleave_expand(const v_uint8x16& src, v_int16x8& even, v_int16x8& odd) { - static const __m128i mask_even = _mm_setr_epi8(0, -1, 2, -1, 4, -1, 6, -1, 8, -1, 10, -1, 12, -1, 14, -1); - static const __m128i mask_odd = _mm_setr_epi8(1, -1, 3, -1, 5, -1, 7, -1, 9, -1, 11, -1, 13, -1, 15, -1); - even.val = _mm_shuffle_epi8(src.val, mask_even); - odd .val = _mm_shuffle_epi8(src.val, mask_odd); -} -#endif - -static inline v_float32x4 v_fma(const v_float32x4& a, float b, const v_float32x4& c) { - return v_fma(a, v_setall_f32(b), c); -} - -static inline v_int16x8 operator+ (const v_int16x8& a, short b) { - return a + v_setall_s16(b); -} - -static inline v_int16x8 operator- (short a, const v_int16x8& b) { - return v_setall_s16(a) - b; -} - -static inline v_float32x4 operator- (float a, const v_float32x4& b) { - return v_setall_f32(a) - b; -} - -static inline v_float32x4 operator* (const v_float32x4& a, float b) { - return a * v_setall_f32(b); -} - -template -static inline v_uint8x16 v_blend_shiftleft(const v_uint8x16& a, const v_uint8x16& b) { - v_uint8x16 res; - res.val = _mm_blend_epi16(a.val, _mm_slli_si128(b.val, shift), mask /*0xCC 0b11001100*/); - return res; -} - -template -static inline v_uint8x16 v_blend_shiftright(const v_uint8x16& a, const v_uint8x16& b) { - v_uint8x16 res; - res.val = _mm_blend_epi16(_mm_srli_si128(a.val, shift), b.val, mask /*0xCC 0b11001100*/); - return res; -} - -static inline v_uint8x16 v_setr_s8(char b0, char b1, char b2, char b3, char b4, - char b5, char b6, char b7, char b8, char b9, - char b10, char b11, char b12, char b13, char b14, - char b15) { - return v_uint8x16(_mm_setr_epi8(b0, b1, b2, b3, b4, b5, b6, b7, b8, - b9, b10, b11, b12, b13, b14, b15)); -} - - -static inline v_uint8x16 v_shuffle_s8(const v_uint8x16& a, const v_uint8x16& mask) -{ - return v_uint8x16(_mm_shuffle_epi8(a.val, mask.val)); -} - -static inline void v_gather_channel(v_uint8x16& vec, const uint8_t tmp[], const short mapsx[], - int chanNum, int c, int x, int shift) -{ - vec.val = _mm_insert_epi32(vec.val, *reinterpret_cast(&tmp[4 * (chanNum * mapsx[x + shift + 0] + c)]), 0); - vec.val = _mm_insert_epi32(vec.val, *reinterpret_cast(&tmp[4 * (chanNum * mapsx[x + shift + 1] + c)]), 1); - - vec.val = _mm_insert_epi32(vec.val, *reinterpret_cast(&tmp[4 * (chanNum * (mapsx[x + shift + 0] + 1) + c)]), 2); - vec.val = _mm_insert_epi32(vec.val, *reinterpret_cast(&tmp[4 * (chanNum * (mapsx[x + shift + 1] + 1) + c)]), 3); -} - -//! @} - -CV_CPU_OPTIMIZATION_HAL_NAMESPACE_END - -//! @endcond - -} // namespace cv - -#endif // OPENCV_HAL_SSE_HPP