diff --git a/.gitmodules b/.gitmodules
index cfeab60491d..45b449a116b 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -13,4 +13,8 @@
 [submodule "inference-engine/samples/thirdparty/gflags"]
 	path = inference-engine/samples/thirdparty/gflags
 	url = https://github.com/gflags/gflags.git
-	ignore = dirty
\ No newline at end of file
+	ignore = dirty
+[submodule "thirdparty/xbyak"]
+	path = thirdparty/xbyak
+	url = https://github.com/herumi/xbyak.git
+	ignore = dirty
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 105181b6f85..4b8c2631c79 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -108,6 +108,7 @@ function(build_ngraph)
     set(NGRAPH_COMPONENT_PREFIX "deployment_tools/ngraph/")
     add_subdirectory(ngraph)
     set(NGRAPH_LIBRARIES ngraph PARENT_SCOPE)
+    set(NGRAPH_REF_LIBRARIES ngraph_reference PARENT_SCOPE)
 endfunction()
 
 function(openvino_developer_export_targets)
@@ -156,6 +157,7 @@ add_subdirectory(docs)
 ie_shellcheck_process(DIRECTORY "${OpenVINO_MAIN_SOURCE_DIR}"
                       SKIP "${OpenVINO_MAIN_SOURCE_DIR}/bin"
                            "${OpenVINO_MAIN_SOURCE_DIR}/build"
+                           "${OpenVINO_MAIN_SOURCE_DIR}/thirdparty"
                            "${IE_MAIN_SOURCE_DIR}/tests/ie_test_utils/common_test_utils/gtest"
                            "${IE_MAIN_SOURCE_DIR}/samples/thirdparty"
                            "${IE_MAIN_SOURCE_DIR}/thirdparty"
diff --git a/inference-engine/src/inference_engine/CMakeLists.txt b/inference-engine/src/inference_engine/CMakeLists.txt
index 98328b91763..52278facade 100644
--- a/inference-engine/src/inference_engine/CMakeLists.txt
+++ b/inference-engine/src/inference_engine/CMakeLists.txt
@@ -109,7 +109,8 @@ target_compile_definitions(${TARGET_NAME}_obj PRIVATE IMPLEMENT_INFERENCE_ENGINE
                                                       $<TARGET_PROPERTY:ngraph::ngraph,INTERFACE_COMPILE_DEFINITIONS>)
 
 target_include_directories(${TARGET_NAME}_obj SYSTEM PRIVATE $<TARGET_PROPERTY:ngraph::ngraph,INTERFACE_INCLUDE_DIRECTORIES>
-                                                             $<TARGET_PROPERTY:pugixml,INTERFACE_INCLUDE_DIRECTORIES>)
+                                                             $<TARGET_PROPERTY:pugixml,INTERFACE_INCLUDE_DIRECTORIES>
+                                                             $<TARGET_PROPERTY:xbyak,INTERFACE_INCLUDE_DIRECTORIES>)
 
 target_include_directories(${TARGET_NAME}_obj PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}"
                                                       "${IE_MAIN_SOURCE_DIR}/src/readers/ir_reader" # for ie_ir_version.hpp
@@ -119,10 +120,6 @@ target_include_directories(${TARGET_NAME}_obj PRIVATE "${CMAKE_CURRENT_SOURCE_DI
 
 target_link_libraries(${TARGET_NAME}_obj PRIVATE ${TARGET_NAME}_reader_api)
 
-if(ENABLE_MKL_DNN)
-    target_include_directories(${TARGET_NAME}_obj SYSTEM PRIVATE "${IE_MAIN_SOURCE_DIR}/thirdparty/mkl-dnn/src/cpu/x64/xbyak")
-endif()
-
 set_ie_threading_interface_for(${TARGET_NAME}_obj)
 
 add_cpplint_target(${TARGET_NAME}_cpplint FOR_TARGETS ${TARGET_NAME}_obj)
@@ -215,10 +212,6 @@ configure_file("${IE_MAIN_SOURCE_DIR}/cmake/templates/InferenceEngineConfig-vers
 
 # Export for developer package
 
-add_library(xbyak INTERFACE)
-target_include_directories(xbyak INTERFACE ${IE_MAIN_SOURCE_DIR}/thirdparty/mkl-dnn/src/cpu/xbyak)
-
-openvino_developer_export_targets(COMPONENT openvino_common TARGETS xbyak)
 ie_developer_export_targets(${TARGET_NAME} ${TARGET_NAME}_plugin_api)
 
 # install TBB
diff --git a/inference-engine/src/inference_engine/ie_system_conf.cpp b/inference-engine/src/inference_engine/ie_system_conf.cpp
index 5048d89eb43..45dc2bade6f 100644
--- a/inference-engine/src/inference_engine/ie_system_conf.cpp
+++ b/inference-engine/src/inference_engine/ie_system_conf.cpp
@@ -9,89 +9,41 @@
 #include <iostream>
 #include <vector>
 
-#ifdef ENABLE_MKL_DNN
 # define XBYAK_NO_OP_NAMES
 # define XBYAK_UNDEF_JNL
-# include <xbyak_util.h>
-#endif
+# include <xbyak/xbyak_util.h>
 
 namespace InferenceEngine {
 
-#ifdef ENABLE_MKL_DNN
 static Xbyak::util::Cpu& get_cpu_info() {
     static Xbyak::util::Cpu cpu;
     return cpu;
 }
-#endif
 
 bool with_cpu_x86_sse42() {
-#ifdef ENABLE_MKL_DNN
     return get_cpu_info().has(Xbyak::util::Cpu::tSSE42);
-#else
-#if defined(HAVE_SSE)
-    return true;
-#else
-    return false;
-#endif
-#endif
 }
 
 bool with_cpu_x86_avx() {
-#ifdef ENABLE_MKL_DNN
     return get_cpu_info().has(Xbyak::util::Cpu::tAVX);
-#else
-#if defined(HAVE_AVX)
-    return true;
-#else
-    return false;
-#endif
-#endif
 }
 
 bool with_cpu_x86_avx2() {
-#ifdef ENABLE_MKL_DNN
     return get_cpu_info().has(Xbyak::util::Cpu::tAVX2);
-#else
-#if defined(HAVE_AVX2)
-    return true;
-#else
-    return false;
-#endif
-#endif
 }
 
 bool with_cpu_x86_avx512f() {
-#ifdef ENABLE_MKL_DNN
     return get_cpu_info().has(Xbyak::util::Cpu::tAVX512F);
-#else
-#if defined(HAVE_AVX512)
-    return true;
-#else
-    return false;
-#endif
-#endif
 }
 
 bool with_cpu_x86_avx512_core() {
-#ifdef ENABLE_MKL_DNN
        return get_cpu_info().has(Xbyak::util::Cpu::tAVX512F |
                                  Xbyak::util::Cpu::tAVX512DQ |
                                  Xbyak::util::Cpu::tAVX512BW);
-#else
-#if defined(HAVE_AVX512)
-    return true;
-#else
-    return false;
-#endif
-#endif
 }
 
 bool with_cpu_x86_bfloat16() {
-#ifdef ENABLE_MKL_DNN
     return get_cpu_info().has(Xbyak::util::Cpu::tAVX512_BF16);
-#else
-    return false;
-#endif
 }
 
 bool checkOpenMpEnvVars(bool includeOMPNumThreads) {
diff --git a/inference-engine/src/transformations/CMakeLists.txt b/inference-engine/src/transformations/CMakeLists.txt
index 17617ac3855..2403c64f55f 100644
--- a/inference-engine/src/transformations/CMakeLists.txt
+++ b/inference-engine/src/transformations/CMakeLists.txt
@@ -28,7 +28,7 @@ ie_add_vs_version_file(NAME ${TARGET_NAME}
                        FILEDESCRIPTION "Inference Engine Transformations library")
 
 target_link_libraries(${TARGET_NAME} PUBLIC ${NGRAPH_LIBRARIES}
-                                     PRIVATE openvino::conditional_compilation openvino::itt ngraph::builder pugixml)
+                                     PRIVATE ${NGRAPH_REF_LIBRARIES} openvino::conditional_compilation openvino::itt ngraph::builder pugixml)
 
 target_include_directories(${TARGET_NAME} PUBLIC ${PUBLIC_HEADERS_DIR}
                                           PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/src")
diff --git a/inference-engine/src/transformations/src/transformations/convert_precision.cpp b/inference-engine/src/transformations/src/transformations/convert_precision.cpp
index 2e09b554cd7..e4b3d83378f 100644
--- a/inference-engine/src/transformations/src/transformations/convert_precision.cpp
+++ b/inference-engine/src/transformations/src/transformations/convert_precision.cpp
@@ -14,6 +14,8 @@
 #include <ngraph/opsets/opset1.hpp>
 #include <ngraph_ops/type_relaxed.hpp>
 
+#include <ngraph/runtime/reference/convert.hpp>
+
 using namespace ngraph;
 
 bool fuse_type_to_constant(std::shared_ptr<ngraph::Node> & node, ngraph::element::Type to, const std::vector<ngraph::Input<ngraph::Node>> & consumers);
@@ -341,25 +343,44 @@ inline int32_t convert_value<uint32_t, int32_t>(uint32_t val) {
     return static_cast<int32_t>(val);
 }
 
-template <element::Type_t PREC_FROM, element::Type_t PREC_TO>
-static std::shared_ptr<Node> change_constant_precision(std::shared_ptr<opset4::Constant>& constant) {
-    using src_type = typename element_type_traits<PREC_FROM>::value_type;
-    using dst_type = typename element_type_traits<PREC_TO>::value_type;
+namespace {
+    template <element::Type_t PREC_FROM, element::Type_t PREC_TO>
+    std::shared_ptr<Node> change_constant_precision(std::shared_ptr<opset4::Constant>& constant) {
+        using src_type = typename element_type_traits<PREC_FROM>::value_type;
+        using dst_type = typename element_type_traits<PREC_TO>::value_type;
 
-    const auto * src_data = constant->get_data_ptr<src_type>();
-    const auto size = shape_size(constant->get_shape());
+        const auto * src_data = constant->get_data_ptr<src_type>();
+        const auto size = shape_size(constant->get_shape());
 
-    auto new_constant = std::make_shared<ngraph::opset4::Constant>(PREC_TO, constant->get_shape());
-    auto * dst_data = const_cast<dst_type *>(reinterpret_cast<const dst_type *>(new_constant->get_data_ptr()));
-    if (dst_data == nullptr)
-        throw ngraph_error("Can't get destination data pointer");
+        auto new_constant = std::make_shared<ngraph::opset4::Constant>(PREC_TO, constant->get_shape());
+        auto * dst_data = const_cast<dst_type *>(reinterpret_cast<const dst_type *>(new_constant->get_data_ptr()));
+        if (dst_data == nullptr)
+            throw ngraph_error("Can't get destination data pointer");
 
-    std::vector<dst_type> final_data;
-    for (size_t i = 0; i < size; ++i) {
-        dst_data[i] = convert_value<src_type, dst_type>(src_data[i]);
+        for (size_t i = 0; i < size; ++i) {
+            dst_data[i] = convert_value<src_type, dst_type>(src_data[i]);
+        }
+        return new_constant;
     }
-    return new_constant;
-}
+
+    template <>
+    std::shared_ptr<Node> change_constant_precision<element::Type_t::f16, element::Type_t::f32>(std::shared_ptr<opset4::Constant>& constant) {
+        using src_type = typename element_type_traits<element::Type_t::f16>::value_type;
+        using dst_type = typename element_type_traits<element::Type_t::f32>::value_type;
+
+        const auto * src_data = constant->get_data_ptr<src_type>();
+        const auto size = shape_size(constant->get_shape());
+
+        auto new_constant = std::make_shared<ngraph::opset4::Constant>(element::Type_t::f32, constant->get_shape());
+        auto * dst_data = const_cast<dst_type *>(reinterpret_cast<const dst_type *>(new_constant->get_data_ptr()));
+        if (dst_data == nullptr)
+            throw ngraph_error("Can't get destination data pointer");
+
+        ngraph::runtime::reference::convert<src_type, dst_type>(src_data, dst_data, size);
+
+        return new_constant;
+    }
+}   // namespace
 
 bool fuse_type_to_constant(std::shared_ptr<Node> & node, element::Type to, const std::vector<Input<Node>> & consumers) {
     if (auto constant = as_type_ptr<opset4::Constant>(node)) {
diff --git a/ngraph/core/reference/CMakeLists.txt b/ngraph/core/reference/CMakeLists.txt
index 2a96cb44c24..62749a650b4 100644
--- a/ngraph/core/reference/CMakeLists.txt
+++ b/ngraph/core/reference/CMakeLists.txt
@@ -37,9 +37,13 @@ if(COMMAND ie_faster_build)
     )
 endif()
 
+target_compile_definitions(${TARGET_NAME} PRIVATE XBYAK_NO_OP_NAMES XBYAK64)
+
 # Defines macro in C++ to load backend plugin
 target_include_directories(${TARGET_NAME} PUBLIC ${REF_IMPL_INCLUDE_DIR} ${NGRAPH_INCLUDE_PATH})
 
+target_link_libraries(${TARGET_NAME} PRIVATE xbyak)
+
 # Add an alias so that library can be used inside the build tree, e.g. when testing
 add_library(ngraph::reference ALIAS ${TARGET_NAME})
 
diff --git a/ngraph/core/reference/include/ngraph/runtime/reference/convert.hpp b/ngraph/core/reference/include/ngraph/runtime/reference/convert.hpp
index 8669a6048e0..f68daf84ceb 100644
--- a/ngraph/core/reference/include/ngraph/runtime/reference/convert.hpp
+++ b/ngraph/core/reference/include/ngraph/runtime/reference/convert.hpp
@@ -18,6 +18,8 @@
 
 #include <cstddef>
 
+#include "ngraph/type/float16.hpp"
+
 namespace ngraph
 {
     namespace runtime
@@ -34,6 +36,11 @@ namespace ngraph
                 }
             }
 
+            template <>
+            void convert<uint8_t, float16>(const uint8_t* arg, float16* out, size_t count);
+            template <>
+            void convert<float16, float>(const float16* arg, float* out, size_t count);
+
             template <typename TI, typename TO>
             typename std::enable_if<std::is_same<TO, char>::value>::type
                 convert(const TI* arg, TO* out, size_t count)
diff --git a/ngraph/core/reference/src/runtime/reference/convert.cpp b/ngraph/core/reference/src/runtime/reference/convert.cpp
new file mode 100644
index 00000000000..dd67d308a5d
--- /dev/null
+++ b/ngraph/core/reference/src/runtime/reference/convert.cpp
@@ -0,0 +1,201 @@
+//*****************************************************************************
+// Copyright 2021 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//*****************************************************************************
+
+#include "ngraph/runtime/reference/convert.hpp"
+#include "jit_generator.hpp"
+
+namespace ngraph
+{
+    namespace runtime
+    {
+        namespace reference
+        {
+            namespace
+            {
+                template <typename src_t, typename dst_t>
+                void jit_convert_vec(jit::Generator&, const Xbyak::RegExp&, const Xbyak::RegExp&);
+
+                template <>
+                void jit_convert_vec<uint8_t, float16>(jit::Generator& gen,
+                                                       const Xbyak::RegExp& src,
+                                                       const Xbyak::RegExp& dst)
+                {
+                    auto u8vec = gen.xmm1;
+                    auto i32vec = gen.ymm2;
+                    auto f16vec = gen.xmm3;
+                    auto fvec = gen.ymm4;
+
+                    gen.movq(u8vec, gen.qword[src]);
+                    gen.vpmovzxbd(i32vec, u8vec);
+                    gen.vcvtdq2ps(fvec, i32vec);
+                    gen.vcvtps2ph(f16vec, fvec, 0);
+                    gen.movdqu(gen.xword[dst], f16vec);
+                }
+
+                template <>
+                void jit_convert_vec<float16, float>(jit::Generator& gen,
+                                                     const Xbyak::RegExp& src,
+                                                     const Xbyak::RegExp& dst)
+                {
+                    auto f16vec = gen.xmm3;
+                    auto f32vec = gen.ymm4;
+
+                    gen.movdqu(f16vec, gen.xword[src]);
+                    gen.vcvtph2ps(f32vec, f16vec);
+                    gen.vmovups(gen.yword[dst], f32vec);
+                }
+
+                class jit_convert_array : public jit::Generator
+                {
+                    typedef struct context
+                    {
+                        struct
+                        {
+                            size_t type_size;
+                            void (jit::Generator::*copy)(const Xbyak::Reg64& dst,
+                                                         const Xbyak::Reg64& src,
+                                                         const Xbyak::Reg64& size);
+                        } src, dst;
+                        void (*convert_vec)(jit::Generator&,
+                                            const Xbyak::RegExp&,
+                                            const Xbyak::RegExp&);
+                    } context_t;
+
+                    jit_convert_array(const context_t& ctx)
+                    {
+                        using namespace Xbyak;
+
+                        const uint32_t vlen = 8u;
+
+                        auto reg_src = rax;
+                        auto reg_dst = rbx;
+                        auto reg_sz = rdx;
+
+                        Label tail, exit;
+
+                        preamble();
+
+                        mov(reg_src, ptr[param + offsetof(args_t, src)]);
+                        mov(reg_dst, ptr[param + offsetof(args_t, out)]);
+                        mov(reg_sz, ptr[param + offsetof(args_t, count)]);
+
+                        xor_(rsi, rsi);
+                        mov(r8, reg_sz);
+                        shr(r8, 3);
+
+                        foreach (rsi, 1, r8, [&, this](const Xbyak::Reg64& idx) {
+                            ctx.convert_vec(*this, reg_src, reg_dst);
+                            add(reg_src, ctx.src.type_size * vlen);
+                            add(reg_dst, ctx.dst.type_size * vlen);
+                        })
+                            ;
+
+                        L(tail);
+
+                        shl(rsi, 3);
+                        sub(reg_sz, rsi);
+                        test(reg_sz, reg_sz);
+                        jz(exit);
+
+                        // allocate array for 8 floats on stack
+                        sub(rsp, vlen * sizeof(float));
+                        mov(r8, rsp);
+
+                        vpxor(ymm4, ymm4, ymm4);
+                        vmovups(yword[r8], ymm4);
+
+                        // Tail conversion
+                        (this->*ctx.src.copy)(r8, reg_src, reg_sz);
+                        ctx.convert_vec(*this, r8, r8);
+                        (this->*ctx.dst.copy)(reg_dst, r8, reg_sz);
+
+                        // Free the array on stack
+                        add(rsp, vlen * sizeof(float));
+
+                        L(exit);
+
+                        postamble();
+                    }
+
+                public:
+                    typedef struct
+                    {
+                        const void* src;
+                        void* out;
+                        const size_t count;
+                    } args_t;
+
+                    typedef void (*fn_t)(const args_t*);
+
+                    template <typename src_t, typename dst_t>
+                    static fn_t get()
+                    {
+                        if (is_x64() && mayiuse(avx) && mayiuse(avx2) && mayiuse(fp16))
+                        {
+                            static const jit_convert_array::context_t context{
+                                {sizeof(src_t), &jit::Generator::copy<src_t>},
+                                {sizeof(dst_t), &jit::Generator::copy<dst_t>},
+                                jit_convert_vec<src_t, dst_t>};
+
+                            static jit_convert_array generator(context);
+
+                            return (fn_t)generator.getCode();
+                        }
+                        return nullptr;
+                    }
+                };
+            } // namespace
+
+            template <>
+            void convert<uint8_t, float16>(const uint8_t* arg, float16* out, size_t count)
+            {
+                auto converter = jit_convert_array::get<uint8_t, float16>();
+
+                if (converter)
+                {
+                    jit_convert_array::args_t args = {arg, out, count};
+                    converter(&args);
+                }
+                else
+                {
+                    for (size_t i = 0; i < count; ++i)
+                    {
+                        out[i] = static_cast<float16>(arg[i]);
+                    }
+                }
+            }
+
+            template <>
+            void convert<float16, float>(const float16* arg, float* out, size_t count)
+            {
+                auto converter = jit_convert_array::get<float16, float>();
+
+                if (converter)
+                {
+                    jit_convert_array::args_t args = {arg, out, count};
+                    converter(&args);
+                }
+                else
+                {
+                    for (size_t i = 0; i < count; ++i)
+                    {
+                        out[i] = static_cast<float>(arg[i]);
+                    }
+                }
+            }
+        }
+    }
+}
diff --git a/ngraph/core/reference/src/runtime/reference/jit_generator.cpp b/ngraph/core/reference/src/runtime/reference/jit_generator.cpp
new file mode 100644
index 00000000000..1b1bf9fcfa4
--- /dev/null
+++ b/ngraph/core/reference/src/runtime/reference/jit_generator.cpp
@@ -0,0 +1,229 @@
+//*****************************************************************************
+// Copyright 2017-2021 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//*****************************************************************************
+#include "jit_generator.hpp"
+#include "ngraph/type/float16.hpp"
+
+#include <xbyak/xbyak_util.h>
+
+namespace ngraph
+{
+    namespace runtime
+    {
+        namespace jit
+        {
+            using namespace Xbyak;
+
+#ifdef XBYAK64
+            static const Xbyak::Operand::Code abi_save_gpr_regs[] = {
+                Xbyak::Operand::RBX,
+                Xbyak::Operand::RBP,
+                Xbyak::Operand::R12,
+                Xbyak::Operand::R13,
+                Xbyak::Operand::R14,
+                Xbyak::Operand::R15,
+#ifdef _WIN32
+                Xbyak::Operand::RDI,
+                Xbyak::Operand::RSI,
+#endif
+            };
+
+#ifdef _WIN32
+            static const Xbyak::Reg64 abi_param1(Xbyak::Operand::RCX),
+                abi_param2(Xbyak::Operand::RDX), abi_param3(Xbyak::Operand::R8),
+                abi_param4(Xbyak::Operand::R9), abi_not_param1(Xbyak::Operand::RDI);
+#else
+            static const Xbyak::Reg64 abi_param1(Xbyak::Operand::RDI),
+                abi_param2(Xbyak::Operand::RSI), abi_param3(Xbyak::Operand::RDX),
+                abi_param4(Xbyak::Operand::RCX), abi_param5(Xbyak::Operand::R8),
+                abi_param6(Xbyak::Operand::R9), abi_not_param1(Xbyak::Operand::RCX);
+#endif
+#endif
+
+            const size_t Generator::num_abi_save_gpr_regs =
+                sizeof(abi_save_gpr_regs) / sizeof(abi_save_gpr_regs[0]);
+
+            const Xbyak::Reg64 Generator::param = abi_param1;
+
+            bool Generator::mayiuse(const cpu_isa_t cpu_isa)
+            {
+                static Xbyak::util::Cpu cpu;
+
+                using namespace Xbyak::util;
+
+                switch (cpu_isa)
+                {
+                case sse42: return cpu.has(Cpu::tSSE42);
+                case avx: return cpu.has(Cpu::tAVX);
+                case avx2: return cpu.has(Cpu::tAVX2);
+                case avx512_common: return cpu.has(Cpu::tAVX512F);
+                case avx512_core:
+                    return cpu.has(Cpu::tAVX512F) && cpu.has(Cpu::tAVX512BW) &&
+                           cpu.has(Cpu::tAVX512VL) && cpu.has(Cpu::tAVX512DQ);
+                case avx512_core_vnni:
+                    return cpu.has(Cpu::tAVX512F) && cpu.has(Cpu::tAVX512BW) &&
+                           cpu.has(Cpu::tAVX512VL) && cpu.has(Cpu::tAVX512DQ) &&
+                           cpu.has(Cpu::tAVX512_VNNI);
+                case avx512_mic:
+                    return cpu.has(Cpu::tAVX512F) && cpu.has(Cpu::tAVX512CD) &&
+                           cpu.has(Cpu::tAVX512ER) && cpu.has(Cpu::tAVX512PF);
+                case avx512_mic_4ops:
+                    return mayiuse(avx512_mic) && cpu.has(Cpu::tAVX512_4FMAPS) &&
+                           cpu.has(Cpu::tAVX512_4VNNIW);
+                case avx512_core_bf16:
+                    return mayiuse(avx512_core_vnni) && cpu.has(Cpu::tAVX512_BF16);
+                case avx512_vpopcnt: return true && cpu.has(Cpu::tAVX512_VPOPCNTDQ);
+                case fp16: return cpu.has(Cpu::tF16C);
+                case isa_any: return true;
+                }
+                return false;
+            }
+
+            bool Generator::is_x64() { return sizeof(void*) == 8; }
+            Generator::Generator(void* code_ptr, size_t code_size)
+                : Xbyak::CodeGenerator(code_size, code_ptr)
+                , size_of_abi_save_regs(num_abi_save_gpr_regs * rax.getBit() / 8 +
+                                        xmm_to_preserve * xmm_len)
+                , reg_EVEX_max_8b_offt(rbp)
+            {
+            }
+
+            void Generator::preamble()
+            {
+                if (xmm_to_preserve)
+                {
+                    sub(rsp, xmm_to_preserve * xmm_len);
+                    for (size_t i = 0; i < xmm_to_preserve; ++i)
+                        movdqu(ptr[rsp + i * xmm_len], Xbyak::Xmm(xmm_to_preserve_start + i));
+                }
+                for (size_t i = 0; i < num_abi_save_gpr_regs; ++i)
+                    push(Xbyak::Reg64(abi_save_gpr_regs[i]));
+                if (mayiuse(avx512_common))
+                {
+                    mov(reg_EVEX_max_8b_offt, 2 * EVEX_max_8b_offt);
+                }
+            }
+
+            void Generator::postamble()
+            {
+                for (size_t i = 0; i < num_abi_save_gpr_regs; ++i)
+                    pop(Xbyak::Reg64(abi_save_gpr_regs[num_abi_save_gpr_regs - 1 - i]));
+                if (xmm_to_preserve)
+                {
+                    for (size_t i = 0; i < xmm_to_preserve; ++i)
+                        movdqu(Xbyak::Xmm(xmm_to_preserve_start + i), ptr[rsp + i * xmm_len]);
+                    add(rsp, xmm_to_preserve * xmm_len);
+                }
+                if (mayiuse(avx) && !mayiuse(avx512_mic))
+                    vzeroupper();
+                ret();
+            }
+
+            void Generator::foreach (const Xbyak::Reg64& idx,
+                                     size_t step,
+                                     const Xbyak::Reg64& end,
+                                     std::function<void(const Xbyak::Reg64&)> && fn)
+            {
+                Label loop, exit;
+
+                L(loop);
+                cmp(idx, end);
+                jge(exit);
+
+                fn(idx);
+
+                add(idx, step);
+                jmp(loop);
+                L(exit);
+            }
+
+            template <>
+            void Generator::copy<uint8_t>(const Xbyak::Reg64& dst,
+                                          const Xbyak::Reg64& src,
+                                          const Xbyak::Reg64& size)
+            {
+                push(rsi);
+                push(r15);
+
+                xor_(rsi, rsi);
+
+                foreach (rsi, 1, size, [&, this](const Xbyak::Reg64& idx) {
+                    mov(r15b, byte[src + idx * sizeof(uint8_t)]);
+                    mov(byte[dst + idx * sizeof(uint8_t)], r15b);
+                })
+                    ;
+
+                pop(r15);
+                pop(rsi);
+            }
+
+            template <>
+            void Generator::copy<uint16_t>(const Xbyak::Reg64& dst,
+                                           const Xbyak::Reg64& src,
+                                           const Xbyak::Reg64& size)
+            {
+                push(rsi);
+                push(r15);
+
+                xor_(rsi, rsi);
+
+                foreach (rsi, 1, size, [&, this](const Xbyak::Reg64& idx) {
+                    mov(r15w, word[src + idx * sizeof(uint16_t)]);
+                    mov(word[dst + idx * sizeof(uint16_t)], r15w);
+                })
+                    ;
+
+                pop(r15);
+                pop(rsi);
+            }
+
+            template <>
+            void Generator::copy<uint32_t>(const Xbyak::Reg64& dst,
+                                           const Xbyak::Reg64& src,
+                                           const Xbyak::Reg64& size)
+            {
+                push(rsi);
+                push(r15);
+
+                xor_(rsi, rsi);
+
+                foreach (rsi, 1, size, [&, this](const Xbyak::Reg64& idx) {
+                    mov(r15d, dword[src + idx * sizeof(uint32_t)]);
+                    mov(dword[dst + idx * sizeof(uint32_t)], r15d);
+                })
+                    ;
+
+                pop(r15);
+                pop(rsi);
+            }
+
+            template <>
+            void Generator::copy<float16>(const Xbyak::Reg64& dst,
+                                          const Xbyak::Reg64& src,
+                                          const Xbyak::Reg64& size)
+            {
+                copy<uint16_t>(dst, src, size);
+            }
+
+            template <>
+            void Generator::copy<float>(const Xbyak::Reg64& dst,
+                                        const Xbyak::Reg64& src,
+                                        const Xbyak::Reg64& size)
+            {
+                copy<uint32_t>(dst, src, size);
+            }
+        }
+    }
+}
diff --git a/ngraph/core/reference/src/runtime/reference/jit_generator.hpp b/ngraph/core/reference/src/runtime/reference/jit_generator.hpp
new file mode 100644
index 00000000000..a4f16b1ca82
--- /dev/null
+++ b/ngraph/core/reference/src/runtime/reference/jit_generator.hpp
@@ -0,0 +1,82 @@
+//*****************************************************************************
+// Copyright 2017-2021 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//*****************************************************************************
+#pragma once
+
+#include <functional>
+#include <xbyak/xbyak.h>
+
+namespace ngraph
+{
+    namespace runtime
+    {
+        namespace jit
+        {
+            class Generator : public Xbyak::CodeGenerator
+            {
+                static constexpr size_t xmm_len = 16;
+
+#ifdef _WIN32
+                static constexpr size_t xmm_to_preserve_start = 6;
+                static constexpr size_t xmm_to_preserve = 10;
+#else
+                static constexpr size_t xmm_to_preserve_start = 0;
+                static constexpr size_t xmm_to_preserve = 0;
+#endif
+
+                static const size_t num_abi_save_gpr_regs;
+                const size_t size_of_abi_save_regs;
+
+                const Xbyak::Reg64 reg_EVEX_max_8b_offt;
+                static constexpr int EVEX_max_8b_offt = 0x200;
+
+            public:
+                static const Xbyak::Reg64 param;
+
+                typedef enum {
+                    isa_any,
+                    sse42,
+                    avx,
+                    avx2,
+                    avx512_common,
+                    avx512_core,
+                    avx512_core_vnni,
+                    avx512_mic,
+                    avx512_mic_4ops,
+                    avx512_core_bf16,
+                    avx512_vpopcnt,
+                    fp16
+                } cpu_isa_t;
+
+                static bool mayiuse(const cpu_isa_t cpu_isa);
+                static bool is_x64();
+
+                Generator(void* code_ptr = nullptr, size_t code_size = 16 * 1024);
+                void preamble();
+                void postamble();
+
+                void foreach (const Xbyak::Reg64& idx,
+                              size_t step,
+                              const Xbyak::Reg64& end,
+                              std::function<void(const Xbyak::Reg64&)> && fn);
+
+                template <typename T>
+                void copy(const Xbyak::Reg64& dst,
+                          const Xbyak::Reg64& src,
+                          const Xbyak::Reg64& size);
+            };
+        }
+    }
+}
diff --git a/ngraph/core/reference/src/runtime/reference/non_max_suppression.cpp b/ngraph/core/reference/src/runtime/reference/non_max_suppression.cpp
index b016f5948e3..0ffdf92bfb1 100644
--- a/ngraph/core/reference/src/runtime/reference/non_max_suppression.cpp
+++ b/ngraph/core/reference/src/runtime/reference/non_max_suppression.cpp
@@ -25,115 +25,118 @@
 using namespace ngraph;
 using namespace ngraph::runtime::reference;
 
-struct Rectangle
-{
-    Rectangle(float y_left, float x_left, float y_right, float x_right)
-        : y1{y_left}
-        , x1{x_left}
-        , y2{y_right}
-        , x2{x_right}
-    {
-    }
-
-    Rectangle() = default;
-
-    float y1 = 0.0f;
-    float x1 = 0.0f;
-    float y2 = 0.f;
-    float x2 = 0.0f;
-};
-
-static float intersectionOverUnion(const Rectangle& boxI, const Rectangle& boxJ)
-{
-    float areaI = (boxI.y2 - boxI.y1) * (boxI.x2 - boxI.x1);
-    float areaJ = (boxJ.y2 - boxJ.y1) * (boxJ.x2 - boxJ.x1);
-
-    if (areaI <= 0.0f || areaJ <= 0.0f)
-    {
-        return 0.0f;
-    }
-
-    float intersection_ymin = std::max(boxI.y1, boxJ.y1);
-    float intersection_xmin = std::max(boxI.x1, boxJ.x1);
-    float intersection_ymax = std::min(boxI.y2, boxJ.y2);
-    float intersection_xmax = std::min(boxI.x2, boxJ.x2);
-
-    float intersection_area = std::max(intersection_ymax - intersection_ymin, 0.0f) *
-                              std::max(intersection_xmax - intersection_xmin, 0.0f);
-
-    return intersection_area / (areaI + areaJ - intersection_area);
-}
-
-struct SelectedIndex
-{
-    SelectedIndex(int64_t batch_idx, int64_t class_idx, int64_t box_idx)
-        : batch_index(batch_idx)
-        , class_index(class_idx)
-        , box_index(box_idx)
-    {
-    }
-
-    SelectedIndex() = default;
-
-    int64_t batch_index = 0;
-    int64_t class_index = 0;
-    int64_t box_index = 0;
-};
-
-struct SelectedScore
-{
-    SelectedScore(float batch_idx, float class_idx, float score)
-        : batch_index{batch_idx}
-        , class_index{class_idx}
-        , box_score{score}
-    {
-    }
-
-    SelectedScore() = default;
-
-    float batch_index = 0.0f;
-    float class_index = 0.0f;
-    float box_score = 0.0f;
-};
-
-struct BoxInfo
-{
-    BoxInfo(const Rectangle& r,
-            int64_t idx,
-            float sc,
-            int64_t suppress_idx,
-            int64_t batch_idx,
-            int64_t class_idx)
-        : box{r}
-        , index{idx}
-        , suppress_begin_index{suppress_idx}
-        , batch_index{batch_idx}
-        , class_index{class_idx}
-        , score{sc}
-    {
-    }
-
-    BoxInfo() = default;
-
-    inline bool operator<(const BoxInfo& rhs) const
-    {
-        return score < rhs.score || (score == rhs.score && index > rhs.index);
-    }
-
-    Rectangle box;
-    int64_t index = 0;
-    int64_t suppress_begin_index = 0;
-    int64_t batch_index = 0;
-    int64_t class_index = 0;
-    float score = 0.0f;
-};
-
 namespace ngraph
 {
     namespace runtime
     {
         namespace reference
         {
+            namespace
+            {
+                struct Rectangle
+                {
+                    Rectangle(float y_left, float x_left, float y_right, float x_right)
+                        : y1{y_left}
+                        , x1{x_left}
+                        , y2{y_right}
+                        , x2{x_right}
+                    {
+                    }
+
+                    Rectangle() = default;
+
+                    float y1 = 0.0f;
+                    float x1 = 0.0f;
+                    float y2 = 0.f;
+                    float x2 = 0.0f;
+                };
+
+                static float intersectionOverUnion(const Rectangle& boxI, const Rectangle& boxJ)
+                {
+                    float areaI = (boxI.y2 - boxI.y1) * (boxI.x2 - boxI.x1);
+                    float areaJ = (boxJ.y2 - boxJ.y1) * (boxJ.x2 - boxJ.x1);
+
+                    if (areaI <= 0.0f || areaJ <= 0.0f)
+                    {
+                        return 0.0f;
+                    }
+
+                    float intersection_ymin = std::max(boxI.y1, boxJ.y1);
+                    float intersection_xmin = std::max(boxI.x1, boxJ.x1);
+                    float intersection_ymax = std::min(boxI.y2, boxJ.y2);
+                    float intersection_xmax = std::min(boxI.x2, boxJ.x2);
+
+                    float intersection_area =
+                        std::max(intersection_ymax - intersection_ymin, 0.0f) *
+                        std::max(intersection_xmax - intersection_xmin, 0.0f);
+
+                    return intersection_area / (areaI + areaJ - intersection_area);
+                }
+
+                struct SelectedIndex
+                {
+                    SelectedIndex(int64_t batch_idx, int64_t class_idx, int64_t box_idx)
+                        : batch_index(batch_idx)
+                        , class_index(class_idx)
+                        , box_index(box_idx)
+                    {
+                    }
+
+                    SelectedIndex() = default;
+
+                    int64_t batch_index = 0;
+                    int64_t class_index = 0;
+                    int64_t box_index = 0;
+                };
+
+                struct SelectedScore
+                {
+                    SelectedScore(float batch_idx, float class_idx, float score)
+                        : batch_index{batch_idx}
+                        , class_index{class_idx}
+                        , box_score{score}
+                    {
+                    }
+
+                    SelectedScore() = default;
+
+                    float batch_index = 0.0f;
+                    float class_index = 0.0f;
+                    float box_score = 0.0f;
+                };
+
+                struct BoxInfo
+                {
+                    BoxInfo(const Rectangle& r,
+                            int64_t idx,
+                            float sc,
+                            int64_t suppress_idx,
+                            int64_t batch_idx,
+                            int64_t class_idx)
+                        : box{r}
+                        , index{idx}
+                        , suppress_begin_index{suppress_idx}
+                        , batch_index{batch_idx}
+                        , class_index{class_idx}
+                        , score{sc}
+                    {
+                    }
+
+                    BoxInfo() = default;
+
+                    inline bool operator<(const BoxInfo& rhs) const
+                    {
+                        return score < rhs.score || (score == rhs.score && index > rhs.index);
+                    }
+
+                    Rectangle box;
+                    int64_t index = 0;
+                    int64_t suppress_begin_index = 0;
+                    int64_t batch_index = 0;
+                    int64_t class_index = 0;
+                    float score = 0.0f;
+                };
+            }
             void non_max_suppression(const float* boxes_data,
                                      const Shape& boxes_data_shape,
                                      const float* scores_data,
diff --git a/ngraph/test/backend/convert.in.cpp b/ngraph/test/backend/convert.in.cpp
index cbc61abde86..d9dc3741c6f 100644
--- a/ngraph/test/backend/convert.in.cpp
+++ b/ngraph/test/backend/convert.in.cpp
@@ -16,6 +16,7 @@
 
 #include "gtest/gtest.h"
 #include "ngraph/ngraph.hpp"
+#include "ngraph/runtime/reference/convert.hpp"
 #include "ngraph/runtime/tensor.hpp"
 #include "runtime/backend.hpp"
 #include "util/all_close.hpp"
@@ -171,3 +172,21 @@ NGRAPH_TEST(${BACKEND_NAME}, convert_bf16_float32)
                              1.5f}),
               read_vector<float>(result));
 }
+
+NGRAPH_TEST(${BACKEND_NAME}, convert_fp16_float32)
+{
+    std::vector<float> f32vec = {-20.5, -15, -10.5, -0.5, 0, 0.5, 10.5, 15, 20.5};
+    std::vector<float16> f16vec(std::begin(f32vec), std::end(f32vec));
+    std::vector<float> result(f32vec.size());
+    runtime::reference::convert(f16vec.data(), result.data(), f32vec.size());
+    EXPECT_EQ(result, f32vec);
+}
+
+NGRAPH_TEST(${BACKEND_NAME}, convert_uint8_fp16)
+{
+    std::vector<uint8_t> u8vec = {0, 10, 15, 20, 43, 56, 78, 99, 102, 130, 142};
+    std::vector<float16> f16vec(std::begin(u8vec), std::end(u8vec));
+    std::vector<float16> result(u8vec.size());
+    runtime::reference::convert(u8vec.data(), result.data(), u8vec.size());
+    EXPECT_EQ(result, f16vec);
+}
diff --git a/thirdparty/CMakeLists.txt b/thirdparty/CMakeLists.txt
index efaa7c3ecdd..8f91f7510d4 100644
--- a/thirdparty/CMakeLists.txt
+++ b/thirdparty/CMakeLists.txt
@@ -16,3 +16,6 @@
 
 add_subdirectory(ittapi)
 add_subdirectory(itt_collector)
+add_subdirectory(xbyak)
+
+openvino_developer_export_targets(COMPONENT openvino_common TARGETS xbyak)
diff --git a/thirdparty/xbyak b/thirdparty/xbyak
new file mode 160000
index 00000000000..8d1e41b6508
--- /dev/null
+++ b/thirdparty/xbyak
@@ -0,0 +1 @@
+Subproject commit 8d1e41b650890080fb77548372b6236bbd4079f9