From bb75829a88c73bf07bb3c9ab473f7d5a1170d28c Mon Sep 17 00:00:00 2001 From: Pawel Raasz Date: Mon, 10 Jun 2024 09:15:45 +0200 Subject: [PATCH] [core] Add f4e2m1 element type and float4_e2m1 type (#24863) ### Details: - Add new element into OpenVINO `f4e2m1` - Add new type `float4_e2m1` ### Tickets: - [CVS-141565](https://jira.devtools.intel.com/browse/CVS-141565) --------- Co-authored-by: Michal Lukaszewski --- .../ncc_naming_style/openvino.style | 4 +- src/bindings/c/docs/api_overview.md | 2 + src/bindings/c/include/openvino/c/ov_common.h | 1 + src/bindings/c/src/ov_tensor.cpp | 3 +- .../python/src/pyopenvino/core/common.cpp | 1 + .../pyopenvino/graph/types/element_type.cpp | 1 + .../openvino/core/type/element_type.hpp | 9 +- .../core/type/element_type_traits.hpp | 5 + .../openvino/core/type/float4_e2m1.hpp | 151 ++++++++++++++++++ src/core/include/openvino/op/constant.hpp | 2 + src/core/src/pass/serialize.cpp | 2 + src/core/src/pass/visualize_tree.cpp | 1 + src/core/src/type/element_type.cpp | 28 +++- src/core/src/type/float4_e2m1.cpp | 120 ++++++++++++++ src/core/tests/element_type.cpp | 2 + src/core/tests/float4_e2m1.cpp | 138 ++++++++++++++++ .../src/nodes/executors/type_mask.hpp | 5 +- .../include/common_test_utils/type_ranges.hpp | 1 + 18 files changed, 463 insertions(+), 13 deletions(-) create mode 100644 src/core/include/openvino/core/type/float4_e2m1.hpp create mode 100644 src/core/src/type/float4_e2m1.cpp create mode 100644 src/core/tests/float4_e2m1.cpp diff --git a/cmake/developer_package/ncc_naming_style/openvino.style b/cmake/developer_package/ncc_naming_style/openvino.style index 141f02b9ab5..8bfa4164490 100644 --- a/cmake/developer_package/ncc_naming_style/openvino.style +++ b/cmake/developer_package/ncc_naming_style/openvino.style @@ -1,6 +1,6 @@ # custom OpenVINO values CppMethod: '^(operator\W+|[a-z_\d]+|signaling_NaN|quiet_NaN|OPENVINO_OP)$' -ClassName: '^([A-Z][\w]+|b?float16|float8_e4m3|float8_e5m2|numeric_limits|ngraph_error|stopwatch|unsupported_op)$' +ClassName: '^([A-Z][\w]+|b?float16|float8_e4m3|float8_e5m2|float4_e2m1|numeric_limits|ngraph_error|stopwatch|unsupported_op)$' StructName: '^([A-Z][\w]+|element_type_traits|hash|oi_pair|stat)$' FunctionName: '^(operator\W+|[a-z_\d]+)|PrintTo$' Namespace: '^([a-z\d_]*|InferenceEngine)$' @@ -18,7 +18,7 @@ VariableReference: '^\w+$' EnumName: '^[A-Z][\w]+$' # excepts element_type -EnumConstantName: '^([A-Z\d_]+|undefined|dynamic|boolean|bf16|f16|f32|f64|i4|i8|i16|i32|i64|u1|u2|u3|u4|u6|u8|u16|u32|u64|nf4|f8e4m3|f8e5m2|string|asymmetric|align_corners|round_prefer_floor|round_prefer_ceil|floor|ceil|simple|nearest|linear|linear_onnx|cubic|area|scales|sizes|half_pixel|tf_half_pixel_for_nn|pytorch_half_pixel|asymetric)$' +EnumConstantName: '^([A-Z\d_]+|undefined|dynamic|boolean|bf16|f16|f32|f64|i4|i8|i16|i32|i64|u1|u2|u3|u4|u6|u8|u16|u32|u64|nf4|f8e4m3|f8e5m2|f4e2m1|string|asymmetric|align_corners|round_prefer_floor|round_prefer_ceil|floor|ceil|simple|nearest|linear|linear_onnx|cubic|area|scales|sizes|half_pixel|tf_half_pixel_for_nn|pytorch_half_pixel|asymetric)$' # TODO: align UsingDeclaration: '^.*$' TypedefName: '^.*$' diff --git a/src/bindings/c/docs/api_overview.md b/src/bindings/c/docs/api_overview.md index 3deedeb5dde..6649582d9eb 100644 --- a/src/bindings/c/docs/api_overview.md +++ b/src/bindings/c/docs/api_overview.md @@ -205,6 +205,8 @@ typedef enum { STRING, //!< string element type + F4E2M1, //!< f4e2m1 element type + } ov_element_type_e; ``` diff --git a/src/bindings/c/include/openvino/c/ov_common.h b/src/bindings/c/include/openvino/c/ov_common.h index 22ac486fc81..8c42e35247b 100644 --- a/src/bindings/c/include/openvino/c/ov_common.h +++ b/src/bindings/c/include/openvino/c/ov_common.h @@ -194,6 +194,7 @@ typedef enum { F8E4M3, //!< f8e4m3 element type F8E5M3, //!< f8e5m2 element type STRING, //!< string element type + F4E2M1, //!< f4e2m1 element type } ov_element_type_e; /** diff --git a/src/bindings/c/src/ov_tensor.cpp b/src/bindings/c/src/ov_tensor.cpp index 952f9cb394b..729f911cdbb 100644 --- a/src/bindings/c/src/ov_tensor.cpp +++ b/src/bindings/c/src/ov_tensor.cpp @@ -30,7 +30,8 @@ const std::map element_type_map = { {ov_element_type_e::NF4, ov::element::nf4}, {ov_element_type_e::F8E4M3, ov::element::f8e4m3}, {ov_element_type_e::F8E5M3, ov::element::f8e5m2}, - {ov_element_type_e::STRING, ov::element::string}}; + {ov_element_type_e::STRING, ov::element::string}, + {ov_element_type_e::F4E2M1, ov::element::f4e2m1}}; ov_element_type_e find_ov_element_type_e(ov::element::Type type) { for (auto iter = element_type_map.begin(); iter != element_type_map.end(); iter++) { diff --git a/src/bindings/python/src/pyopenvino/core/common.cpp b/src/bindings/python/src/pyopenvino/core/common.cpp index 17900212796..63cec728f9e 100644 --- a/src/bindings/python/src/pyopenvino/core/common.cpp +++ b/src/bindings/python/src/pyopenvino/core/common.cpp @@ -29,6 +29,7 @@ const std::map& ov_type_to_dtype() { {ov::element::u4, py::dtype("uint8")}, {ov::element::nf4, py::dtype("uint8")}, {ov::element::i4, py::dtype("int8")}, {ov::element::f8e4m3, py::dtype("uint8")}, {ov::element::f8e5m2, py::dtype("uint8")}, {ov::element::string, py::dtype("bytes_")}, + {ov::element::f4e2m1, py::dtype("uint8")}, }; return ov_type_to_dtype_mapping; } diff --git a/src/bindings/python/src/pyopenvino/graph/types/element_type.cpp b/src/bindings/python/src/pyopenvino/graph/types/element_type.cpp index e65590a4a04..bc1f19a70a1 100644 --- a/src/bindings/python/src/pyopenvino/graph/types/element_type.cpp +++ b/src/bindings/python/src/pyopenvino/graph/types/element_type.cpp @@ -53,6 +53,7 @@ void regclass_graph_Type(py::module m) { type.attr("f8e4m3") = ov::element::f8e4m3; type.attr("f8e5m2") = ov::element::f8e5m2; type.attr("string") = ov::element::string; + type.attr("f4e2m1") = ov::element::f4e2m1; type.def("__hash__", &ov::element::Type::hash); type.def("__repr__", [](const ov::element::Type& self) { diff --git a/src/core/include/openvino/core/type/element_type.hpp b/src/core/include/openvino/core/type/element_type.hpp index b0d456fa0bd..52a285c6c82 100644 --- a/src/core/include/openvino/core/type/element_type.hpp +++ b/src/core/include/openvino/core/type/element_type.hpp @@ -20,6 +20,7 @@ #include "openvino/core/rtti.hpp" #include "openvino/core/type/bfloat16.hpp" #include "openvino/core/type/float16.hpp" +#include "openvino/core/type/float4_e2m1.hpp" #include "openvino/core/type/float8_e4m3.hpp" #include "openvino/core/type/float8_e5m2.hpp" @@ -59,7 +60,8 @@ enum class Type_t { nf4, //!< nf4 element type f8e4m3, //!< f8e4m3 element type f8e5m2, //!< f8e5m2 element type - string //!< string element type + string, //!< string element type + f4e2m1 //!< f4e2m1 element type }; /// \brief Base class to define element type @@ -209,6 +211,9 @@ constexpr Type f8e5m2(Type_t::f8e5m2); /// \brief string element type /// \ingroup ov_element_cpp_api constexpr Type string(Type_t::string); +/// \brief f4e2m1 element type +/// \ingroup ov_element_cpp_api +constexpr Type f4e2m1(Type_t::f4e2m1); template Type from() { @@ -248,6 +253,8 @@ template <> OPENVINO_API Type from(); template <> OPENVINO_API Type from(); +template <> +OPENVINO_API Type from(); OPENVINO_API Type fundamental_type_for(const Type& type); diff --git a/src/core/include/openvino/core/type/element_type_traits.hpp b/src/core/include/openvino/core/type/element_type_traits.hpp index 94f3c25372e..4938c79bfd2 100644 --- a/src/core/include/openvino/core/type/element_type_traits.hpp +++ b/src/core/include/openvino/core/type/element_type_traits.hpp @@ -127,4 +127,9 @@ template <> struct element_type_traits { using value_type = std::string; }; + +template <> +struct element_type_traits { + using value_type = ov::float4_e2m1; +}; } // namespace ov diff --git a/src/core/include/openvino/core/type/float4_e2m1.hpp b/src/core/include/openvino/core/type/float4_e2m1.hpp new file mode 100644 index 00000000000..b9208f3eb7d --- /dev/null +++ b/src/core/include/openvino/core/type/float4_e2m1.hpp @@ -0,0 +1,151 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "openvino/core/core_visibility.hpp" + +namespace ov { + +/** + * @brief Class to represent the f4e2m1 type. + */ +class OPENVINO_API float4_e2m1 { +public: + float4_e2m1() = default; + float4_e2m1(float value); + + template + explicit float4_e2m1(I value) : float4_e2m1(static_cast(value)) {} + + template + bool operator==(const T& other) const; + template + bool operator!=(const T& other) const { + return !(*this == other); + } + + template + bool operator<(const T& other) const; + template + bool operator<=(const T& other) const; + template + bool operator>(const T& other) const; + template + bool operator>=(const T& other) const; + template + float4_e2m1 operator+(const T& other) const; + template + float4_e2m1 operator+=(const T& other); + template + float4_e2m1 operator-(const T& other) const; + template + float4_e2m1 operator-=(const T& other); + template + float4_e2m1 operator*(const T& other) const; + template + float4_e2m1 operator*=(const T& other); + template + float4_e2m1 operator/(const T& other) const; + template + float4_e2m1 operator/=(const T& other); + + operator float() const; + + static constexpr float4_e2m1 from_bits(uint8_t bits) { + return float4_e2m1(bits, true); + } + + uint8_t to_bits() const; + + friend std::ostream& operator<<(std::ostream& out, const float4_e2m1& obj) { + out << static_cast(obj); + return out; + } + +private: + constexpr float4_e2m1(uint8_t x, bool) : m_value{x} {} + + uint8_t m_value; +}; + +#if defined(_MSC_VER) +# pragma warning(push) +# pragma warning(disable : 4756) +#endif +template +bool float4_e2m1::operator==(const T& other) const { + return (static_cast(*this) == static_cast(other)); +} + +template +bool float4_e2m1::operator<(const T& other) const { + return (static_cast(*this) < static_cast(other)); +} + +template +bool float4_e2m1::operator<=(const T& other) const { + return (static_cast(*this) <= static_cast(other)); +} + +template +bool float4_e2m1::operator>(const T& other) const { + return (static_cast(*this) > static_cast(other)); +} + +template +bool float4_e2m1::operator>=(const T& other) const { + return (static_cast(*this) >= static_cast(other)); +} + +template +float4_e2m1 float4_e2m1::operator+(const T& other) const { + return {static_cast(*this) + static_cast(other)}; +} + +template +float4_e2m1 float4_e2m1::operator+=(const T& other) { + return *this = *this + other; +} + +template +float4_e2m1 float4_e2m1::operator-(const T& other) const { + return {static_cast(*this) - static_cast(other)}; +} + +template +float4_e2m1 float4_e2m1::operator-=(const T& other) { + return *this = *this - other; +} + +template +float4_e2m1 float4_e2m1::operator*(const T& other) const { + return {static_cast(*this) * static_cast(other)}; +} + +template +float4_e2m1 float4_e2m1::operator*=(const T& other) { + return *this = *this * other; +} + +template +float4_e2m1 float4_e2m1::operator/(const T& other) const { + return {static_cast(*this) / static_cast(other)}; +} + +template +float4_e2m1 float4_e2m1::operator/=(const T& other) { + return *this = *this / other; +} +#if defined(_MSC_VER) +# pragma warning(pop) +#endif +} // namespace ov diff --git a/src/core/include/openvino/op/constant.hpp b/src/core/include/openvino/op/constant.hpp index b7e4d7d9a8d..1b9005822e1 100644 --- a/src/core/include/openvino/op/constant.hpp +++ b/src/core/include/openvino/op/constant.hpp @@ -156,6 +156,7 @@ public: case Type_t::string: fill_data(value); break; + case Type_t::f4e2m1: case Type_t::undefined: case Type_t::dynamic: OPENVINO_THROW("unsupported type"); @@ -707,6 +708,7 @@ private: case Type_t::string: write_buffer(source); break; + case Type_t::f4e2m1: case Type_t::undefined: case Type_t::dynamic: OPENVINO_THROW("unsupported type"); diff --git a/src/core/src/pass/serialize.cpp b/src/core/src/pass/serialize.cpp index f38500bc354..19e2a8031b3 100644 --- a/src/core/src/pass/serialize.cpp +++ b/src/core/src/pass/serialize.cpp @@ -767,6 +767,8 @@ std::string get_precision_name(const ov::element::Type& elem_type) { return "F8E5M2"; case ::ov::element::Type_t::string: return "STRING"; + case ::ov::element::Type_t::f4e2m1: + return "F4E2M1"; default: OPENVINO_THROW("Unsupported precision: ", elem_type); } diff --git a/src/core/src/pass/visualize_tree.cpp b/src/core/src/pass/visualize_tree.cpp index 866093096a0..4d1184397c2 100644 --- a/src/core/src/pass/visualize_tree.cpp +++ b/src/core/src/pass/visualize_tree.cpp @@ -457,6 +457,7 @@ static std::string get_value(const std::shared_ptr& consta case ov::element::Type_t::i4: case ov::element::Type_t::f8e4m3: case ov::element::Type_t::f8e5m2: + case ov::element::Type_t::f4e2m1: ss << constant->get_output_element_type(0).get_type_name() << " value"; break; case ov::element::Type_t::bf16: diff --git a/src/core/src/type/element_type.cpp b/src/core/src/type/element_type.cpp index 8a529d6d0e6..3d8e6413a86 100644 --- a/src/core/src/type/element_type.cpp +++ b/src/core/src/type/element_type.cpp @@ -83,6 +83,8 @@ inline TypeInfo get_type_info(ov::element::Type_t type) { return {8, true, true, true, "f8e5m2", "f8e5m2"}; case ov::element::Type_t::string: return {8 * sizeof(std::string), false, false, false, "string", "string"}; + case ov::element::Type_t::f4e2m1: + return {4, true, true, true, "f4e2m1", "f4e2m1"}; default: OPENVINO_THROW("ov::element::Type_t not supported: ", type); } @@ -139,6 +141,8 @@ ov::element::Type type_from_string(const std::string& type) { return ::ov::element::Type(::ov::element::Type_t::f8e4m3); } else if (type == "f8e5m2" || type == "F8E5M2") { return ::ov::element::Type(::ov::element::Type_t::f8e5m2); + } else if (type == "f4e2m1" || type == "F4E2M1") { + return ::ov::element::Type(::ov::element::Type_t::f4e2m1); } else { OPENVINO_THROW("Incorrect type: ", type); } @@ -147,11 +151,11 @@ ov::element::Type type_from_string(const std::string& type) { std::vector ov::element::Type::get_known_types() { std::vector rc = { - &ov::element::dynamic, &ov::element::boolean, &ov::element::bf16, &ov::element::f16, &ov::element::f32, - &ov::element::f64, &ov::element::i4, &ov::element::i8, &ov::element::i16, &ov::element::i32, - &ov::element::i64, &ov::element::u1, &ov::element::u2, &ov::element::u3, &ov::element::u4, - &ov::element::u6, &ov::element::u8, &ov::element::u16, &ov::element::u32, &ov::element::u64, - &ov::element::nf4, &ov::element::f8e4m3, &ov::element::f8e5m2, &ov::element::string}; + &ov::element::dynamic, &ov::element::boolean, &ov::element::bf16, &ov::element::f16, &ov::element::f32, + &ov::element::f64, &ov::element::i4, &ov::element::i8, &ov::element::i16, &ov::element::i32, + &ov::element::i64, &ov::element::u1, &ov::element::u2, &ov::element::u3, &ov::element::u4, + &ov::element::u6, &ov::element::u8, &ov::element::u16, &ov::element::u32, &ov::element::u64, + &ov::element::nf4, &ov::element::f8e4m3, &ov::element::f8e5m2, &ov::element::string, &ov::element::f4e2m1}; return rc; } @@ -187,6 +191,7 @@ ov::element::Type::Type(size_t bitwidth, {ov::element::Type_t::f8e4m3, {8, true, true, true, "f8e4m3", "f8e4m3"}}, {ov::element::Type_t::f8e5m2, {8, true, true, true, "f8e5m2", "f8e5m2"}}, {ov::element::Type_t::string, {8 * sizeof(std::string), false, false, false, "string", "string"}}, + {ov::element::Type_t::f8e5m2, {4, true, true, true, "f4e2m1", "f4e2m1"}}, }; for (const auto& t : elements_map) { const TypeInfo& info = t.second; @@ -290,6 +295,10 @@ template <> Type from() { return Type_t::string; } +template <> +Type from() { + return Type_t::f4e2m1; +} Type fundamental_type_for(const Type& type) { switch (type) { @@ -339,6 +348,8 @@ Type fundamental_type_for(const Type& type) { return from::value_type>(); case Type_t::string: return from::value_type>(); + case Type_t::f4e2m1: + return from::value_type>(); default: OPENVINO_THROW("Unsupported Data type: ", type); } @@ -359,8 +370,7 @@ std::istream& ov::element::operator>>(std::istream& in, ov::element::Type& obj) {"U16", ov::element::u16}, {"U32", ov::element::u32}, {"U64", ov::element::u64}, {"FP32", ov::element::f32}, {"FP64", ov::element::f64}, {"FP16", ov::element::f16}, {"BIN", ov::element::u1}, {"NF4", ov::element::nf4}, {"F8E4M3", ov::element::f8e4m3}, - {"F8E5M2", ov::element::f8e5m2}, {"STRING", ov::element::string}, - }; + {"F8E5M2", ov::element::f8e5m2}, {"STRING", ov::element::string}, {"F4E2M1", ov::element::f4e2m1}}; std::string str; in >> str; auto it_legacy = legacy.find(str); @@ -448,6 +458,7 @@ inline size_t compiler_byte_size(ov::element::Type_t et) { ET_CASE(f8e4m3); ET_CASE(f8e5m2); ET_CASE(string); + ET_CASE(f4e2m1); #undef ET_CASE case ov::element::Type_t::undefined: return 0; @@ -486,7 +497,8 @@ OPENVINO_API EnumNames& EnumNames::get() { {"nf4", element::Type_t::nf4}, {"f8e4m3", element::Type_t::f8e4m3}, {"f8e5m2", element::Type_t::f8e5m2}, - {"string", element::Type_t::string}}); + {"string", element::Type_t::string}, + {"f4e2m1", element::Type_t::f4e2m1}}); return enum_names; } diff --git a/src/core/src/type/float4_e2m1.cpp b/src/core/src/type/float4_e2m1.cpp new file mode 100644 index 00000000000..9893414f79b --- /dev/null +++ b/src/core/src/type/float4_e2m1.cpp @@ -0,0 +1,120 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "openvino/core/type/float4_e2m1.hpp" + +#include +#include + +#include "openvino/reference/fake_convert.hpp" + +namespace ov { +static_assert(sizeof(float4_e2m1) == 1, "type f4e2m1 must be exactly 1 byte"); +static_assert(std::is_trivially_constructible::value, "should be trivially constructible"); +static_assert(std::is_trivially_copyable::value, "must be trivially copyable"); +static_assert(std::is_trivially_destructible::value, "must be trivially destructible"); + +// clang-format off +static constexpr std::array f4e2m1_to_f32_lut{ + 0.0f, 0.5f, + 1.0f, 1.5f, + 2.0f, 3.0f, + 4.0f, 6.0f, + -0.0f, -0.5f, + -1.0f, -1.5f, + -2.0f, -3.0f, + -4.0f, -6.0f}; +// clang-format on + +namespace { + +constexpr uint32_t three_bytes_shift = 24; + +union f32_t { + float value; + uint32_t bits; +}; + +constexpr uint8_t f4e2m1_e_size = 2; // f4e2m1 exponent bit size +constexpr uint8_t f4e2m1_e_mask = 0x06; // f4e2m1 exponent bit mask +constexpr uint8_t f4e2m1_e_bias = 1; // f4e2m1 exponent bias +constexpr uint8_t f4e2m1_e_max = 0x03; // f4e2m1 exponent max value +constexpr uint8_t f4e2m1_m_size = 1; // f4e2m1 mantissa bits size +constexpr uint8_t f4e2m1_m_mask = 0x01; // f4e2m1 mantissa bit mask + +uint8_t f32_to_f4e2m1_bits(const float value) { + constexpr uint32_t f32_s_mask = 0x80000000; // f32 sign bit mask + constexpr uint32_t f32_e_mask = 0x7F800000; // f32 exponent bits mask + constexpr uint32_t f32_e_bias = 127; // f32 exponent bias + constexpr uint32_t f32_e_size = 8; // f32 exponent bits size + constexpr uint32_t f32_m_mask = 0x007fffff; // f32 mantissa bits mask + constexpr uint32_t f32_m_size = 23; // f32 mantissa bits size + + constexpr uint32_t f_e_mask = f4e2m1_e_mask << three_bytes_shift; // f4 exponent bits mask (on u32) + constexpr uint32_t f_m_mask = f4e2m1_m_mask << three_bytes_shift; // f4 mantissa bits mask (on u32) + constexpr uint32_t f_m_hidden_one_mask = 0x02000000; // f4 mantissa hidden one bits mask (on u32) + + constexpr uint32_t round_half = 0x01ffffff; // value for half to even round for f4 + constexpr uint32_t round_norm = 0x07ffffff; // value for normal round for f4 + constexpr uint32_t round_even = 0x00800000; // value for half to even round for f4 + constexpr uint32_t round_odd = 0x01800000; // value for an non-half to even round for f4 + + const auto input = f32_t{value}; + auto f4_bits = static_cast((input.bits & f32_s_mask) >> (three_bytes_shift + 4U)); + + uint32_t f32_e_field = input.bits & f32_e_mask; + + if (f32_e_field == f32_e_mask) { + f4_bits |= (f4e2m1_e_mask | f4e2m1_m_mask); + } else if (f32_e_field != 0) { + int32_t target_f_biased_exp = (f32_e_field >> f32_m_size) - (f32_e_bias - f4e2m1_e_bias); + uint32_t fractional = (input.bits & f32_m_mask) << (f32_e_size - f4e2m1_e_size - 4U); + + // for normalized values round apply rounding change target fractional and biased exponent + if ((fractional & round_half) == round_odd || (fractional & round_norm) != 0) { + fractional += round_even; + if (0 != (fractional & f_e_mask)) { + fractional &= f_e_mask; + ++target_f_biased_exp; + } + } + fractional &= f_m_mask; + + // set exponent and mantissa on target bits + if (target_f_biased_exp > f4e2m1_e_max) { + // Use NAN as this type has no infinity + f4_bits |= (f4e2m1_e_mask | f4e2m1_m_mask); + } else if (target_f_biased_exp > 0) { + f4_bits |= (target_f_biased_exp << f4e2m1_m_size) | (fractional >> (three_bytes_shift)); + } else { + // Restore the hidden 1 in target mantissa for subnormal calculation + fractional = f_m_hidden_one_mask | (input.bits & f32_m_mask) << (f32_e_size - f4e2m1_e_size - 4U); + // Will any bits be shifted off? + int32_t shift = target_f_biased_exp < -(f4e2m1_e_max) ? 0 : (1U << (1 - target_f_biased_exp)); + uint32_t sticky = (fractional & (shift - 1)) ? 1 : 0; + + fractional = ((1 + target_f_biased_exp) > f4e2m1_e_max - 1) ? 0 : fractional >> (1 - target_f_biased_exp); + fractional |= sticky; + // apply rounding + if (((fractional & round_half) == round_odd) || ((fractional & round_norm) != 0)) { + fractional += round_even; + } + f4_bits |= fractional >> three_bytes_shift; + } + } + + return f4_bits; +} +} // namespace + +float4_e2m1::float4_e2m1(const float value) : m_value(f32_to_f4e2m1_bits(value)){}; + +float4_e2m1::operator float() const { + return f4e2m1_to_f32_lut[m_value]; +} + +uint8_t float4_e2m1::to_bits() const { + return m_value; +} +} // namespace ov diff --git a/src/core/tests/element_type.cpp b/src/core/tests/element_type.cpp index 847e0de9849..407056cb810 100644 --- a/src/core/tests/element_type.cpp +++ b/src/core/tests/element_type.cpp @@ -73,6 +73,8 @@ TEST(element_type, from_string) { EXPECT_EQ(element::Type("F8E5M2"), element::f8e5m2); EXPECT_EQ(element::Type("string"), element::string); EXPECT_EQ(element::Type("STRING"), element::string); + EXPECT_EQ(element::Type("F4E2M1"), element::f4e2m1); + EXPECT_EQ(element::Type("F4E2M1"), element::f4e2m1); EXPECT_EQ(element::Type("undefined"), element::undefined); EXPECT_EQ(element::Type("UNSPECIFIED"), element::undefined); diff --git a/src/core/tests/float4_e2m1.cpp b/src/core/tests/float4_e2m1.cpp new file mode 100644 index 00000000000..68d3b0aef29 --- /dev/null +++ b/src/core/tests/float4_e2m1.cpp @@ -0,0 +1,138 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "openvino/core/type/float4_e2m1.hpp" + +#include + +#include +#include + +#include "common_test_utils/float_util.hpp" + +namespace ov { +namespace test { + +template +std::vector> enumerate(const TContainer& values) { + std::vector> enum_values; + int i = 0; + for (const auto& v : values) { + enum_values.emplace_back(i, v); + ++i; + } + return enum_values; +} + +TEST(F4E2M1Test, f32_inf) { + const auto f4 = ov::float4_e2m1(std::numeric_limits::infinity()); + // f4 is max as there is no infinity + EXPECT_EQ(f4.to_bits(), 0b0111); +} + +TEST(F4E2M1Test, f32_minus_inf) { + const auto f4 = ov::float4_e2m1(-std::numeric_limits::infinity()); + // f4 is max as there is no infinity + EXPECT_EQ(f4.to_bits(), 0b1111); +} + +TEST(F4E2M1Test, f32_gt_zero_round_to_f4_zero) { + const auto f4 = ov::float4_e2m1(0.218749985098838806152f); + + EXPECT_EQ(f4.to_bits(), 0b0000); +} + +TEST(F4E2M1Test, f32_gt_zero_round_to_f4_lowest_subnormal) { + const auto f4 = ov::float4_e2m1(0.21875f); + + EXPECT_EQ(f4.to_bits(), 0b0001); +} + +TEST(F4E2M1Test, f32_normal_fractional_rounding) { + const auto f4 = ov::float4_e2m1(1.75f); + + EXPECT_EQ(f4.to_bits(), 0b0100); +} + +TEST(F4E2M1Test, f32_normal_negative_fractional_rounding) { + const auto f4 = ov::float4_e2m1(-2.1f); + + EXPECT_EQ(f4.to_bits(), 0b1100); +} + +TEST(F4E2M1Test, f32_ge_f4_max_within_round_to_max) { + const auto f4 = ov::float4_e2m1(6.1f); + + EXPECT_EQ(f4.to_bits(), 0b0111); +} + +TEST(F4E2M1Test, f32_ge_f8_max_not_within_round_to_max) { + const auto f4 = ov::float4_e2m1(7.0f); + + EXPECT_EQ(f4.to_bits(), 0b0111); +} + +TEST(F4E2M1Test, f32_le_f8_lowest_within_round_to_lowest) { + const auto f4 = ov::float4_e2m1(-6.5f); + + EXPECT_EQ(f4.to_bits(), 0b1111); +} + +TEST(F4E2M1Test, f32_le_f8_lowest_not_within_round_to_lowest) { + const auto f4 = ov::float4_e2m1(-7.0f); + + EXPECT_EQ(f4.to_bits(), 0b1111); +} + +TEST(F4E2M1Test, stream_operator) { + std::stringstream s; + s << ov::float4_e2m1(-1.5f); + + EXPECT_EQ(s.str(), "-1.5"); +} + +TEST(F4E2M1Test, to_string) { + const auto f4 = ov::float4_e2m1::from_bits(0b00000110); + + EXPECT_EQ(std::to_string(f4), "4.000000"); +} + +// clang-format off +const auto exp_floats = std::vector{ + 0.0f, 0.5f, + 1.0f, 1.5f, + 2.0f, 3.0f, + 4.0f, 6.0f, + -0.0f, -0.5f, + -1.0f, -1.5f, + -2.0f, -3.0f, + -4.0f, -6.0f}; +// clang-format on + +using f4m2e1_params = std::tuple; +class F4E2M1PTest : public testing::TestWithParam {}; + +INSTANTIATE_TEST_SUITE_P(convert, + F4E2M1PTest, + testing::ValuesIn(enumerate(exp_floats)), + testing::PrintToStringParamName()); + +TEST_P(F4E2M1PTest, f4_bits_to_f32) { + const auto& params = GetParam(); + const auto& exp_value = std::get<1>(params); + const auto f4 = ov::float4_e2m1::from_bits(std::get<0>(params)); + + EXPECT_EQ(static_cast(f4), exp_value); +} + +TEST_P(F4E2M1PTest, f32_to_f4_bits) { + const auto& params = GetParam(); + const auto& exp_value = std::get<0>(params); + const auto& value = std::get<1>(params); + const auto f4 = ov::float4_e2m1(value); + + EXPECT_EQ(f4.to_bits(), exp_value); +} +} // namespace test +} // namespace ov diff --git a/src/plugins/intel_cpu/src/nodes/executors/type_mask.hpp b/src/plugins/intel_cpu/src/nodes/executors/type_mask.hpp index 199a44f9c28..7a1ace29623 100644 --- a/src/plugins/intel_cpu/src/nodes/executors/type_mask.hpp +++ b/src/plugins/intel_cpu/src/nodes/executors/type_mask.hpp @@ -34,7 +34,8 @@ struct TypeMask { _nf4 = 1 << 18, _f8e4m3 = 1 << 19, _f8e5m2 = 1 << 20, - _string = 1 << 21 + _string = 1 << 21, + _f4e2m1 = 1 << 22 }; TypeMask(const ov::element::Type precision) : value(generateMask(precision)), precision(precision) {} @@ -81,6 +82,7 @@ private: CASE(f8e4m3) CASE(f8e5m2) CASE(string) + CASE(f4e2m1) default: return _undefined; } @@ -114,6 +116,7 @@ DEFINE_TYPE_ALIAS(_nf4); DEFINE_TYPE_ALIAS(_f8e4m3); DEFINE_TYPE_ALIAS(_f8e5m2); DEFINE_TYPE_ALIAS(_string); +DEFINE_TYPE_ALIAS(_f4e2m1); constexpr auto _any_float = _f64 | _f32 | _f16 | _bf16; constexpr auto _half_float = _f16 | _bf16; constexpr auto _quant = _u8 | _i8; diff --git a/src/tests/test_utils/common_test_utils/include/common_test_utils/type_ranges.hpp b/src/tests/test_utils/common_test_utils/include/common_test_utils/type_ranges.hpp index a62f7f51e3d..1135288c1ab 100644 --- a/src/tests/test_utils/common_test_utils/include/common_test_utils/type_ranges.hpp +++ b/src/tests/test_utils/common_test_utils/include/common_test_utils/type_ranges.hpp @@ -79,6 +79,7 @@ static ov::test::utils::InputGenerateData get_range_by_type( } CASE_OV_TYPE(ov::element::Type_t::f8e4m3) CASE_OV_TYPE(ov::element::Type_t::f8e5m2) + CASE_OV_TYPE(ov::element::Type_t::f4e2m1) CASE_OV_TYPE(ov::element::Type_t::bf16) CASE_OV_TYPE(ov::element::Type_t::f16) CASE_C_TYPE(ov::element::Type_t::f32)