timekiller
/
mindspore2022
forked from huawei/mindspore2022
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

!28130 support for shared weight & support shared weight && compatible with r1.1 

Merge pull request !28130 from yeyunpeng2020/quant
This commit is contained in:
i-robot 2021-12-24 08:32:01 +00:00 committed by Gitee
parent cf68597b43 4249617792
commit 9c69bcbc9b
17 changed files with 361 additions and 371 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
mindspore/lite/src/common/quant_utils.cc
View File

@ -198,7 +198,9 @@ int CalWeightQuantBias(const float *raw_datas, size_t elem_count, const std::vec
quant_params->at(bucket_index).varCorr = 1;
if (var_raws[bucket_index] != 0 && var_dequants[bucket_index] != 0) {
auto temp_var_corr = var_raws[bucket_index] / var_dequants[bucket_index];
if (temp_var_corr > 0 && temp_var_corr < 10) {
const int min_var_corr = 0;
const int max_var_corr = 10;
if (temp_var_corr > min_var_corr && temp_var_corr < max_var_corr) {
quant_params->at(bucket_index).varCorr = temp_var_corr;
} else {
MS_LOG(WARNING) << "unexpected var_corr: " << temp_var_corr;

37
mindspore/lite/src/weight_decoder.cc
View File

@ -20,8 +20,43 @@
#include "tools/converter/quantizer/fse_decoder.h"
namespace mindspore::lite {
namespace {
constexpr int kBit8 = 8;
constexpr int kBit32 = 32;
bool HasInitQuantParam(const std::vector<LiteQuantParam> &quant_params) {
if (quant_params.empty()) {
return false;
}
if (std::all_of(quant_params.cbegin(), quant_params.cend(),
[](const LiteQuantParam &quant_param) { return quant_param.inited; })) {
return true;
}
return false;
}
bool CheckNeedWeightQuant(OpParameter *op_parameter, const std::vector<Tensor *> &in_tensors) {
if (op_parameter->quant_type_ == schema::QuantType_QUANT_WEIGHT) {
return true;
}
// compatible with r1.1
if (op_parameter->quant_type_ == schema::QuantType_QUANT_NONE) {
const size_t min_quant_size = 2;
if (in_tensors.size() < min_quant_size) {
return false;
}
for (auto tensor : in_tensors) {
if (!tensor->IsConst() && HasInitQuantParam(tensor->quant_params())) {
MS_LOG(DEBUG) << tensor->tensor_name()
<< " is a non-const tensor, but there are quantization parameters, which may belong to full "
"quantization.";
return false;
}
}
return true;
}
return false;
}
} // namespace
std::vector<bool> StringToBitVector(const std::string &str) {
std::vector<bool> vec(str.size() * kBit8);
size_t index = 0;
@ -355,7 +390,7 @@ int WeightDecoder::UnPack(const SchemaTensorWrapper &src_tensor, lite::Tensor *d
int WeightDecoder::DequantNode(OpParameter *op_parameter, const std::vector<Tensor *> &in_tensors,
TypeId dst_data_type) {
if (op_parameter->quant_type_ != schema::QuantType_QUANT_WEIGHT) {
if (!CheckNeedWeightQuant(op_parameter, in_tensors)) {
return RET_OK;
}
int index = 0;

2
mindspore/lite/test/st/scripts/dpico/run_simulation_3403.sh
View File

@ -247,11 +247,9 @@ om_generated_path=${basepath}/om_generated
# Write converter result to temp file
run_converter_log_file=${basepath}/run_converter_log.txt
rm ${run_converter_log_file}
echo ' ' > ${run_converter_log_file}
run_converter_result_file=${basepath}/run_converter_result.txt
rm ${run_converter_result_file}
echo ' ' > ${run_converter_result_file}
# Run converter

4
mindspore/lite/tools/anf_exporter/anf_exporter.cc
View File

@ -141,9 +141,9 @@ static STATUS CompressTensor(schema::TensorT *tensor_input, const std::unique_pt
return RET_ERROR;
}
} else if (bit_num <= kBitNum8) {
repetition_packed = PackRepetition<int8_t>(bit_num, tensor_input);
repetition_packed = quant::PackRepetition<int8_t>(bit_num, tensor_input);
} else {
repetition_packed = PackRepetition<int16_t>(bit_num, tensor_input);
repetition_packed = quant::PackRepetition<int16_t>(bit_num, tensor_input);
}
}
if (bit_num != kBitNum8 && bit_num != kBitNum16 && !repetition_packed &&

2
mindspore/lite/tools/anf_exporter/fetch_content.cc
View File

@ -26,7 +26,7 @@
#include "utils/ms_utils_secure.h"
#include "tools/optimizer/common/format_utils.h"
#include "nnacl/op_base.h"
#include "tools/anf_exporter/anf_exporter.h"
#include "tools/common/node_util.h"
#include "src/ops/ops_utils.h"
#include "src/ops/populate/populate_register.h"

19
mindspore/lite/tools/common/graph_util.cc
View File

@ -415,25 +415,6 @@ std::vector<int> GetTransposePerm(MetaGraphT *graph, const std::unique_ptr<CNode
return perm;
}
namespace {
constexpr size_t kBitNumPerByte = 8;
}
std::string BoolVectorToString(const std::vector<bool> &bool_vec) {
size_t size_in_byte = ceil(bool_vec.size() / kBitNumPerByte);
std::string str(size_in_byte, '\0');
auto iter = str.begin();
size_t shift = kBitNumPerByte;
for (bool bit : bool_vec) {
*iter |= bit << (shift - 1);
if (--shift == 0) {
iter++;
shift = kBitNumPerByte;
}
}
return str;
}
TypeId GetAbstractTensorDtype(const abstract::AbstractTensorPtr &tensor) {
if (tensor == nullptr || tensor->element() == nullptr) {
MS_LOG(ERROR) << "abstract_tensor or abstract_tensor->element() is nullptr";

206
mindspore/lite/tools/common/graph_util.h
View File

@ -82,8 +82,6 @@ std::string GetModelName(const std::string &modelFile);
std::vector<int> GetTransposePerm(schema::MetaGraphT *graph, const std::unique_ptr<schema::CNodeT> &cnode);
std::string BoolVectorToString(const std::vector<bool> &bool_vec);
TypeId GetAbstractTensorDtype(const abstract::AbstractTensorPtr &tensor);
TypeId GetParameterDtype(const ParameterPtr &param_node);
@ -93,210 +91,6 @@ STATUS UpdateFuncGraphInputsAndOutputsDtype(const FuncGraphPtr &func_graph);
STATUS UpdateGraphOutputName(schema::MetaGraphT *meta_graph);
int TransferMetaGraph(const schema::MetaGraphT &graph, void **model_buf, size_t *size);
template <typename T>
bool IndexingCompress(const std::set<T> &quant_data_set, const std::map<T, size_t> &unique_value_index_map,
size_t unique_value_bit, size_t unique_value_cnt, size_t pack_repetition_size_in_byte,
size_t bit_num, schema::TensorT *tensor) {
auto quant_data_array = reinterpret_cast<T *>(tensor->data.data());
std::vector<T> quant_data(quant_data_array, quant_data_array + tensor->data.size() / sizeof(T));
std::vector<bool> bits(pack_repetition_size_in_byte * 8);
size_t index = 0;
// write unique_value_cnt: bit_num bit for unsigned
for (size_t i = 0; i < bit_num; i++) {
bits[index++] = (unique_value_cnt >> (bit_num - i - 1)) & (0x1);
}
// write the unique value set: each value has bit_num bit signed
for (auto unique_value : quant_data_set) {
for (size_t i = 0; i < bit_num; i++) {
bits[index++] = ((unique_value + (1 << (bit_num - 1))) >> (bit_num - i - 1)) & (0x1);
}
}
// write the index: each index has unique_value_bit unsigned
for (auto quant_value : quant_data) {
for (size_t i = 0; i < unique_value_bit; i++) {
bits[index++] = (unique_value_index_map.at(quant_value) >> (unique_value_bit - i - 1)) & (0x1);
}
}
if (index > pack_repetition_size_in_byte * 8) {
MS_LOG(ERROR) << "unexpected index: " << index << " should not be greater than "
<< pack_repetition_size_in_byte * 8;
return false;
}
// update tensor data
auto new_data_str = BoolVectorToString(bits);
auto ret = memcpy_s(tensor->data.data(), tensor->data.size(), new_data_str.c_str(), new_data_str.size());
if (ret != EOK) {
MS_LOG(ERROR) << "memcpy error";
return false;
}
tensor->data.resize(new_data_str.size());
tensor->weightQunatCompressType = schema::WeightQunatCompressType_INDEXING;
MS_LOG(DEBUG) << "set WeightQunatCompressType_INDEXING";
return true;
}
template <typename T>
bool SparsityCompress(const std::set<T> &quant_data_set, const std::map<T, size_t> &unique_value_index_map,
size_t unique_value_bit, size_t unique_value_cnt, size_t pack_sparsity_size_in_byte,
size_t nz_cnt, size_t coor_best_bit, size_t bit_num, schema::TensorT *tensor) {
auto quant_data_array = reinterpret_cast<T *>(tensor->data.data());
std::vector<T> quant_data(quant_data_array, quant_data_array + tensor->data.size() / sizeof(T));
auto &quant_params = tensor->quantParams;
auto elem_cnt = quant_data.size();
auto channel_cnt = quant_params.size();
MS_CHECK_TRUE_MSG(channel_cnt != 0, false, "div zero.");
auto elem_perchannel = elem_cnt / channel_cnt;
std::vector<bool> bits(pack_sparsity_size_in_byte * 8);
int index = 0;
// coor_best_bit
for (size_t i = 0; i < 8; i++) {
bits[index++] = (coor_best_bit >> (8 - i - 1)) & 0x1;
}
// nz_cnt
for (size_t i = 0; i < 32; i++) {
bits[index++] = (nz_cnt >> (32 - i - 1)) & 0x1;
}
// unique_value cnt
for (size_t i = 0; i < bit_num; i++) {
bits[index++] = (unique_value_cnt >> (bit_num - i - 1)) & 0x1;
}
// unique_values
for (auto unique_value : quant_data_set) {
for (size_t i = 0; i < bit_num; i++) {
bits[index++] = ((unique_value + (1 << (bit_num - 1))) >> (bit_num - i - 1)) & (0x1);
}
}
// nz values indexing && get coor
std::vector<size_t> coors(nz_cnt);
size_t coors_index = 0;
size_t prev_index = -1;
for (size_t di = 0; di < elem_cnt; di++) {
auto cur_channel = di / elem_perchannel;
auto zp = quant_params[cur_channel]->zeroPoint;
auto nz_value = quant_data[di];
if (nz_value != zp || (di - prev_index) >= (size_t)(1 << coor_best_bit)) {
MS_ASSERT(coors_index < nz_cnt);
coors[coors_index++] = di - prev_index - 1;
prev_index = di;
for (size_t i = 0; i < unique_value_bit; i++) {
bits[index++] = (unique_value_index_map.at(nz_value) >> (unique_value_bit - i - 1)) & (0x1);
}
}
}
// write coor
for (auto coor : coors) {
for (size_t i = 0; i < coor_best_bit; i++) {
bits[index++] = (coor >> (coor_best_bit - i - 1)) & 0x1;
}
}
if ((unsigned int)index > pack_sparsity_size_in_byte * 8) {
MS_LOG(ERROR) << "unexpected index: " << index << " should not be greater than " << pack_sparsity_size_in_byte * 8;
return false;
}
auto new_data_str = BoolVectorToString(bits);
auto ret = memcpy_s(tensor->data.data(), tensor->data.size(), new_data_str.c_str(), new_data_str.size());
if (ret != EOK) {
MS_LOG(ERROR) << "memcpy error";
return false;
}
tensor->data.resize(new_data_str.size());
tensor->weightQunatCompressType = schema::WeightQunatCompressType_SPARSE;
MS_LOG(INFO) << "set WeightQunatCompressType_SPARSITY";
return true;
}
template <typename T>
size_t CalCoorBestBit(const std::vector<T> &quant_data, size_t elem_cnt,
const std::vector<std::unique_ptr<schema::QuantParamT>> &quant_params, int unique_value_bit,
size_t *coor_best_bit) {
size_t best_nn_cnt = 0;
size_t min_len_in_bit = std::numeric_limits<size_t>::max();
for (int bit = 2; bit <= 10; bit++) {
// search
size_t nn_cnt = 0;
size_t prev_index = -1;
auto channel_cnt = quant_params.size();
auto elem_perchannel = elem_cnt / channel_cnt;
for (size_t i = 0; i < elem_cnt; i++) {
auto cur_channel = i / elem_perchannel;
auto zp = quant_params[cur_channel]->zeroPoint;
if (quant_data[i] != zp || (i - prev_index) >= (size_t)(1 << bit)) {
nn_cnt++;
prev_index = i;
}
}
size_t len_in_bit = nn_cnt * bit + nn_cnt * unique_value_bit;
if (len_in_bit < min_len_in_bit) {
min_len_in_bit = len_in_bit;
*coor_best_bit = bit;
best_nn_cnt = nn_cnt;
}
}
return best_nn_cnt;
}
template <typename T>
bool PackRepetition(size_t bit_num, schema::TensorT *tensor) {
auto quant_data_array = reinterpret_cast<T *>(tensor->data.data());
std::vector<T> quant_data(quant_data_array, quant_data_array + tensor->data.size() / sizeof(T));
auto elem_cnt = quant_data.size();
auto dims = tensor->dims;
size_t elem_cnt_by_dims = std::accumulate(dims.begin(), dims.end(), 1, std::multiplies<>());
if (elem_cnt != elem_cnt_by_dims) {
MS_LOG(ERROR) << "elem_cnt: " << elem_cnt << " not equal elem_cnt_by_dims: " << elem_cnt_by_dims;
return false;
}
auto &quant_params = tensor->quantParams;
std::set<T> quant_data_set;
for (auto quant_value : quant_data) {
quant_data_set.insert(quant_value);
}
std::map<T, size_t> unique_value_index_map;
auto index = 0;
for (auto value : quant_data_set) {
unique_value_index_map[value] = index++;
}
auto unique_value_cnt = quant_data_set.size();
size_t unique_value_bit = ceil(log2(unique_value_cnt));
auto pack_repetition_size_in_bit = bit_num + bit_num * unique_value_cnt + unique_value_bit * elem_cnt;
size_t pack_repetition_size_in_byte = ceil(pack_repetition_size_in_bit / 8.0);
size_t origin_size_in_byte = ceil(bit_num * elem_cnt / 8.0);
size_t coor_best_bit = 0;
auto nz_cnt = CalCoorBestBit<T>(quant_data, elem_cnt, quant_params, unique_value_bit, &coor_best_bit);
// 1. coor_best_bit 2. nz_cnt 3. quant_data_set size 4. unique_values 5. unique_value indexing 6. nz values coord
auto pack_sparsity_size_in_bit =
1 * 8 + 4 * 8 + bit_num + bit_num * unique_value_cnt + unique_value_bit * nz_cnt + nz_cnt * coor_best_bit;
size_t pack_sparsity_size_in_byte = ceil(pack_sparsity_size_in_bit / 8.0);
MS_LOG(DEBUG) << "coor_best_bit: " << coor_best_bit << " ori: " << origin_size_in_byte
<< " indexing: " << pack_repetition_size_in_byte << " sparse: " << pack_sparsity_size_in_byte;
auto min_byte_need = std::min({origin_size_in_byte, pack_repetition_size_in_byte, pack_sparsity_size_in_byte});
if (min_byte_need == origin_size_in_byte) {
return false;
} else if (min_byte_need == pack_repetition_size_in_byte) {
MS_LOG(DEBUG) << "from " << origin_size_in_byte << " to " << pack_repetition_size_in_byte;
return IndexingCompress<T>(quant_data_set, unique_value_index_map, unique_value_bit, unique_value_cnt,
pack_repetition_size_in_byte, bit_num, tensor);
} else if (min_byte_need == pack_sparsity_size_in_byte) {
MS_LOG(DEBUG) << "from " << origin_size_in_byte << " to " << pack_sparsity_size_in_byte;
return SparsityCompress<T>(quant_data_set, unique_value_index_map, unique_value_bit, unique_value_cnt,
pack_sparsity_size_in_byte, nz_cnt, coor_best_bit, bit_num, tensor);
} else {
MS_LOG(DEBUG) << "unexpected: " << min_byte_need << " not in {" << origin_size_in_byte << " "
<< pack_repetition_size_in_byte << " " << pack_sparsity_size_in_byte << "}";
}
return false;
}
} // namespace lite
} // namespace mindspore

25
mindspore/lite/tools/converter/quantizer/data_distribution.cc
View File

@ -26,16 +26,20 @@ int DataDistribution::RecordMaxMinValueArray(const std::vector<float> &data) {
if (data.empty()) {
return RET_ERROR;
}
float max_num = data.at(0);
float min_num = data.at(0);
float max_num = data.at(0);
for (float val : data) {
max_num = std::max(val, max_num);
min_num = std::min(val, min_num);
max_num = std::max(val, max_num);
}
real_max_ = std::max(max_num, real_max_);
real_min_ = std::min(min_num, real_min_);
this->max_datas_.emplace_back(max_num);
this->min_datas_.emplace_back(min_num);
real_max_ = std::max(max_num, real_max_);
auto quantile_min = Quantile(data, 0.0001);
auto quantile_max = Quantile(data, 0.9999);
MS_LOG(DEBUG) << "real_min_:" << real_min_ << " real_max_:" << real_max_ << "quantile_min:" << quantile_min
<< " quantile_max:" << quantile_max;
this->min_datas_.emplace_back(quantile_min);
this->max_datas_.emplace_back(quantile_max);
return RET_OK;
}
@ -166,10 +170,19 @@ int DataDistribution::ComputeThreshold() {
double DataDistribution::CalculateMinMaxScale() { return CalculateScaleAndZp(this->real_min_, this->real_max_); }
double DataDistribution::CalculateRemovalOutlierScale() {
this->percent_result_ = OutlierMethod(min_datas_, max_datas_);
this->percent_result_ = CalQuantileMinMax(min_datas_, max_datas_);
return CalculateScaleAndZp(percent_result_.first, percent_result_.second);
}
std::pair<float, float> DataDistribution::CalQuantileMinMax(const std::vector<float> &min_datas,
const std::vector<float> &max_datas) {
MS_ASSERT(!min_datas.empty());
MS_ASSERT(!max_datas.empty());
auto avg_min = accumulate(min_datas.begin(), min_datas.end(), 0.0) / min_datas.size();
auto avg_max = accumulate(max_datas.begin(), max_datas.end(), 0.0) / max_datas.size();
return {avg_min, avg_max};
}
double DataDistribution::CalculateScaleAndZp(float min_value, float max_value) {
if (symmetry_) {
auto abs_max = std::max(fabs(min_value), fabs(max_value));

6
mindspore/lite/tools/converter/quantizer/data_distribution.h
View File

@ -76,13 +76,15 @@ class DataDistribution {
double CalculateKLScale();
double CalculateScaleAndZp(float min_value, float max_value);
std::pair<float, float> CalQuantileMinMax(const std::vector<float> &min_datas, const std::vector<float> &max_datas);
private:
std::vector<float> histogram_;
CNodePtr cnode_;
int bin_num_ = 0;
float interval_ = 0;
float real_max_ = 0.0f;
float real_min_ = 0.0f;
float real_max_ = FLT_MIN;
float real_min_ = FLT_MAX;
float best_T_ = 0.0f;
size_t bit_num_ = 0;
float encode_min_ = 0.0f;

44
mindspore/lite/tools/converter/quantizer/debug_info_manager.cc
View File

@ -562,6 +562,28 @@ int DebugInfoManager::SaveQuantParam(const std::string &file_path) {
return RET_OK;
}
int DebugInfoManager::GetClipAndCos() {
for (auto &info : compared_info_) {
auto iter = origin_info_.find(info.primary_key);
if (iter == origin_info_.end()) {
continue;
}
if (iter->second.tensor_data.data_type != info.tensor_data.data_type ||
iter->second.tensor_data.size != info.tensor_data.size ||
iter->second.tensor_data.elements_num != info.tensor_data.elements_num) {
MS_LOG(ERROR) << info.primary_key << " "
<< " data is not match origin";
FreeBuffer();
return RET_ERROR;
}
info.cos_similarity = mindspore::lite::GetCosSimilarity(iter->second.tensor_data.data, info.tensor_data.data,
info.tensor_data.elements_num, info.tensor_data.data_type);
info.clip = mindspore::lite::GetClipRate(iter->second.tensor_data.data, info.tensor_data.data,
info.tensor_data.elements_num, info.tensor_data.data_type);
}
return RET_OK;
}
int DebugInfoManager::CompareOriginWithQuant(const quant::SessionModel &origin, const quant::SessionModel &quant,
const std::map<std::string, OpParameter *> &op_parameters,
const std::string &debug_info_save_path,
@ -617,24 +639,10 @@ int DebugInfoManager::CompareOriginWithQuant(const quant::SessionModel &origin,
FreeBuffer();
return ret;
}
for (auto &info : compared_info_) {
auto iter = origin_info_.find(info.primary_key);
if (iter == origin_info_.end()) {
continue;
}
if (iter->second.tensor_data.data_type != info.tensor_data.data_type ||
iter->second.tensor_data.size != info.tensor_data.size ||
iter->second.tensor_data.elements_num != info.tensor_data.elements_num) {
MS_LOG(ERROR) << info.primary_key << " "
<< " data is not match origin";
FreeBuffer();
return RET_ERROR;
}
info.cos_similarity =
mindspore::lite::GetCosSimilarity(iter->second.tensor_data.data, info.tensor_data.data,
info.tensor_data.elements_num, info.tensor_data.data_type);
info.clip = mindspore::lite::GetClipRate(iter->second.tensor_data.data, info.tensor_data.data,
info.tensor_data.elements_num, info.tensor_data.data_type);
ret = GetClipAndCos();
if (ret != RET_OK) {
MS_LOG(ERROR) << "Get clip and cos failed.";
return ret;
}
auto info_save_path = debug_info_save_path + FILE_SEPARATOR + "round" + "_" + std::to_string(round) + ".csv";
ret = SaveInfo(info_save_path);

3
mindspore/lite/tools/converter/quantizer/debug_info_manager.h
View File

@ -138,6 +138,9 @@ class DebugInfoManager {
void PrintQuantParam();
int SaveQuantParam(const std::string &file_path);
int GetClipAndCos();
template <typename T>
void GetStatByTensor(const T *tensor_data, size_t element_num, QuantDebugInfo *infos) {
MS_ASSERT(infos != nullptr);

3
mindspore/lite/tools/converter/quantizer/fse_decoder.cc
View File

@ -135,7 +135,8 @@ int FSEDecoder::DeCompress(const SchemaTensorWrapper &src_tensor, Tensor *dst_te
return RET_ERROR;
}
bs.SetChunkCount(*(reinterpret_cast<uint32_t *>(&data8[i])));
bs.SetCurrChunkIndex(bs.GetChunkCount() - 2);
const int offset = 2;
bs.SetCurrChunkIndex(bs.GetChunkCount() - offset);
i += sizeof(uint32_t);
if (i > total_size) {
MS_LOG(ERROR) << "index over total size"

28
mindspore/lite/tools/converter/quantizer/full_quant_quantizer.cc
View File

@ -307,7 +307,14 @@ int FullQuantQuantizer::IsSupportWeightQuant(const CNodePtr &cnode, const AnfNod
}
// support for share weight.
if (type_id == kNumberTypeInt8) {
return RET_NO_CHANGE;
auto iter = weight_quant_params_bak.find(input_node->fullname_with_scope());
if (iter == weight_quant_params_bak.end()) {
return RET_ERROR;
} else {
auto quant_param_holder = GetCNodeQuantHolder(primitive);
quant_param_holder->set_input_quant_param(input_index - 1, iter->second);
return RET_NO_CHANGE;
}
}
// Only data the data type is fp32 can be quant.
if (type_id != kNumberTypeFloat32) {
@ -378,7 +385,6 @@ int FullQuantQuantizer::QuantNodeSimpleOp(const CNodePtr &cnode) {
auto op_name = cnode->fullname_with_scope();
auto primitive_quant_holder = GetCNodeQuantHolder(primitive);
MS_CHECK_TRUE_MSG(primitive_quant_holder != nullptr, RET_NULL_PTR, "primitive_quant_holder is nullptr.");
size_t activation_input_index = 0;
int ret;
for (size_t i = 1; i < cnode->inputs().size(); i++) {
auto input_node = cnode->input(i);
@ -386,7 +392,7 @@ int FullQuantQuantizer::QuantNodeSimpleOp(const CNodePtr &cnode) {
bool is_graph_input = IsGraphInput(input_node);
if (is_graph_input) {
// do input quant
auto &info = (*inputs_diverg_info)[op_name][activation_input_index++];
auto &info = (*inputs_diverg_info)[op_name][i - 1];
ret = SetInOutQuantParam(input_node, info, primitive, true, i - 1);
if (ret != RET_OK) {
MS_LOG(ERROR) << input_node->fullname_with_scope() << " Set activation quant failed.";
@ -406,10 +412,9 @@ int FullQuantQuantizer::QuantNodeSimpleOp(const CNodePtr &cnode) {
if (input_primitive_quant_holder->IsOutputQuantParamsInited()) {
auto quant_param = input_primitive_quant_holder->get_output_quant_params().front();
primitive_quant_holder->set_input_quant_param(i - 1, quant_param);
activation_input_index++;
} else {
// do input quant
auto &info = (*inputs_diverg_info)[op_name][activation_input_index++];
auto &info = (*inputs_diverg_info)[op_name][i - 1];
ret = SetInOutQuantParam(input_node, info, primitive, true, i - 1);
if (ret != RET_OK) {
MS_LOG(ERROR) << input_node->fullname_with_scope() << " Set activation quant failed.";
@ -424,6 +429,9 @@ int FullQuantQuantizer::QuantNodeSimpleOp(const CNodePtr &cnode) {
MS_LOG(ERROR) << input_node->fullname_with_scope() << " Do parameter node quant failed.";
return ret;
}
// support shared weight
weight_quant_params_bak[input_node->fullname_with_scope()] =
primitive_quant_holder->get_input_quant_params()[i - 1];
} else if (input_node->isa<mindspore::ValueNode>()) {
ret = DoValueNodeQuant(cnode, input_node->cast<ValueNodePtr>(), i);
if (ret == RET_NO_CHANGE) {
@ -432,6 +440,9 @@ int FullQuantQuantizer::QuantNodeSimpleOp(const CNodePtr &cnode) {
MS_LOG(ERROR) << input_node->fullname_with_scope() << " Do value node quant failed.";
return ret;
}
// support shared weight
weight_quant_params_bak[input_node->fullname_with_scope()] =
primitive_quant_holder->get_input_quant_params()[i - 1];
} else {
MS_LOG(ERROR) << input_node->fullname_with_scope() << ":" << input_node->type_name() << " is not support type";
return RET_ERROR;
@ -891,8 +902,6 @@ int FullQuantQuantizer::BiasCorrection(const FuncGraphPtr &func_graph, const CNo
return RET_OK;
}
int FullQuantQuantizer::ComputeThreshold() { return this->calibrator_->ComputeThreshold(); }
int FullQuantQuantizer::DoQuantize(FuncGraphPtr func_graph) {
MS_LOG(INFO) << "start to parse config file";
if (flags_.dataPreProcessParam.calibrate_path.empty()) {
@ -937,7 +946,7 @@ int FullQuantQuantizer::DoQuantize(FuncGraphPtr func_graph) {
return status;
}
MS_LOG(INFO) << "compute the best threshold";
status = ComputeThreshold();
status = this->calibrator_->ComputeThreshold();
if (status != RET_OK) {
MS_LOG(ERROR) << "compute threshold failed.";
return status;
@ -974,7 +983,8 @@ int FullQuantQuantizer::DoQuantize(FuncGraphPtr func_graph) {
MS_LOG(INFO) << "do bias correction";
status = BiasCorrection(func_graph);
if (status != RET_OK) {
MS_LOG(WARNING) << "BiasCorrection failed.";
MS_LOG(ERROR) << "BiasCorrection failed.";
return status;
}
}
}

6
mindspore/lite/tools/converter/quantizer/full_quant_quantizer.h
View File

@ -56,6 +56,7 @@ class FullQuantQuantizer : public Quantizer {
private:
bool OpInputDataHandle(OperationType type, const string &op_name, std::vector<float> *data);
bool OpOutputChMeanDataHandle(OperationType type, const string &op_name, std::vector<float> *data);
int PreProcess(const FuncGraphPtr &func_graph);
@ -66,8 +67,6 @@ class FullQuantQuantizer : public Quantizer {
int UpdateDivergeInterval();
int ComputeThreshold();
int QuantNodeSimpleOp(const CNodePtr &cnode);
int QuantNode(const FuncGraphPtr &func_graph);
@ -129,6 +128,9 @@ class FullQuantQuantizer : public Quantizer {
std::map<std::string, std::vector<float>> op_bias_diff_map_; // only use by int8 model
std::mutex mutex_op_input_;
std::mutex mutex_op_output_;
// key is tensor_name
std::map<std::string, std::vector<schema::QuantParamT>> weight_quant_params_bak;
};
} // namespace mindspore::lite::quant
#endif // MINDSPORE_LITE_TOOLS_CONVERTER_QUANTIZER_FULL_QUANT_QUANTIZER_H

2
mindspore/lite/tools/converter/quantizer/parameter_tunner.cc
View File

@ -141,7 +141,7 @@ int ParameterOptimizer::WeightQuantModelInference(const FuncGraphPtr &func_graph
break;
}
}
MS_CHECK_TRUE_MSG(weight_quant_size > 0, RET_ERROR, "weight quant size must large 0");
auto compress_ratio = 1.0 * origin_model_size / weight_quant_size;
std::cout << " round:" << round << " scale:" << scale << " cos_sim:" << cos_sim << " mean_error:" << mean_error
<< " ratio:" << compress_ratio << std::endl;

122
mindspore/lite/tools/converter/quantizer/quantize_util.cc
View File

@ -38,13 +38,15 @@ using std::string;
using std::vector;
namespace mindspore::lite::quant {
const int kLstmInputWeightIndex = 1;
const int kLstmStateWeightIndex = 2;
const int kLstmWeightShapeSize = 3;
const int kSingleDirBiasTensorSize = 4;
const int kLstmBiasShapeSize = 2;
const int kLstmBiasIndex = 3;
namespace {
constexpr int kLstmInputWeightIndex = 1;
constexpr int kLstmStateWeightIndex = 2;
constexpr int kLstmWeightShapeSize = 3;
constexpr int kSingleDirBiasTensorSize = 4;
constexpr int kLstmBiasShapeSize = 2;
constexpr int kLstmBiasIndex = 3;
constexpr size_t kBitNumPerByte = 8;
} // namespace
QuantParamHolderPtr GetCNodeQuantHolder(const PrimitivePtr &primitive) {
MS_CHECK_TRUE_RET(primitive != nullptr, nullptr);
QuantParamHolderPtr quant_params_holder = nullptr;
@ -77,97 +79,6 @@ bool TensorQuantParamsInited(const schema::TensorT &tensor) {
return true;
}
static bool SearchLowerBound(const std::vector<float> &data, const size_t &index, const float &max_tmp, float *min_tmp,
size_t *min_idx) {
MS_ASSERT(!data.empty());
size_t length = data.size();
if (max_tmp - data.at(index) < delta) {
return false;
}
if (fabs(max_tmp - *min_tmp) <= 0.0f || fabs(length - *min_idx) <= 0.0f) {
MS_LOG(INFO) << "divisor cannot be 0";
return false;
}
float range_ratio = (data.at(index) - *min_tmp) / (max_tmp - *min_tmp);
float index_ratio = static_cast<float>(index - *min_idx) / (length - *min_idx);
if (fabs(index_ratio) <= 0.0f) {
MS_LOG(INFO) << "divisor cannot be 0";
return false;
}
if (index_ratio > 0 && range_ratio / index_ratio > ratio) {
*min_idx = index;
*min_tmp = data.at(index);
}
return true;
}
static bool SearchUpperBound(const std::vector<float> &data, const size_t &index, float *max_tmp, const float &min_tmp,
size_t *max_idx) {
MS_ASSERT(!data.empty());
size_t length = data.size();
if (data.at(index) - min_tmp < delta) {
return false;
}
if (fabs(*max_tmp - min_tmp) <= 0.0f || fabs(length - *max_idx) <= 0.0f) {
MS_LOG(INFO) << "divisor cannot be 0";
return false;
}
float range_ratio = (*max_tmp - data.at(index)) / (*max_tmp - min_tmp);
float index_ratio = static_cast<float>(index - *max_idx) / (length - *max_idx);
if (fabs(index_ratio) <= 0.0f) {
MS_LOG(INFO) << "divisor cannot be 0";
return false;
}
if (index_ratio > 0 && range_ratio / index_ratio > ratio) {
*max_idx = index;
*max_tmp = data.at(index);
}
return true;
}
static float CalPercentile(const std::vector<float> &data, const int &outlier_percent) {
MS_ASSERT(!data.empty());
const int size = data.size();
float val = outlier_percent / kPercentBase * size;
int index = std::ceil(val);
float result;
if (index - val > 0) {
MS_ASSERT(index - 1 >= 0);
result = data.at(index - 1);
} else {
MS_ASSERT(index - 1 >= 0);
result = (data.at(index - 1) + data.at(index)) / 2;
}
return result;
}
std::pair<float, float> OutlierMethod(std::vector<float> min_datas, std::vector<float> max_datas) {
MS_ASSERT(!min_datas.empty());
MS_ASSERT(!max_datas.empty());
std::sort(max_datas.begin(), max_datas.end());
std::sort(min_datas.begin(), min_datas.end());
float min_val = CalPercentile(min_datas, percent);
float max_val = CalPercentile(max_datas, kPercentBase - percent);
std::reverse(max_datas.begin(), max_datas.end());
MS_ASSERT(min_val < max_val);
MS_ASSERT(min_datas.size() == max_datas.size());
float min_tmp = min_val;
float max_tmp = max_val;
size_t min_idx = 0;
size_t max_idx = 0;
size_t length = min_datas.size();
for (size_t i = 0; i < length; i++) {
if (!SearchLowerBound(min_datas, i, max_tmp, &min_tmp, &min_idx)) {
break;
}
if (!SearchUpperBound(min_datas, i, &max_tmp, min_tmp, &max_idx)) {
break;
}
}
std::pair<float, float> result{min_tmp, max_tmp};
return result;
}
static std::vector<float> InitClusters(float *data, size_t elem_count, size_t k) {
MS_ASSERT(data != nullptr);
std::set<float> set_unique{};
@ -533,4 +444,19 @@ bool CheckNodeInSet(const CNodePtr &cnode, const std::set<PrimitivePtr> &support
}
return false;
}
std::string BoolVectorToString(const std::vector<bool> &bool_vec) {
size_t size_in_byte = ceil(bool_vec.size() / kBitNumPerByte);
std::string str(size_in_byte, '\0');
auto iter = str.begin();
size_t shift = kBitNumPerByte;
for (bool bit : bool_vec) {
*iter |= bit << (shift - 1);
if (--shift == 0) {
iter++;
shift = kBitNumPerByte;
}
}
return str;
}
} // namespace mindspore::lite::quant

219
mindspore/lite/tools/converter/quantizer/quantize_util.h
View File

@ -30,6 +30,8 @@
#include <algorithm>
#include <limits>
#include <utility>
#include <map>
#include <functional>
#include "ops/mat_mul.h"
#include "ops/lstm.h"
#include "ops/fusion/full_connection.h"
@ -55,8 +57,11 @@ enum WeightQuantType {
FIXED_BIT_PER_LAYER = 1,
MIXED_BIT_PER_LAYER = 2,
};
constexpr size_t k2Bit = 2;
constexpr size_t k8Bit = 8;
constexpr size_t k10Bit = 10;
constexpr size_t k16Bit = 16;
constexpr size_t k32Bit = 32;
constexpr size_t kMaxNum1024 = 1024;
constexpr float kPercentBase = 100.0;
constexpr size_t kMillisecondsBase = 10;
@ -71,8 +76,6 @@ struct SessionModel {
QuantParamHolderPtr GetCNodeQuantHolder(const PrimitivePtr &primitive);
std::pair<float, float> OutlierMethod(std::vector<float> min_datas, std::vector<float> max_datas);
std::vector<int8_t> KMeans(float *data, size_t elem_count, size_t k, size_t epochs, schema::QuantParamT *quantParam);
int UpdateTensorDataAndSize(const AnfNodePtr &node, const tensor::TensorPtr &weight, void *quant_datas, int new_size,
@ -174,5 +177,217 @@ SessionModel CreateSessionByFuncGraph(const FuncGraphPtr &func_graph, const conv
void GetLiteParameter(const AnfNodePtr &node, ParameterPtr *param_node, tensor::TensorPtr *tensor_info);
bool CheckNodeInSet(const CNodePtr &cnode, const std::set<PrimitivePtr> &support_primitive_types);
std::string BoolVectorToString(const std::vector<bool> &bool_vec);
template <typename T>
bool IndexingCompress(const std::set<T> &quant_data_set, const std::map<T, size_t> &unique_value_index_map,
size_t unique_value_bit, size_t unique_value_cnt, size_t pack_repetition_size_in_byte,
size_t bit_num, schema::TensorT *tensor) {
auto quant_data_array = reinterpret_cast<T *>(tensor->data.data());
std::vector<T> quant_data(quant_data_array, quant_data_array + tensor->data.size() / sizeof(T));
std::vector<bool> bits(pack_repetition_size_in_byte * k8Bit);
size_t index = 0;
// write unique_value_cnt: bit_num bit for unsigned
for (size_t i = 0; i < bit_num; i++) {
bits[index++] = (unique_value_cnt >> (bit_num - i - 1)) & (0x1);
}
// write the unique value set: each value has bit_num bit signed
for (auto unique_value : quant_data_set) {
for (size_t i = 0; i < bit_num; i++) {
bits[index++] = ((unique_value + (1 << (bit_num - 1))) >> (bit_num - i - 1)) & (0x1);
}
}
// write the index: each index has unique_value_bit unsigned
for (auto quant_value : quant_data) {
for (size_t i = 0; i < unique_value_bit; i++) {
bits[index++] = (unique_value_index_map.at(quant_value) >> (unique_value_bit - i - 1)) & (0x1);
}
}
if (index > pack_repetition_size_in_byte * k8Bit) {
MS_LOG(ERROR) << "unexpected index: " << index << " should not be greater than "
<< pack_repetition_size_in_byte * k8Bit;
return false;
}
// update tensor data
auto new_data_str = BoolVectorToString(bits);
auto ret = memcpy_s(tensor->data.data(), tensor->data.size(), new_data_str.c_str(), new_data_str.size());
if (ret != EOK) {
MS_LOG(ERROR) << "memcpy error";
return false;
}
tensor->data.resize(new_data_str.size());
tensor->weightQunatCompressType = schema::WeightQunatCompressType_INDEXING;
MS_LOG(DEBUG) << "set WeightQunatCompressType_INDEXING";
return true;
}
template <typename T>
bool SparsityCompress(const std::set<T> &quant_data_set, const std::map<T, size_t> &unique_value_index_map,
size_t unique_value_bit, size_t unique_value_cnt, size_t pack_sparsity_size_in_byte,
size_t nz_cnt, size_t coor_best_bit, size_t bit_num, schema::TensorT *tensor) {
auto quant_data_array = reinterpret_cast<T *>(tensor->data.data());
std::vector<T> quant_data(quant_data_array, quant_data_array + tensor->data.size() / sizeof(T));
auto &quant_params = tensor->quantParams;
auto elem_cnt = quant_data.size();
auto channel_cnt = quant_params.size();
MS_CHECK_TRUE_MSG(channel_cnt != 0, false, "div zero.");
auto elem_perchannel = elem_cnt / channel_cnt;
std::vector<bool> bits(pack_sparsity_size_in_byte * k8Bit);
int index = 0;
// coor_best_bit
for (size_t i = 0; i < k8Bit; i++) {
bits[index++] = (coor_best_bit >> (k8Bit - i - 1)) & 0x1;
}
// nz_cnt
for (size_t i = 0; i < k32Bit; i++) {
bits[index++] = (nz_cnt >> (k32Bit - i - 1)) & 0x1;
}
// unique_value cnt
for (size_t i = 0; i < bit_num; i++) {
bits[index++] = (unique_value_cnt >> (bit_num - i - 1)) & 0x1;
}
// unique_values
for (auto unique_value : quant_data_set) {
for (size_t i = 0; i < bit_num; i++) {
bits[index++] = ((unique_value + (1 << (bit_num - 1))) >> (bit_num - i - 1)) & (0x1);
}
}
// nz values indexing && get coor
std::vector<size_t> coors(nz_cnt);
size_t coors_index = 0;
size_t prev_index = -1;
for (size_t di = 0; di < elem_cnt; di++) {
auto cur_channel = di / elem_perchannel;
auto zp = quant_params[cur_channel]->zeroPoint;
auto nz_value = quant_data[di];
if (nz_value != zp || (di - prev_index) >= static_cast<size_t>((1 << coor_best_bit))) {
MS_ASSERT(coors_index < nz_cnt);
coors[coors_index++] = di - prev_index - 1;
prev_index = di;
for (size_t i = 0; i < unique_value_bit; i++) {
bits[index++] = (unique_value_index_map.at(nz_value) >> (unique_value_bit - i - 1)) & (0x1);
}
}
}
// write coor
for (auto coor : coors) {
for (size_t i = 0; i < coor_best_bit; i++) {
bits[index++] = (coor >> (coor_best_bit - i - 1)) & 0x1;
}
}
if ((unsigned int)index > pack_sparsity_size_in_byte * k8Bit) {
MS_LOG(ERROR) << "unexpected index: " << index << " should not be greater than "
<< pack_sparsity_size_in_byte * k8Bit;
return false;
}
auto new_data_str = BoolVectorToString(bits);
auto ret = memcpy_s(tensor->data.data(), tensor->data.size(), new_data_str.c_str(), new_data_str.size());
if (ret != EOK) {
MS_LOG(ERROR) << "memcpy error";
return false;
}
tensor->data.resize(new_data_str.size());
tensor->weightQunatCompressType = schema::WeightQunatCompressType_SPARSE;
MS_LOG(INFO) << "set WeightQunatCompressType_SPARSITY";
return true;
}
template <typename T>
size_t CalCoorBestBit(const std::vector<T> &quant_data, size_t elem_cnt,
const std::vector<std::unique_ptr<schema::QuantParamT>> &quant_params, int unique_value_bit,
size_t *coor_best_bit) {
MS_ASSERT(!quant_params.empty());
size_t best_nn_cnt = 0;
size_t min_len_in_bit = std::numeric_limits<size_t>::max();
for (size_t bit = k2Bit; bit <= k10Bit; bit++) {
// search
size_t nn_cnt = 0;
size_t prev_index = -1;
auto channel_cnt = quant_params.size();
MS_ASSERT(channel_cnt > 0);
auto elem_perchannel = elem_cnt / channel_cnt;
for (size_t i = 0; i < elem_cnt; i++) {
auto cur_channel = i / elem_perchannel;
auto zp = quant_params[cur_channel]->zeroPoint;
if (quant_data[i] != zp || (i - prev_index) >= static_cast<size_t>((1 << bit))) {
nn_cnt++;
prev_index = i;
}
}
size_t len_in_bit = nn_cnt * bit + nn_cnt * unique_value_bit;
if (len_in_bit < min_len_in_bit) {
min_len_in_bit = len_in_bit;
*coor_best_bit = bit;
best_nn_cnt = nn_cnt;
}
}
return best_nn_cnt;
}
template <typename T>
bool PackRepetition(size_t bit_num, schema::TensorT *tensor) {
if (tensor->weightQunatCompressType != schema::WeightQunatCompressType_NONE) {
MS_LOG(INFO) << tensor->name << " is shared weight.";
return true;
}
auto quant_data_array = reinterpret_cast<T *>(tensor->data.data());
std::vector<T> quant_data(quant_data_array, quant_data_array + tensor->data.size() / sizeof(T));
auto elem_cnt = quant_data.size();
auto dims = tensor->dims;
size_t elem_cnt_by_dims = std::accumulate(dims.begin(), dims.end(), 1, std::multiplies<>());
if (elem_cnt != elem_cnt_by_dims) {
MS_LOG(ERROR) << tensor->name << " elem_cnt: " << elem_cnt << " not equal elem_cnt_by_dims: " << elem_cnt_by_dims;
return false;
}
auto &quant_params = tensor->quantParams;
std::set<T> quant_data_set;
for (auto quant_value : quant_data) {
quant_data_set.insert(quant_value);
}
std::map<T, size_t> unique_value_index_map;
auto index = 0;
for (auto value : quant_data_set) {
unique_value_index_map[value] = index++;
}
auto unique_value_cnt = quant_data_set.size();
size_t unique_value_bit = ceil(log2(unique_value_cnt));
auto pack_repetition_size_in_bit = bit_num + bit_num * unique_value_cnt + unique_value_bit * elem_cnt;
size_t pack_repetition_size_in_byte = ceil(1.0 * pack_repetition_size_in_bit / k8Bit);
size_t origin_size_in_byte = ceil(1.0 * bit_num * elem_cnt / k8Bit);
size_t coor_best_bit = 0;
auto nz_cnt = CalCoorBestBit<T>(quant_data, elem_cnt, quant_params, unique_value_bit, &coor_best_bit);
// 1. coor_best_bit 2. nz_cnt 3. quant_data_set size 4. unique_values 5. unique_value indexing 6. nz values coord
auto pack_sparsity_size_in_bit =
1 * k8Bit + 4 * k8Bit + bit_num + bit_num * unique_value_cnt + unique_value_bit * nz_cnt + nz_cnt * coor_best_bit;
size_t pack_sparsity_size_in_byte = ceil(1.0 * pack_sparsity_size_in_bit / k8Bit);
MS_LOG(DEBUG) << "coor_best_bit: " << coor_best_bit << " ori: " << origin_size_in_byte
<< " indexing: " << pack_repetition_size_in_byte << " sparse: " << pack_sparsity_size_in_byte;
auto min_byte_need = std::min({origin_size_in_byte, pack_repetition_size_in_byte, pack_sparsity_size_in_byte});
if (min_byte_need == origin_size_in_byte) {
return false;
} else if (min_byte_need == pack_repetition_size_in_byte) {
MS_LOG(DEBUG) << "from " << origin_size_in_byte << " to " << pack_repetition_size_in_byte;
return IndexingCompress<T>(quant_data_set, unique_value_index_map, unique_value_bit, unique_value_cnt,
pack_repetition_size_in_byte, bit_num, tensor);
} else if (min_byte_need == pack_sparsity_size_in_byte) {
MS_LOG(DEBUG) << "from " << origin_size_in_byte << " to " << pack_sparsity_size_in_byte;
return SparsityCompress<T>(quant_data_set, unique_value_index_map, unique_value_bit, unique_value_cnt,
pack_sparsity_size_in_byte, nz_cnt, coor_best_bit, bit_num, tensor);
} else {
MS_LOG(DEBUG) << "unexpected: " << min_byte_need << " not in {" << origin_size_in_byte << " "
<< pack_repetition_size_in_byte << " " << pack_sparsity_size_in_byte << "}";
}
return false;
}
} // namespace mindspore::lite::quant
#endif // MINDSPORE_LITE_TOOLS_CONVERTER_QUANTIZER_QUANTIZE_UTIL_H_