代码评注赛初阶段成果提交火箭队 #2
|
|
@ -0,0 +1,40 @@
|
|||
/**
|
||||
* Copyright 2019 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_COMMON_SHARD_PYBIND_H_
|
||||
#define MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_COMMON_SHARD_PYBIND_H_
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include "minddata/mindrecord/include/common/shard_utils.h"
|
||||
#include "pybind11/pybind11.h"
|
||||
|
||||
namespace py = pybind11;
|
||||
namespace nlohmann {
|
||||
template <>
|
||||
struct adl_serializer<py::object> {
|
||||
py::object FromJson(const json &j);
|
||||
|
||||
void ToJson(json *j, const py::object &obj);
|
||||
};
|
||||
|
||||
namespace detail {
|
||||
py::object FromJsonImpl(const json &j);
|
||||
|
||||
json ToJsonImpl(const py::handle &obj);
|
||||
} // namespace detail
|
||||
} // namespace nlohmann
|
||||
#endif // MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_COMMON_SHARD_PYBIND_H_
|
||||
|
|
@ -0,0 +1,210 @@
|
|||
/**
|
||||
* Copyright 2019 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_COMMON_SHARD_UTILS_H_
|
||||
#define MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_COMMON_SHARD_UTILS_H_
|
||||
|
||||
#include <libgen.h>
|
||||
#include <limits.h>
|
||||
#include <stdlib.h>
|
||||
#include <sys/stat.h>
|
||||
#if !defined(_WIN32) && !defined(_WIN64) && !defined(__APPLE__)
|
||||
#include <sys/statfs.h>
|
||||
#include <sys/wait.h>
|
||||
#endif
|
||||
#include <unistd.h>
|
||||
#include <cassert>
|
||||
#include <cmath>
|
||||
#include <cstdio>
|
||||
#include <ctime>
|
||||
#include <future>
|
||||
#include <iostream>
|
||||
#include <map>
|
||||
#include <memory>
|
||||
#include <random>
|
||||
#include <set>
|
||||
#include <sstream>
|
||||
#include <string>
|
||||
#include <thread>
|
||||
#include <unordered_map>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
#include "minddata/mindrecord/include/shard_error.h"
|
||||
#include "nlohmann/json.hpp"
|
||||
#include "./sqlite3.h"
|
||||
#include "utils/log_adapter.h"
|
||||
|
||||
/* To be used when dlog is ok #include "./slog.h" */
|
||||
#ifdef DEBUG
|
||||
#define MS_ASSERT(f) assert(f)
|
||||
#else
|
||||
#define MS_ASSERT(f) ((void)0)
|
||||
#endif
|
||||
|
||||
namespace mindspore {
|
||||
namespace mindrecord {
|
||||
using json = nlohmann::json;
|
||||
|
||||
//定义int型常量KInt1，KInt2，KInt3，kUnsignedInt4
|
||||
const int kInt0 = 0;
|
||||
const int kInt1 = 1;
|
||||
const int kInt2 = 2;
|
||||
const int kInt3 = 3;
|
||||
const int kUnsignedInt4 = 4;
|
||||
|
||||
enum LabelCategory { kSchemaLabel, kStatisticsLabel, kIndexLabel };
|
||||
|
||||
const char kVersion[] = "3.0"; //将3.0赋给字符型数组kVersion[]
|
||||
const std::vector<std::string> kSupportedVersion = {"2.0", kVersion}; //将2.0，kVersion赋给vector<std::string> kSupportedVersion
|
||||
|
||||
enum ShardType {
|
||||
kNLP = 0,
|
||||
kCV = 1,
|
||||
};
|
||||
|
||||
enum TaskType {
|
||||
kCommonTask = 0,
|
||||
kPaddedTask = 1,
|
||||
};
|
||||
enum SamplerType { kCustomTopNSampler, kCustomTopPercentSampler, kSubsetRandomSampler, kPKSampler, kSubsetSampler };
|
||||
|
||||
enum ShuffleType { kShuffleCategory, kShuffleSample };
|
||||
|
||||
const double kEpsilon = 1e-7;
|
||||
|
||||
const int kThreadNumber = 14;
|
||||
|
||||
// Shard default parameters Shard默认参数
|
||||
const uint64_t kDefaultHeaderSize = 1 << 24; // 16MB
|
||||
const uint64_t kDefaultPageSize = 1 << 25; // 32MB
|
||||
|
||||
// HeaderSize [16KB, 128MB]
|
||||
const int kMinHeaderSize = 1 << 14; // 16KB
|
||||
const int kMaxHeaderSize = 1 << 27; // 128MB
|
||||
|
||||
// PageSize [32KB, 256MB]
|
||||
const int kMinPageSize = 1 << 15; // 32KB
|
||||
const int kMaxPageSize = 1 << 28; // 256MB
|
||||
|
||||
// used by value length / schema id length / statistic id length ... 由值长度/架构id长度/统计id长度使用。。。
|
||||
const uint64_t kInt64Len = 8;
|
||||
|
||||
// Minimum file size 最小文件大小
|
||||
const uint64_t kMinFileSize = kInt64Len;
|
||||
|
||||
const int kMinShardCount = 1;
|
||||
const int kMaxShardCount = 1000; // write
|
||||
const int kMaxFileCount = 4096; // read
|
||||
|
||||
const int kMinConsumerCount = 1;
|
||||
const int kMaxConsumerCount = 128;
|
||||
|
||||
const int kMaxSchemaCount = 1;
|
||||
const int kMaxThreadCount = 32;
|
||||
const int kMaxFieldCount = 100;
|
||||
|
||||
// Minimum free disk size 最小可用磁盘大小
|
||||
const int kMinFreeDiskSize = 10; // 10M
|
||||
|
||||
// dummy json
|
||||
const json kDummyId = R"({"id": 0})"_json;
|
||||
|
||||
// translate type in schema to type in sqlite3(NULL, INTEGER, REAL, TEXT, BLOB) 将模式中的类型转换为sqlite3中的类型（NULL、INTEGER、REAL、TEXT、BLOB）
|
||||
const std::unordered_map<std::string, std::string> kDbJsonMap = {
|
||||
{"string", "TEXT"}, {"date", "DATE"}, {"date-time", "DATETIME"}, {"null", "NULL"},
|
||||
{"integer", "INTEGER"}, {"boolean", "BOOLEAN"}, {"array", "BLOB"}, {"number", "NUMERIC"},
|
||||
{"int32", "INTEGER"}, {"int64", "INTEGER"}, {"float32", "NUMERIC"}, {"float64", "NUMERIC"},
|
||||
{"bytes", "BLOB"}};
|
||||
|
||||
const char kPoint = '.';
|
||||
|
||||
const char kPathSeparator =
|
||||
#if defined(_WIN32) || defined(_WIN64)
|
||||
'\\';
|
||||
#else
|
||||
'/';
|
||||
#endif
|
||||
|
||||
// field type used by check schema validation 检查架构验证使用的字段类型
|
||||
const std::set<std::string> kFieldTypeSet = {"bytes", "string", "int32", "int64", "float32", "float64"};
|
||||
|
||||
// can be searched field list 可搜索字段列表
|
||||
const std::set<std::string> kScalarFieldTypeSet = {"string", "int32", "int64", "float32", "float64"};
|
||||
|
||||
// number field list 数字字段列表
|
||||
const std::set<std::string> kNumberFieldTypeSet = {"int32", "int64", "float32", "float64"};
|
||||
|
||||
const std::unordered_map<std::string, std::string> kTypesMap = {
|
||||
{"bool", "int32"}, {"int8", "int32"}, {"uint8", "bytes"}, {"int16", "int32"},
|
||||
{"uint16", "int32"}, {"int32", "int32"}, {"uint32", "int64"}, {"int64", "int64"},
|
||||
{"float16", "float32"}, {"float32", "float32"}, {"float64", "float64"}, {"string", "string"}};
|
||||
|
||||
/// \brief the max number of samples to enable lazy load 启用延迟加载的最大样本数
|
||||
const uint32_t LAZY_LOAD_THRESHOLD = 5000000;
|
||||
|
||||
/// \brief split a string using a character 使用字符拆分字符串
|
||||
/// \param[in] field target string 目标字符串
|
||||
/// \param[in] separator a character for splitting 用于拆分的字符
|
||||
/// \return vector type result 矢量类型结果
|
||||
std::vector<std::string> StringSplit(const std::string &field, char separator);
|
||||
|
||||
/// \brief validate field name is composed of '0-9' or 'a-z' or 'A-Z' or '_' or '-' 验证字段名由“0-9”或“a-z”或“a-z”或“_”或“-”组成
|
||||
/// \param[in] str target string 目标字符串
|
||||
/// \return
|
||||
bool ValidateFieldName(const std::string &str);
|
||||
|
||||
/// \brief get the filename by the path 通过路径获取文件名
|
||||
/// \param s file path 文件路径
|
||||
/// \param fn_ptr shared ptr of file name 文件名的共享指针
|
||||
/// \return Status
|
||||
Status GetFileName(const std::string &path, std::shared_ptr<std::string> *fn_ptr);
|
||||
|
||||
/// \brief get parent dir 获取父目录
|
||||
/// \param path file path 文件路径
|
||||
/// \param pd_ptr shared ptr of parent path 父路径的共享指针
|
||||
/// \return Status
|
||||
Status GetParentDir(const std::string &path, std::shared_ptr<std::string> *pd_ptr);
|
||||
|
||||
bool CheckIsValidUtf8(const std::string &str);
|
||||
|
||||
/// \brief judge if a path is legal file 判断路径是否为合法文件
|
||||
/// \param path file path 文件路径
|
||||
/// \return Whether the path is legal or not 路径是否合法
|
||||
bool IsLegalFile(const std::string &path);
|
||||
|
||||
enum DiskSizeType { kTotalSize = 0, kFreeSize };
|
||||
|
||||
/// \brief get the free space about the disk 获取磁盘的可用空间
|
||||
/// \param str_dir file path 文件路径
|
||||
/// \param disk_type: kTotalSize / kFreeSize
|
||||
/// \param size: shared ptr of size in Megabytes 以MB为单位的共享指针
|
||||
/// \return Status
|
||||
Status GetDiskSize(const std::string &str_dir, const DiskSizeType &disk_type, std::shared_ptr<uint64_t> *size);
|
||||
|
||||
/// \brief get the max hardware concurrency //获取最大硬件并发
|
||||
/// \return max concurrency 最大并发数
|
||||
uint32_t GetMaxThreadNum();
|
||||
|
||||
/// \brief get absolute path of all mindrecord files 获取所有mindrecord文件的绝对路径
|
||||
/// \param path path to one fo mindrecord files 一个fo-mindrecord文件的路径
|
||||
/// \param addresses relative path of all mindrecord files 所有思维记录文件的相对路径
|
||||
/// \param ds shared ptr of vector of absolute path 绝对路径矢量的共享指针
|
||||
/// \return Status
|
||||
Status GetDatasetFiles(const std::string &path, const json &addresses, std::shared_ptr<std::vector<std::string>> *ds);
|
||||
} // namespace mindrecord
|
||||
} // namespace mindspore
|
||||
|
||||
#endif // MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_COMMON_SHARD_UTILS_H_
|
||||
|
|
@ -0,0 +1,63 @@
|
|||
/**
|
||||
* Copyright 2019-2021 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_CATEGORY_H_
|
||||
#define MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_CATEGORY_H_
|
||||
|
||||
#include <algorithm>
|
||||
#include <limits>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
#include "minddata/mindrecord/include/shard_operator.h"
|
||||
|
||||
namespace mindspore {
|
||||
namespace mindrecord {
|
||||
class __attribute__((visibility("default"))) ShardCategory : public ShardOperator {
|
||||
public:
|
||||
explicit ShardCategory(const std::vector<std::pair<std::string, std::string>> &categories,
|
||||
int64_t num_elements = std::numeric_limits<int64_t>::max(), bool replacement = false); //定义64位int型num_elements，定义布尔型replacement为false
|
||||
|
||||
ShardCategory(const std::string &category_field, int64_t num_elements,
|
||||
int64_t num_categories = std::numeric_limits<int64_t>::max(), bool replacement = false);
|
||||
|
||||
~ShardCategory() override{};
|
||||
|
||||
const std::vector<std::pair<std::string, std::string>> &GetCategories() const { return categories_; } //返回categories的函数
|
||||
|
||||
const std::string GetCategoryField() const { return category_field_; } //返回category_field_的函数
|
||||
|
||||
int64_t GetNumElements() const { return num_elements_; } //返回num_elements_的函数
|
||||
|
||||
int64_t GetNumCategories() const { return num_categories_; } //返回num_categories_的函数
|
||||
|
||||
bool GetReplacement() const { return replacement_; } //返回replacement_的函数
|
||||
|
||||
Status Execute(ShardTaskList &tasks) override;
|
||||
|
||||
int64_t GetNumSamples(int64_t dataset_size, int64_t num_classes) override;
|
||||
|
||||
private: //私有
|
||||
std::vector<std::pair<std::string, std::string>> categories_;
|
||||
std::string category_field_;
|
||||
int64_t num_elements_;
|
||||
int64_t num_categories_;
|
||||
bool replacement_;
|
||||
};
|
||||
} // namespace mindrecord
|
||||
} // namespace mindspore
|
||||
|
||||
#endif // MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_CATEGORY_H_
|
||||
|
|
@ -0,0 +1,175 @@
|
|||
/**
|
||||
* Copyright 2020 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_COLUMN_H_
|
||||
#define MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_COLUMN_H_
|
||||
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <unordered_map>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
#include "minddata/mindrecord/include/shard_header.h"
|
||||
|
||||
namespace mindspore {
|
||||
namespace mindrecord {
|
||||
const uint64_t kUnsignedOne = 1;
|
||||
const uint64_t kBitsOfByte = 8;
|
||||
const uint64_t kDataTypeBits = 2;
|
||||
const uint64_t kNumDataOfByte = 4;
|
||||
const uint64_t kBytesOfColumnLen = 4;
|
||||
const uint64_t kDataTypeBitMask = 3;
|
||||
const uint64_t kDataTypes = 6;
|
||||
|
||||
enum IntegerType { kInt8Type = 0, kInt16Type, kInt32Type, kInt64Type };
|
||||
|
||||
enum ColumnCategory { ColumnInRaw, ColumnInBlob, ColumnNotFound };
|
||||
|
||||
enum ColumnDataType {
|
||||
ColumnBytes = 0,
|
||||
ColumnString = 1,
|
||||
ColumnInt32 = 2,
|
||||
ColumnInt64 = 3,
|
||||
ColumnFloat32 = 4,
|
||||
ColumnFloat64 = 5,
|
||||
ColumnNoDataType = 6
|
||||
};
|
||||
|
||||
const uint32_t ColumnDataTypeSize[kDataTypes] = {1, 1, 4, 8, 4, 8};
|
||||
|
||||
const std::vector<std::string> ColumnDataTypeNameNormalized = {"uint8", "string", "int32",
|
||||
"int64", "float32", "float64"};
|
||||
|
||||
const std::unordered_map<std::string, ColumnDataType> ColumnDataTypeMap = {
|
||||
{"bytes", ColumnBytes}, {"string", ColumnString}, {"int32", ColumnInt32},
|
||||
{"int64", ColumnInt64}, {"float32", ColumnFloat32}, {"float64", ColumnFloat64}};
|
||||
|
||||
class __attribute__((visibility("default"))) ShardColumn {
|
||||
public:
|
||||
explicit ShardColumn(const std::shared_ptr<ShardHeader> &shard_header, bool compress_integer = true);
|
||||
explicit ShardColumn(const json &schema_json, bool compress_integer = true);
|
||||
|
||||
~ShardColumn() = default;
|
||||
|
||||
/// \brief get column value by column name
|
||||
Status GetColumnValueByName(const std::string &column_name, const std::vector<uint8_t> &columns_blob,
|
||||
const json &columns_json, const unsigned char **data,
|
||||
std::unique_ptr<unsigned char[]> *data_ptr, uint64_t *const n_bytes,
|
||||
ColumnDataType *column_data_type, uint64_t *column_data_type_size,
|
||||
std::vector<int64_t> *column_shape);
|
||||
|
||||
/// \brief compress blob
|
||||
std::vector<uint8_t> CompressBlob(const std::vector<uint8_t> &blob, int64_t *compression_size);
|
||||
|
||||
/// \brief check if blob compressed
|
||||
bool CheckCompressBlob() const { return has_compress_blob_; }
|
||||
|
||||
/// \brief getter
|
||||
uint64_t GetNumBlobColumn() const { return num_blob_column_; }
|
||||
|
||||
/// \brief getter
|
||||
std::vector<std::string> GetColumnName() { return column_name_; }
|
||||
|
||||
/// \brief getter
|
||||
std::vector<ColumnDataType> GeColumnDataType() { return column_data_type_; }
|
||||
|
||||
/// \brief getter
|
||||
std::vector<std::vector<int64_t>> GetColumnShape() { return column_shape_; }
|
||||
|
||||
/// \brief get column value from blob
|
||||
Status GetColumnFromBlob(const std::string &column_name, const std::vector<uint8_t> &columns_blob,
|
||||
const unsigned char **data, std::unique_ptr<unsigned char[]> *data_ptr,
|
||||
uint64_t *const n_bytes);
|
||||
|
||||
/// \brief get column type
|
||||
Status GetColumnTypeByName(const std::string &column_name, ColumnDataType *column_data_type,
|
||||
uint64_t *column_data_type_size, std::vector<int64_t> *column_shape,
|
||||
ColumnCategory *column_category);
|
||||
|
||||
/// \brief get column value from json
|
||||
Status GetColumnFromJson(const std::string &column_name, const json &columns_json,
|
||||
std::unique_ptr<unsigned char[]> *data_ptr, uint64_t *n_bytes);
|
||||
|
||||
private:
|
||||
/// \brief initialization
|
||||
void Init(const json &schema_json, bool compress_integer = true);
|
||||
|
||||
/// \brief get float value from json
|
||||
template <typename T>
|
||||
Status GetFloat(std::unique_ptr<unsigned char[]> *data_ptr, const json &json_column_value, bool use_double);
|
||||
|
||||
/// \brief get integer value from json
|
||||
template <typename T>
|
||||
Status GetInt(std::unique_ptr<unsigned char[]> *data_ptr, const json &json_column_value);
|
||||
|
||||
/// \brief get column offset address and size from blob
|
||||
Status GetColumnAddressInBlock(const uint64_t &column_id, const std::vector<uint8_t> &columns_blob,
|
||||
uint64_t *num_bytes, uint64_t *shift_idx);
|
||||
|
||||
/// \brief check if column name is available
|
||||
ColumnCategory CheckColumnName(const std::string &column_name);
|
||||
|
||||
/// \brief compress integer column
|
||||
static vector<uint8_t> CompressInt(const vector<uint8_t> &src_bytes, const IntegerType &int_type);
|
||||
|
||||
/// \brief uncompress integer array column
|
||||
template <typename T>
|
||||
static Status UncompressInt(const uint64_t &column_id, std::unique_ptr<unsigned char[]> *const data_ptr,
|
||||
const std::vector<uint8_t> &columns_blob, uint64_t *num_bytes, uint64_t shift_idx);
|
||||
|
||||
/// \brief convert big-endian bytes to unsigned int
|
||||
/// \param bytes_array bytes array
|
||||
/// \param pos shift address in bytes array
|
||||
/// \param i_type integer type
|
||||
/// \return unsigned int
|
||||
static uint64_t BytesBigToUInt64(const std::vector<uint8_t> &bytes_array, const uint64_t &pos,
|
||||
const IntegerType &i_type);
|
||||
|
||||
/// \brief convert unsigned int to big-endian bytes
|
||||
/// \param value integer value
|
||||
/// \param i_type integer type
|
||||
/// \return bytes
|
||||
static std::vector<uint8_t> UIntToBytesBig(uint64_t value, const IntegerType &i_type);
|
||||
|
||||
/// \brief convert unsigned int to little-endian bytes
|
||||
/// \param value integer value
|
||||
/// \param i_type integer type
|
||||
/// \return bytes
|
||||
static std::vector<uint8_t> UIntToBytesLittle(uint64_t value, const IntegerType &i_type);
|
||||
|
||||
/// \brief convert unsigned int to little-endian bytes
|
||||
/// \param bytes_array bytes array
|
||||
/// \param pos shift address in bytes array
|
||||
/// \param src_i_type source integer typ0e
|
||||
/// \param dst_i_type (output), destination integer type
|
||||
/// \return integer
|
||||
static int64_t BytesLittleToMinIntType(const std::vector<uint8_t> &bytes_array, const uint64_t &pos,
|
||||
const IntegerType &src_i_type, IntegerType *dst_i_type = nullptr);
|
||||
|
||||
private:
|
||||
std::vector<std::string> column_name_; // column name list
|
||||
std::vector<ColumnDataType> column_data_type_; // column data type list
|
||||
std::vector<std::vector<int64_t>> column_shape_; // column shape list
|
||||
std::unordered_map<string, uint64_t> column_name_id_; // column name id map
|
||||
std::vector<std::string> blob_column_; // blob column list
|
||||
std::unordered_map<std::string, uint64_t> blob_column_id_; // blob column name id map
|
||||
bool has_compress_blob_; // if has compress blob
|
||||
uint64_t num_blob_column_; // number of blob columns
|
||||
};
|
||||
} // namespace mindrecord
|
||||
} // namespace mindspore
|
||||
|
||||
#endif // MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_COLUMN_H_
|
||||
|
|
@ -0,0 +1,55 @@
|
|||
/**
|
||||
* Copyright 2020-2021 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_DISTRIBUTED_SAMPLE_H_
|
||||
#define MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_DISTRIBUTED_SAMPLE_H_
|
||||
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
#include "minddata/mindrecord/include/shard_operator.h"
|
||||
#include "minddata/mindrecord/include/shard_shuffle.h"
|
||||
#include "minddata/mindrecord/include/shard_sample.h"
|
||||
|
||||
namespace mindspore {
|
||||
namespace mindrecord {
|
||||
class __attribute__((visibility("default"))) ShardDistributedSample : public ShardSample {
|
||||
public:
|
||||
ShardDistributedSample(int num_shards, int shard_id, int64_t no_of_padded_samples, bool shuffle, uint32_t seed,
|
||||
int64_t no_of_samples = 0, int64_t offset = -1);
|
||||
|
||||
ShardDistributedSample(int num_shards, int shard_id, bool shuffle, uint32_t seed, int64_t no_of_samples = 0,
|
||||
int64_t offset = -1);
|
||||
|
||||
void SetNumPaddedSamples(int64_t no_of_padded_samples) { no_of_padded_samples_ = no_of_padded_samples; }
|
||||
|
||||
~ShardDistributedSample() override{};
|
||||
|
||||
Status PreExecute(ShardTaskList &tasks) override;
|
||||
|
||||
int64_t GetNumSamples(int64_t dataset_size, int64_t num_classes) override;
|
||||
|
||||
private:
|
||||
bool shuffle_;
|
||||
int64_t no_of_padded_samples_;
|
||||
bool first_epoch_; // check (num_sample + num_padded) % num_shards == 0 in first epoch
|
||||
ShardTaskList task_; // maintain the input tasks in first epoch
|
||||
};
|
||||
} // namespace mindrecord
|
||||
} // namespace mindspore
|
||||
|
||||
#endif // MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_DISTRIBUTED_SAMPLE_H_
|
||||
|
|
@ -0,0 +1,74 @@
|
|||
/**
|
||||
* Copyright 2019 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_ERROR_H_
|
||||
#define MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_ERROR_H_
|
||||
|
||||
#include <map>
|
||||
#include <string>
|
||||
#include "include/api/status.h"
|
||||
|
||||
namespace mindspore { //命名mindspore空间
|
||||
namespace mindrecord { //命名mindrecord空间
|
||||
#define RETURN_IF_NOT_OK(_s) \ //参数_s
|
||||
do { \
|
||||
Status __rc = (_s); \ //将_s赋给_rc
|
||||
if (__rc.IsError()) { \ //如果_rc.IsError()返回值为1，则返回_rc
|
||||
return __rc; \
|
||||
} \
|
||||
} while (false) //停止循环
|
||||
|
||||
#define RELEASE_AND_RETURN_IF_NOT_OK(_s, _db, _in) \ //参数_s,_db,_in
|
||||
do { \
|
||||
Status __rc = (_s); \ //将_s赋给_rc
|
||||
if (__rc.IsError()) { \ //当_rc.IsError()返回值为1时，如果_db不为空，关闭_db数据库链接
|
||||
if ((_db) != nullptr) { \
|
||||
sqlite3_close(_db); \
|
||||
} \
|
||||
(_in).close(); \ //关闭_in
|
||||
return __rc; \ //返回_rc
|
||||
} \
|
||||
} while (false) //停止循环
|
||||
|
||||
#define CHECK_FAIL_RETURN_UNEXPECTED(_condition, _e) \ //参数_condition,_e
|
||||
do { \
|
||||
if (!(_condition)) { \ //如果_condition为0，返回StatusCOde下的kMDUnexpectedError, __LINE__, __FILE__, _e
|
||||
return Status(StatusCode::kMDUnexpectedError, __LINE__, __FILE__, _e); \
|
||||
} \
|
||||
} while (false) //停止循环
|
||||
|
||||
#define RETURN_UNEXPECTED_IF_NULL(_ptr) \ //参数_ptr
|
||||
do { \
|
||||
if ((_ptr) == nullptr) { \ //如果_ptr为空指针，则err_msg为The pointer[" + std::string(#_ptr) + "] is null
|
||||
std::string err_msg = "The pointer[" + std::string(#_ptr) + "] is null."; \
|
||||
RETURN_STATUS_UNEXPECTED(err_msg); \ //将err_msg传入RETURN_STATUS_UNEXPECTED()函数，返回StatusCOde下的kMDUnexpectedError, __LINE__, __FILE__,err_msg
|
||||
} \
|
||||
} while (false) //停止循环
|
||||
|
||||
#define RETURN_STATUS_UNEXPECTED(_e) \ //参数_e
|
||||
do { \
|
||||
return Status(StatusCode::kMDUnexpectedError, __LINE__, __FILE__, _e); \ //返回StatusCOde下的kMDUnexpectedError, __LINE__, __FILE__, _e
|
||||
} while (false) //停止循环
|
||||
|
||||
enum MSRStatus { //失败
|
||||
SUCCESS = 0,
|
||||
FAILED = 1,
|
||||
};
|
||||
|
||||
} // namespace mindrecord
|
||||
} // namespace mindspore
|
||||
|
||||
#endif // MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_ERROR_H_
|
||||
|
|
@ -0,0 +1,200 @@
|
|||
/**
|
||||
* Copyright 2019 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_HEADER_H_
|
||||
#define MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_HEADER_H_
|
||||
|
||||
#include <memory>
|
||||
#include <set>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
#include "minddata/mindrecord/include/common/shard_utils.h"
|
||||
#include "minddata/mindrecord/include/shard_error.h"
|
||||
#include "minddata/mindrecord/include/shard_index.h"
|
||||
#include "minddata/mindrecord/include/shard_page.h"
|
||||
#include "minddata/mindrecord/include/shard_schema.h"
|
||||
#include "minddata/mindrecord/include/shard_statistics.h"
|
||||
|
||||
namespace mindspore {
|
||||
namespace mindrecord {
|
||||
class __attribute__((visibility("default"))) ShardHeader {
|
||||
public:
|
||||
ShardHeader();
|
||||
|
||||
~ShardHeader() = default;
|
||||
|
||||
Status BuildDataset(const std::vector<std::string> &file_paths, bool load_dataset = true);
|
||||
|
||||
static Status BuildSingleHeader(const std::string &file_path, std::shared_ptr<json> *header_ptr);
|
||||
/// \brief add the schema and save it 添加架构并保存
|
||||
/// \param[in] schema the schema needs to be added 需要添加的架构
|
||||
/// \return the last schema's id 返回最后一个架构的 ID
|
||||
int AddSchema(std::shared_ptr<Schema> schema);
|
||||
|
||||
/// \brief add the statistic and save it 添加统计信息并保存
|
||||
/// \param[in] statistic the statistic needs to be added 需要添加的统计数据
|
||||
/// \return the last statistic's id 返回最后一个统计信息的 ID
|
||||
void AddStatistic(std::shared_ptr<Statistics> statistic);
|
||||
|
||||
/// \brief create index and add fields which from schema for each schema 创建索引并添加来自每个架构的架构的字段
|
||||
/// \param[in] fields the index fields needs to be added 需要添加的索引字段
|
||||
/// \return SUCCESS if add successfully, FAILED if not 如果添加成功，返回SUCCESS，否则返回FAILED
|
||||
Status AddIndexFields(std::vector<std::pair<uint64_t, std::string>> fields);
|
||||
|
||||
Status AddIndexFields(const std::vector<std::string> &fields);
|
||||
|
||||
/// \brief get the schema 获取架构
|
||||
/// \return the schema 返回架构
|
||||
std::vector<std::shared_ptr<Schema>> GetSchemas();
|
||||
|
||||
/// \brief get Statistics 获取统计数据
|
||||
/// \return the Statistic 返回统计数据
|
||||
std::vector<std::shared_ptr<Statistics>> GetStatistics();
|
||||
|
||||
/// \brief add the statistic and save it 添加统计信息并保存
|
||||
/// \param[in] statistic info of slim size 苗条尺寸的统计信息
|
||||
/// \return null 返回空
|
||||
int64_t GetSlimSizeStatistic(const json &slim_size_json);
|
||||
|
||||
/// \brief get the fields of the index 获取索引的字段
|
||||
/// \return the fields of the index 返回索引的字段
|
||||
std::vector<std::pair<uint64_t, std::string>> GetFields();
|
||||
|
||||
/// \brief get the index
|
||||
/// \return the index
|
||||
std::shared_ptr<Index> GetIndex();
|
||||
|
||||
/// \brief get the schema by schemaid
|
||||
/// \param[in] schema_id the id of schema needs to be got
|
||||
/// \param[in] schema_ptr the schema obtained by schemaId
|
||||
/// \return Status
|
||||
Status GetSchemaByID(int64_t schema_id, std::shared_ptr<Schema> *schema_ptr);
|
||||
|
||||
/// \brief get the filepath to shard by shardID
|
||||
/// \param[in] shardID the id of shard which filepath needs to be obtained
|
||||
/// \return the filepath obtained by shardID
|
||||
std::string GetShardAddressByID(int64_t shard_id);
|
||||
|
||||
/// \brief get the statistic by statistic id
|
||||
/// \param[in] statistic_id the id of statistic needs to be get
|
||||
/// \param[in] statistics_ptr the statistics obtained by statistic id
|
||||
/// \return Status
|
||||
Status GetStatisticByID(int64_t statistic_id, std::shared_ptr<Statistics> *statistics_ptr);
|
||||
|
||||
Status InitByFiles(const std::vector<std::string> &file_paths);
|
||||
|
||||
void SetIndex(Index index) { index_ = std::make_shared<Index>(index); }
|
||||
|
||||
Status GetPage(const int &shard_id, const int &page_id, std::shared_ptr<Page> *page_ptr);
|
||||
|
||||
Status SetPage(const std::shared_ptr<Page> &new_page);
|
||||
|
||||
Status AddPage(const std::shared_ptr<Page> &new_page);
|
||||
|
||||
int64_t GetLastPageId(const int &shard_id);
|
||||
|
||||
int GetLastPageIdByType(const int &shard_id, const std::string &page_type);
|
||||
|
||||
Status GetPageByGroupId(const int &group_id, const int &shard_id, std::shared_ptr<Page> *page_ptr);
|
||||
|
||||
std::vector<std::string> GetShardAddresses() const { return shard_addresses_; }
|
||||
|
||||
int GetShardCount() const { return shard_count_; }
|
||||
|
||||
int GetSchemaCount() const { return schema_.size(); }
|
||||
|
||||
uint64_t GetHeaderSize() const { return header_size_; }
|
||||
|
||||
uint64_t GetPageSize() const { return page_size_; }
|
||||
|
||||
uint64_t GetCompressionSize() const { return compression_size_; }
|
||||
|
||||
void SetHeaderSize(const uint64_t &header_size) { header_size_ = header_size; }
|
||||
|
||||
void SetPageSize(const uint64_t &page_size) { page_size_ = page_size; }
|
||||
|
||||
void SetCompressionSize(const uint64_t &compression_size) { compression_size_ = compression_size; }
|
||||
|
||||
std::vector<std::string> SerializeHeader();
|
||||
|
||||
Status PagesToFile(const std::string dump_file_name);
|
||||
|
||||
Status FileToPages(const std::string dump_file_name);
|
||||
|
||||
static Status Initialize(const std::shared_ptr<ShardHeader> *header_ptr, const json &schema,
|
||||
const std::vector<std::string> &index_fields, std::vector<std::string> &blob_fields,
|
||||
uint64_t &schema_id);
|
||||
|
||||
private:
|
||||
Status InitializeHeader(const std::vector<json> &headers, bool load_dataset);
|
||||
|
||||
/// \brief get the headers from all the shard data
|
||||
/// \param[in] the shard data real path
|
||||
/// \param[in] the headers which read from the shard data
|
||||
/// \return SUCCESS/FAILED
|
||||
Status GetHeaders(const vector<string> &real_addresses, std::vector<json> &headers);
|
||||
|
||||
Status ValidateField(const std::vector<std::string> &field_name, json schema, const uint64_t &schema_id);
|
||||
|
||||
/// \brief check the binary file status
|
||||
static Status CheckFileStatus(const std::string &path);
|
||||
|
||||
static Status ValidateHeader(const std::string &path, std::shared_ptr<json> *header_ptr);
|
||||
|
||||
void GetHeadersOneTask(int start, int end, std::vector<json> &headers, const vector<string> &realAddresses);
|
||||
|
||||
Status ParseIndexFields(const json &index_fields);
|
||||
|
||||
Status CheckIndexField(const std::string &field, const json &schema);
|
||||
|
||||
Status ParsePage(const json &page, int shard_index, bool load_dataset);
|
||||
|
||||
Status ParseStatistics(const json &statistics);
|
||||
|
||||
Status ParseSchema(const json &schema);
|
||||
|
||||
void ParseShardAddress(const json &address);
|
||||
|
||||
std::string SerializeIndexFields();
|
||||
|
||||
std::vector<std::string> SerializePage();
|
||||
|
||||
std::string SerializeStatistics();
|
||||
|
||||
std::string SerializeSchema();
|
||||
|
||||
std::string SerializeShardAddress();
|
||||
|
||||
std::shared_ptr<Index> InitIndexPtr();
|
||||
|
||||
Status GetAllSchemaID(std::set<uint64_t> &bucket_count);
|
||||
|
||||
uint32_t shard_count_;
|
||||
uint64_t header_size_;
|
||||
uint64_t page_size_;
|
||||
uint64_t compression_size_;
|
||||
|
||||
std::shared_ptr<Index> index_;
|
||||
std::vector<std::string> shard_addresses_;
|
||||
std::vector<std::shared_ptr<Schema>> schema_;
|
||||
std::vector<std::shared_ptr<Statistics>> statistics_;
|
||||
std::vector<std::vector<std::shared_ptr<Page>>> pages_;
|
||||
};
|
||||
} // namespace mindrecord
|
||||
} // namespace mindspore
|
||||
|
||||
#endif // MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_HEADER_H_
|
||||
|
|
@ -0,0 +1,65 @@
|
|||
/**
|
||||
* Copyright 2019 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INDEX_H
|
||||
#define MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INDEX_H
|
||||
#pragma once
|
||||
|
||||
#include <stdio.h>
|
||||
#include <iostream>
|
||||
#include <map>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
#include "minddata/mindrecord/include/common/shard_utils.h"
|
||||
#include "minddata/mindrecord/include/shard_error.h"
|
||||
#include "minddata/mindrecord/include/shard_schema.h"
|
||||
#include "utils/log_adapter.h"
|
||||
|
||||
namespace mindspore {
|
||||
namespace mindrecord {
|
||||
using std::cin;
|
||||
using std::endl;
|
||||
using std::pair;
|
||||
using std::string;
|
||||
using std::vector;
|
||||
|
||||
class __attribute__((visibility("default"))) Index {
|
||||
public:
|
||||
Index();
|
||||
|
||||
~Index() {}
|
||||
|
||||
/// \brief Add field which from schema according to schemaId
|
||||
/// \param[in] schemaId the id of schema to be added
|
||||
/// \param[in] field the field need to be added
|
||||
///
|
||||
/// add the field to the fields_ vector
|
||||
void AddIndexField(const int64_t &schemaId, const std::string &field);
|
||||
|
||||
/// \brief get stored fields
|
||||
/// \return fields stored
|
||||
std::vector<std::pair<uint64_t, std::string> > GetFields();
|
||||
|
||||
private:
|
||||
std::vector<std::pair<uint64_t, std::string> > fields_;
|
||||
string database_name_;
|
||||
string table_name_;
|
||||
};
|
||||
} // namespace mindrecord
|
||||
} // namespace mindspore
|
||||
|
||||
#endif // MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INDEX_H
|
||||
|
|
@ -0,0 +1,122 @@
|
|||
/**
|
||||
* Copyright 2019 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_INDEX_GENERATOR_H_
|
||||
#define MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_INDEX_GENERATOR_H_
|
||||
|
||||
#include <fstream>
|
||||
#include <iostream>
|
||||
#include <map>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <tuple>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
#include "minddata/mindrecord/include/shard_header.h"
|
||||
#include "./sqlite3.h"
|
||||
|
||||
namespace mindspore {
|
||||
namespace mindrecord {
|
||||
using INDEX_FIELDS = std::vector<std::tuple<std::string, std::string, std::string>>;
|
||||
using ROW_DATA = std::vector<std::vector<std::tuple<std::string, std::string, std::string>>>;
|
||||
class __attribute__((visibility("default"))) ShardIndexGenerator {
|
||||
public:
|
||||
explicit ShardIndexGenerator(const std::string &file_path, bool append = false);
|
||||
|
||||
Status Build();
|
||||
|
||||
static Status GenerateFieldName(const std::pair<uint64_t, std::string> &field, std::shared_ptr<std::string> *fn_ptr);
|
||||
|
||||
~ShardIndexGenerator() {}
|
||||
|
||||
/// \brief fetch value in json by field name
|
||||
/// \param[in] field
|
||||
/// \param[in] input
|
||||
/// \param[in] value
|
||||
/// \return Status
|
||||
Status GetValueByField(const string &field, const json &input, std::shared_ptr<std::string> *value);
|
||||
|
||||
/// \brief fetch field type in schema n by field path
|
||||
/// \param[in] field_path
|
||||
/// \param[in] schema
|
||||
/// \return the type of field
|
||||
static std::string TakeFieldType(const std::string &field_path, json &schema);
|
||||
|
||||
/// \brief create databases for indexes
|
||||
Status WriteToDatabase();
|
||||
|
||||
static Status Finalize(const std::vector<std::string> file_names);
|
||||
|
||||
private:
|
||||
static int Callback(void *not_used, int argc, char **argv, char **az_col_name);
|
||||
|
||||
static Status ExecuteSQL(const std::string &statement, sqlite3 *db, const string &success_msg = "");
|
||||
|
||||
static std::string ConvertJsonToSQL(const std::string &json);
|
||||
|
||||
Status CreateDatabase(int shard_no, sqlite3 **db);
|
||||
|
||||
Status GetSchemaDetails(const std::vector<uint64_t> &schema_lens, std::fstream &in,
|
||||
std::shared_ptr<std::vector<json>> *detail_ptr);
|
||||
|
||||
static Status GenerateRawSQL(const std::vector<std::pair<uint64_t, std::string>> &fields,
|
||||
std::shared_ptr<std::string> *sql_ptr);
|
||||
|
||||
Status CheckDatabase(const std::string &shard_address, sqlite3 **db);
|
||||
|
||||
///
|
||||
/// \param shard_no
|
||||
/// \param blob_id_to_page_id
|
||||
/// \param raw_page_id
|
||||
/// \param in
|
||||
/// \return Status
|
||||
Status GenerateRowData(int shard_no, const std::map<int, int> &blob_id_to_page_id, int raw_page_id, std::fstream &in,
|
||||
std::shared_ptr<ROW_DATA> *row_data_ptr);
|
||||
///
|
||||
/// \param db
|
||||
/// \param sql
|
||||
/// \param data
|
||||
/// \return
|
||||
Status BindParameterExecuteSQL(sqlite3 *db, const std::string &sql, const ROW_DATA &data);
|
||||
|
||||
Status GenerateIndexFields(const std::vector<json> &schema_detail, std::shared_ptr<INDEX_FIELDS> *index_fields_ptr);
|
||||
|
||||
Status ExecuteTransaction(const int &shard_no, sqlite3 *db, const std::vector<int> &raw_page_ids,
|
||||
const std::map<int, int> &blob_id_to_page_id);
|
||||
|
||||
Status CreateShardNameTable(sqlite3 *db, const std::string &shard_name);
|
||||
|
||||
Status AddBlobPageInfo(std::vector<std::tuple<std::string, std::string, std::string>> &row_data,
|
||||
const std::shared_ptr<Page> cur_blob_page, uint64_t &cur_blob_page_offset, std::fstream &in);
|
||||
|
||||
Status AddIndexFieldByRawData(const std::vector<json> &schema_detail,
|
||||
std::vector<std::tuple<std::string, std::string, std::string>> &row_data);
|
||||
|
||||
void DatabaseWriter(); // worker thread
|
||||
|
||||
std::string file_path_;
|
||||
bool append_;
|
||||
ShardHeader shard_header_;
|
||||
uint64_t page_size_;
|
||||
uint64_t header_size_;
|
||||
int schema_count_;
|
||||
std::atomic_int task_;
|
||||
std::atomic_bool write_success_;
|
||||
std::vector<std::pair<uint64_t, std::string>> fields_;
|
||||
};
|
||||
} // namespace mindrecord
|
||||
} // namespace mindspore
|
||||
#endif // MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_INDEX_GENERATOR_H_
|
||||
|
|
@ -0,0 +1,85 @@
|
|||
/**
|
||||
* Copyright 2019-2021 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
#ifndef MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_OPERATOR_H_
|
||||
#define MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_OPERATOR_H_
|
||||
|
||||
#include <memory>
|
||||
#include <vector>
|
||||
#include "minddata/mindrecord/include/shard_task_list.h"
|
||||
#include "minddata/dataset/include/dataset/constants.h"
|
||||
|
||||
namespace mindspore {
|
||||
namespace mindrecord {
|
||||
class __attribute__((visibility("default"))) ShardOperator {
|
||||
public:
|
||||
virtual ~ShardOperator() = default;
|
||||
|
||||
Status operator()(ShardTaskList &tasks) {
|
||||
RETURN_IF_NOT_OK(this->PreExecute(tasks));
|
||||
RETURN_IF_NOT_OK(this->Execute(tasks));
|
||||
RETURN_IF_NOT_OK(this->SufExecute(tasks));
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
virtual bool HasChildOp() { return child_op_ != nullptr; }
|
||||
|
||||
virtual Status SetChildOp(const std::shared_ptr<ShardOperator> &child_op) {
|
||||
if (child_op != nullptr) {
|
||||
child_op_ = child_op;
|
||||
}
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
virtual std::shared_ptr<ShardOperator> GetChildOp() { return child_op_; }
|
||||
|
||||
virtual Status PreExecute(ShardTaskList &tasks) { return Status::OK(); }
|
||||
|
||||
virtual Status Execute(ShardTaskList &tasks) = 0;
|
||||
|
||||
virtual Status SufExecute(ShardTaskList &tasks) { return Status::OK(); }
|
||||
|
||||
/// \brief compute actual the num_samples via loading data
|
||||
virtual int64_t GetNumSamples(int64_t dataset_size, int64_t num_classes) { return 0; }
|
||||
|
||||
/// \brief Getter the number of samples which is set via python api
|
||||
virtual int64_t GetNumSamples() const { return num_samples_; }
|
||||
|
||||
/// \brief Setter the number of samples in python
|
||||
virtual void SetNumSamples(int64_t num_samples) { num_samples_ = num_samples; }
|
||||
|
||||
virtual void UpdateShuffleMode(dataset::ShuffleMode shuffle_mode) { shuffle_mode_ = shuffle_mode; }
|
||||
|
||||
virtual dataset::ShuffleMode GetShuffleMode() { return shuffle_mode_; }
|
||||
|
||||
virtual void SetShardSampleCount(const std::vector<int64_t> &shard_sample_count) {
|
||||
shard_sample_count_ = shard_sample_count;
|
||||
}
|
||||
|
||||
virtual std::vector<int64_t> GetShardSampleCount() { return shard_sample_count_; }
|
||||
|
||||
private:
|
||||
int64_t num_samples_ = 0;
|
||||
std::shared_ptr<ShardOperator> child_op_ = nullptr;
|
||||
// indicate shard_id : inc_count
|
||||
// 0 : 15 - shard0 has 15 samples
|
||||
// 1 : 41 - shard1 has 26 samples
|
||||
// 2 : 58 - shard2 has 17 samples
|
||||
std::vector<int64_t> shard_sample_count_;
|
||||
dataset::ShuffleMode shuffle_mode_ = dataset::ShuffleMode::kGlobal;
|
||||
};
|
||||
} // namespace mindrecord
|
||||
} // namespace mindspore
|
||||
#endif // MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_OPERATOR_H_
|
||||
|
|
@ -0,0 +1,106 @@
|
|||
/**
|
||||
* Copyright 2019 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_PAGE_H_
|
||||
#define MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_PAGE_H_
|
||||
|
||||
#include <fstream>
|
||||
#include <iostream>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
#include "minddata/mindrecord/include/common/shard_utils.h"
|
||||
#include "pybind11/pybind11.h"
|
||||
#include "utils/log_adapter.h"
|
||||
|
||||
namespace mindspore {
|
||||
namespace mindrecord {
|
||||
const std::string kPageTypeRaw = "RAW_DATA";
|
||||
const std::string kPageTypeBlob = "BLOB_DATA";
|
||||
const std::string kPageTypeNewColumn = "NEW_COLUMN_DATA";
|
||||
|
||||
class __attribute__((visibility("default"))) Page {
|
||||
public:
|
||||
Page(const int &page_id, const int &shard_id, const std::string &page_type, const int &page_type_id,
|
||||
const uint64_t &start_row_id, const uint64_t end_row_id,
|
||||
const std::vector<std::pair<int, uint64_t>> &row_group_ids, const uint64_t page_size)
|
||||
: page_id_(page_id),
|
||||
shard_id_(shard_id),
|
||||
page_type_(page_type),
|
||||
page_type_id_(page_type_id),
|
||||
start_row_id_(start_row_id),
|
||||
end_row_id_(end_row_id),
|
||||
row_group_ids_(row_group_ids),
|
||||
page_size_(page_size) {}
|
||||
|
||||
~Page() = default;
|
||||
|
||||
/// \brief get the page and its description
|
||||
/// \return the json format of the page and its description
|
||||
json GetPage() const;
|
||||
|
||||
int GetPageID() const { return page_id_; }
|
||||
|
||||
int GetShardID() const { return shard_id_; }
|
||||
|
||||
int GetPageTypeID() const { return page_type_id_; }
|
||||
|
||||
std::string GetPageType() const { return page_type_; }
|
||||
|
||||
uint64_t GetPageSize() const { return page_size_; }
|
||||
|
||||
uint64_t GetStartRowID() const { return start_row_id_; }
|
||||
|
||||
uint64_t GetEndRowID() const { return end_row_id_; }
|
||||
|
||||
void SetEndRowID(const uint64_t &end_row_id) { end_row_id_ = end_row_id; }
|
||||
|
||||
void SetPageSize(const uint64_t &page_size) { page_size_ = page_size; }
|
||||
|
||||
std::pair<int, uint64_t> GetLastRowGroupID() const { return row_group_ids_.back(); }
|
||||
|
||||
std::vector<std::pair<int, uint64_t>> GetRowGroupIds() const { return row_group_ids_; }
|
||||
|
||||
void SetRowGroupIds(const std::vector<std::pair<int, uint64_t>> &last_row_group_ids) {
|
||||
row_group_ids_ = last_row_group_ids;
|
||||
}
|
||||
|
||||
void DeleteLastGroupId();
|
||||
|
||||
private:
|
||||
int page_id_;
|
||||
int shard_id_;
|
||||
std::string page_type_;
|
||||
int page_type_id_;
|
||||
uint64_t start_row_id_;
|
||||
uint64_t end_row_id_;
|
||||
std::vector<std::pair<int, uint64_t>> row_group_ids_;
|
||||
uint64_t page_size_;
|
||||
// JSON page: {
|
||||
// "page_id":X,
|
||||
// "shard_id":X,
|
||||
// "page_type":"XXX", (enum "raw_data", "blob_data", "new_column")
|
||||
// "page_type_id":X,
|
||||
// "start_row_id":X,
|
||||
// "end_row_id":X,
|
||||
// "row_group_ids":[{"id":X, "offset":X}],
|
||||
// "page_size":X,
|
||||
};
|
||||
} // namespace mindrecord
|
||||
} // namespace mindspore
|
||||
|
||||
#endif // MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_PAGE_H_
|
||||
|
|
@ -0,0 +1,53 @@
|
|||
/**
|
||||
* Copyright 2019-2021 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_PK_SAMPLE_H_
|
||||
#define MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_PK_SAMPLE_H_
|
||||
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
#include "minddata/mindrecord/include/shard_operator.h"
|
||||
#include "minddata/mindrecord/include/shard_shuffle.h"
|
||||
#include "minddata/mindrecord/include/shard_category.h"
|
||||
|
||||
namespace mindspore {
|
||||
namespace mindrecord {
|
||||
class __attribute__((visibility("default"))) ShardPkSample : public ShardCategory {
|
||||
public:
|
||||
ShardPkSample(const std::string &category_field, int64_t num_elements, int64_t num_samples);
|
||||
|
||||
ShardPkSample(const std::string &category_field, int64_t num_elements, int64_t num_categories, int64_t num_samples);
|
||||
|
||||
ShardPkSample(const std::string &category_field, int64_t num_elements, int64_t num_categories, uint32_t seed,
|
||||
int64_t num_samples);
|
||||
|
||||
~ShardPkSample() override{};
|
||||
|
||||
Status SufExecute(ShardTaskList &tasks) override;
|
||||
|
||||
int64_t GetNumSamples() const { return num_samples_; }
|
||||
|
||||
private:
|
||||
bool shuffle_;
|
||||
std::shared_ptr<ShardShuffle> shuffle_op_;
|
||||
int64_t num_samples_;
|
||||
};
|
||||
} // namespace mindrecord
|
||||
} // namespace mindspore
|
||||
|
||||
#endif // MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_PK_SAMPLE_H_
|
||||
|
|
@ -0,0 +1,361 @@
|
|||
/**
|
||||
* Copyright 2019-2022 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_READER_H_
|
||||
#define MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_READER_H_
|
||||
|
||||
#include <dirent.h>
|
||||
#include <signal.h>
|
||||
#if !defined(_WIN32) && !defined(_WIN64) && !defined(__APPLE__)
|
||||
#include <sys/prctl.h>
|
||||
#endif
|
||||
#include <sys/stat.h>
|
||||
#include <sys/types.h>
|
||||
#include <unistd.h>
|
||||
#include <algorithm>
|
||||
#include <chrono>
|
||||
#include <cstdint>
|
||||
#include <fstream>
|
||||
#include <iostream>
|
||||
#include <map>
|
||||
#include <memory>
|
||||
#include <mutex>
|
||||
#include <set>
|
||||
#include <stack>
|
||||
#include <string>
|
||||
#include <thread>
|
||||
#include <tuple>
|
||||
#include <unordered_map>
|
||||
#include <unordered_set>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
#include "minddata/mindrecord/include/common/shard_utils.h"
|
||||
#include "minddata/mindrecord/include/shard_category.h"
|
||||
#include "minddata/mindrecord/include/shard_column.h"
|
||||
#include "minddata/mindrecord/include/shard_distributed_sample.h"
|
||||
#include "minddata/mindrecord/include/shard_error.h"
|
||||
#include "minddata/mindrecord/include/shard_index_generator.h"
|
||||
#include "minddata/mindrecord/include/shard_operator.h"
|
||||
#include "minddata/mindrecord/include/shard_pk_sample.h"
|
||||
#include "minddata/mindrecord/include/shard_reader.h"
|
||||
#include "minddata/mindrecord/include/shard_sample.h"
|
||||
#include "minddata/mindrecord/include/shard_shuffle.h"
|
||||
#include "utils/log_adapter.h"
|
||||
|
||||
#define API_PUBLIC __attribute__((visibility("default")))
|
||||
|
||||
namespace mindspore {
|
||||
namespace mindrecord {
|
||||
using ROW_GROUPS = std::pair<std::vector<std::vector<std::vector<uint64_t>>>, std::vector<std::vector<json>>>;
|
||||
using ROW_GROUP_BRIEF = std::tuple<std::string, int, uint64_t, std::vector<std::vector<uint64_t>>, std::vector<json>>;
|
||||
using TASK_CONTENT = std::pair<TaskType, std::vector<std::tuple<std::vector<uint8_t>, json>>>;
|
||||
const int kNumBatchInMap = 1000; // iterator buffer size in row-reader mode
|
||||
|
||||
class API_PUBLIC ShardReader {
|
||||
public:
|
||||
ShardReader();
|
||||
|
||||
virtual ~ShardReader();
|
||||
|
||||
/// \brief open files and initialize reader, c++ API
|
||||
/// \param[in] file_paths the path of ONE file, any file in dataset is fine or file list
|
||||
/// \param[in] load_dataset load dataset from single file or not
|
||||
/// \param[in] n_consumer number of threads when reading
|
||||
/// \param[in] selected_columns column list to be populated
|
||||
/// \param[in] operators operators applied to data, operator type is shuffle, sample or category
|
||||
/// \param[in] num_padded the number of padded samples
|
||||
/// \param[in] lazy_load if the mindrecord dataset is too large, enable lazy load mode to speed up initialization
|
||||
/// \return MSRStatus the status of MSRStatus
|
||||
Status Open(const std::vector<std::string> &file_paths, bool load_dataset, int n_consumer = 4,
|
||||
const std::vector<std::string> &selected_columns = {},
|
||||
const std::vector<std::shared_ptr<ShardOperator>> &operators = {}, const int64_t num_padded = 0,
|
||||
bool lazy_load = false);
|
||||
|
||||
/// \brief close reader
|
||||
/// \return null
|
||||
void Close();
|
||||
|
||||
/// \brief read the file, get schema meta,statistics and index, single-thread mode
|
||||
/// \return MSRStatus the status of MSRStatus
|
||||
Status Open();
|
||||
|
||||
/// \brief read the file, get schema meta,statistics and index, multiple-thread mode
|
||||
/// \return MSRStatus the status of MSRStatus
|
||||
Status Open(int n_consumer);
|
||||
|
||||
/// \brief increase number of random file stream for parallel read
|
||||
/// \param[in] n_new_consumers number of new file streams to be added
|
||||
/// \return MSRStatus the status of MSRStatus
|
||||
Status ExtendRandomFileStreams(const int n_new_consumers);
|
||||
|
||||
/// \brief decrease number of random file streams for parallel read
|
||||
/// \param[in] n_remove_consumers number of file streams to be removed
|
||||
/// \return MSRStatus the status of MSRStatus
|
||||
Status ShrinkRandomFileStreams(const int n_remove_consumers);
|
||||
|
||||
/// \brief launch threads to get batches
|
||||
/// \param[in] is_simple_reader trigger threads if false; do nothing if true
|
||||
/// \return MSRStatus the status of MSRStatus
|
||||
Status Launch(bool is_simple_reader = false);
|
||||
|
||||
/// \brief aim to get the meta data
|
||||
/// \return the metadata
|
||||
std::shared_ptr<ShardHeader> GetShardHeader() const;
|
||||
|
||||
/// \brief aim to get columns context
|
||||
/// \return the columns
|
||||
std::shared_ptr<ShardColumn> GetShardColumn() const;
|
||||
|
||||
/// \brief get the number of shards
|
||||
/// \return # of shards
|
||||
int GetShardCount() const;
|
||||
|
||||
/// \brief get the number of rows in database
|
||||
/// \param[in] file_paths the path of ONE file, any file in dataset is fine or file list
|
||||
/// \param[in] load_dataset load dataset from single file or not
|
||||
/// \param[in] op smart pointer refer to ShardCategory or ShardSample object
|
||||
/// \param[out] count # of rows
|
||||
/// \return MSRStatus the status of MSRStatus
|
||||
Status CountTotalRows(const std::vector<std::string> &file_paths, bool load_dataset,
|
||||
const std::shared_ptr<ShardOperator> &op, int64_t *count, const int64_t num_padded);
|
||||
|
||||
/// \brief shuffle task with incremental seed
|
||||
/// \return void
|
||||
void ShuffleTask();
|
||||
|
||||
/// \brief get the number of rows in database
|
||||
/// \return # of rows
|
||||
int64_t GetNumRows() const;
|
||||
|
||||
/// \brief Read the summary of row groups
|
||||
/// \return the tuple of 4 elements
|
||||
/// 1. Sharding ID
|
||||
/// 2. Row group ID
|
||||
/// 3. The row ID started in row group
|
||||
/// 4. # of rows in row group
|
||||
std::vector<std::tuple<int, int, int, uint64_t>> ReadRowGroupSummary();
|
||||
|
||||
/// \brief Read 1 row group data, excluding images
|
||||
/// \param[in] groupID row group ID
|
||||
/// \param[in] shard_id sharding ID
|
||||
/// \param[in] columns multi-columns retrieved
|
||||
/// \return the tuple of 5 elements
|
||||
/// 1. file name where row group is located
|
||||
/// 2. Actual row group size
|
||||
/// 3. Offset address of row group in file
|
||||
/// 4. The list of image offset in page [startOffset, endOffset)
|
||||
/// 5. The list of columns data
|
||||
Status ReadRowGroupBrief(int group_id, int shard_id, const std::vector<std::string> &columns,
|
||||
std::shared_ptr<ROW_GROUP_BRIEF> *row_group_brief_ptr);
|
||||
|
||||
/// \brief Read 1 row group data, excluding images, following an index field criteria
|
||||
/// \param[in] groupID row group ID
|
||||
/// \param[in] shard_id sharding ID
|
||||
/// \param[in] column-value pair of criteria to fulfill
|
||||
/// \param[in] columns multi-columns retrieved
|
||||
/// \return the tuple of 5 elements
|
||||
/// 1. file name where row group is located
|
||||
/// 2. Actual row group size
|
||||
/// 3. Offset address of row group in file
|
||||
/// 4. The list of image offset in page [startOffset, endOffset)
|
||||
/// 5. The list of columns data
|
||||
Status ReadRowGroupCriteria(int group_id, int shard_id, const std::pair<std::string, std::string> &criteria,
|
||||
const std::vector<std::string> &columns,
|
||||
std::shared_ptr<ROW_GROUP_BRIEF> *row_group_brief_ptr);
|
||||
|
||||
/// \brief return a batch, given that one is ready
|
||||
/// \return a batch of images and image data
|
||||
std::vector<std::tuple<std::vector<uint8_t>, json>> GetNext();
|
||||
|
||||
/// \brief return a row by id
|
||||
/// \return a batch of images and image data
|
||||
TASK_CONTENT GetNextById(const int64_t &task_id, const int32_t &consumer_id);
|
||||
/// \brief get blob filed list
|
||||
/// \return blob field list
|
||||
std::pair<ShardType, std::vector<std::string>> GetBlobFields();
|
||||
|
||||
/// \brief reset reader
|
||||
/// \return null
|
||||
void Reset();
|
||||
|
||||
/// \brief set flag of all-in-index
|
||||
/// \return null
|
||||
void SetAllInIndex(bool all_in_index) { all_in_index_ = all_in_index; }
|
||||
|
||||
/// \brief get all classes
|
||||
Status GetAllClasses(const std::string &category_field, std::shared_ptr<std::set<std::string>> category_ptr);
|
||||
|
||||
/// \brief get a read-only ptr to the sampled ids for this epoch
|
||||
const std::vector<int64_t> *GetSampleIds();
|
||||
|
||||
/// \brief get the size of blob data
|
||||
Status GetTotalBlobSize(int64_t *total_blob_size);
|
||||
|
||||
/// \brief extract uncompressed data based on column list
|
||||
Status UnCompressBlob(const std::vector<uint8_t> &raw_blob_data,
|
||||
std::shared_ptr<std::vector<std::vector<uint8_t>>> *blob_data_ptr);
|
||||
|
||||
protected:
|
||||
/// \brief sqlite call back function
|
||||
static int SelectCallback(void *p_data, int num_fields, char **p_fields, char **p_col_names);
|
||||
|
||||
private:
|
||||
/// \brief wrap up labels to json format
|
||||
Status ConvertLabelToJson(const std::vector<std::vector<std::string>> &labels, std::shared_ptr<std::fstream> fs,
|
||||
std::shared_ptr<std::vector<std::vector<std::vector<uint64_t>>>> offset_ptr, int shard_id,
|
||||
const std::vector<std::string> &columns,
|
||||
std::shared_ptr<std::vector<std::vector<json>>> col_val_ptr);
|
||||
|
||||
/// \brief convert json format to expected type
|
||||
Status ConvertJsonValue(const std::vector<std::string> &label, const std::vector<std::string> &columns,
|
||||
const json &schema, json *value);
|
||||
|
||||
/// \brief read all rows for specified columns
|
||||
Status ReadAllRowGroup(const std::vector<std::string> &columns, std::shared_ptr<ROW_GROUPS> *row_group_ptr);
|
||||
|
||||
/// \brief read row meta by shard_id and sample_id
|
||||
Status ReadRowGroupByShardIDAndSampleID(const std::vector<std::string> &columns, const uint32_t &shard_id,
|
||||
const uint32_t &sample_id, std::shared_ptr<ROW_GROUPS> *row_group_ptr);
|
||||
|
||||
/// \brief read all rows in one shard
|
||||
Status ReadAllRowsInShard(int shard_id, const std::string &sql, const std::vector<std::string> &columns,
|
||||
std::shared_ptr<std::vector<std::vector<std::vector<uint64_t>>>> offset_ptr,
|
||||
std::shared_ptr<std::vector<std::vector<json>>> col_val_ptr);
|
||||
|
||||
/// \brief initialize reader
|
||||
Status Init(const std::vector<std::string> &file_paths, bool load_dataset);
|
||||
|
||||
/// \brief validate column list
|
||||
Status CheckColumnList(const std::vector<std::string> &selected_columns);
|
||||
|
||||
/// \brief populate one row by task list in row-reader mode
|
||||
void ConsumerByRow(int consumer_id);
|
||||
|
||||
/// \brief get offset address of images within page
|
||||
std::vector<std::vector<uint64_t>> GetImageOffset(int group_id, int shard_id,
|
||||
const std::pair<std::string, std::string> &criteria = {"", ""});
|
||||
|
||||
/// \brief get page id by category
|
||||
Status GetPagesByCategory(int shard_id, const std::pair<std::string, std::string> &criteria,
|
||||
std::shared_ptr<std::vector<uint64_t>> *pages_ptr);
|
||||
/// \brief execute sqlite query with prepare statement
|
||||
Status QueryWithCriteria(sqlite3 *db, const string &sql, const string &criteria,
|
||||
std::shared_ptr<std::vector<std::vector<std::string>>> labels_ptr);
|
||||
/// \brief verify the validity of dataset
|
||||
Status VerifyDataset(sqlite3 **db, const string &file);
|
||||
|
||||
/// \brief get column values
|
||||
Status GetLabels(int page_id, int shard_id, const std::vector<std::string> &columns,
|
||||
const std::pair<std::string, std::string> &criteria, std::shared_ptr<std::vector<json>> *labels_ptr);
|
||||
|
||||
/// \brief get column values from raw data page
|
||||
Status GetLabelsFromPage(int page_id, int shard_id, const std::vector<std::string> &columns,
|
||||
const std::pair<std::string, std::string> &criteria,
|
||||
std::shared_ptr<std::vector<json>> *labels_ptr);
|
||||
|
||||
/// \brief create category-applied task list
|
||||
Status CreateTasksByCategory(const std::shared_ptr<ShardOperator> &op);
|
||||
|
||||
/// \brief create task list in row-reader mode
|
||||
Status CreateTasksByRow(const std::vector<std::tuple<int, int, int, uint64_t>> &row_group_summary,
|
||||
const std::vector<std::shared_ptr<ShardOperator>> &operators);
|
||||
|
||||
/// \brief create task list in row-reader mode and lazy mode
|
||||
Status CreateLazyTasksByRow(const std::vector<std::tuple<int, int, int, uint64_t>> &row_group_summary,
|
||||
const std::vector<std::shared_ptr<ShardOperator>> &operators);
|
||||
|
||||
/// \brief crate task list
|
||||
Status CreateTasks(const std::vector<std::tuple<int, int, int, uint64_t>> &row_group_summary,
|
||||
const std::vector<std::shared_ptr<ShardOperator>> &operators);
|
||||
|
||||
/// \brief check if all specified columns are in index table
|
||||
void CheckIfColumnInIndex(const std::vector<std::string> &columns);
|
||||
|
||||
/// \brief open multiple file handle
|
||||
void FileStreamsOperator();
|
||||
|
||||
/// \brief read one row by one task
|
||||
Status ConsumerOneTask(int64_t task_id, uint32_t consumer_id, std::shared_ptr<TASK_CONTENT> *task_content_pt);
|
||||
|
||||
/// \brief get labels from binary file
|
||||
Status GetLabelsFromBinaryFile(int shard_id, const std::vector<std::string> &columns,
|
||||
const std::vector<std::vector<std::string>> &label_offsets,
|
||||
std::shared_ptr<std::vector<json>> *labels_ptr);
|
||||
|
||||
/// \brief get classes in one shard
|
||||
void GetClassesInShard(sqlite3 *db, int shard_id, const std::string &sql,
|
||||
std::shared_ptr<std::set<std::string>> category_ptr);
|
||||
|
||||
/// \brief get number of classes
|
||||
int64_t GetNumClasses(const std::string &category_field);
|
||||
|
||||
/// \brief get meta of header
|
||||
Status GetMeta(const std::string &file_path, std::shared_ptr<json> meta_data_ptr,
|
||||
std::shared_ptr<std::vector<std::string>> *addresses_ptr);
|
||||
|
||||
protected:
|
||||
uint64_t header_size_; // header size
|
||||
uint64_t page_size_; // page size
|
||||
int shard_count_; // number of shards
|
||||
std::shared_ptr<ShardHeader> shard_header_; // shard header
|
||||
std::shared_ptr<ShardColumn> shard_column_; // shard column
|
||||
|
||||
std::vector<sqlite3 *> database_paths_; // sqlite handle list
|
||||
std::vector<string> file_paths_; // file paths
|
||||
std::vector<std::shared_ptr<std::fstream>> file_streams_; // single-file handle list
|
||||
std::vector<std::vector<std::shared_ptr<std::fstream>>> file_streams_random_; // multiple-file handle list
|
||||
|
||||
private:
|
||||
int n_consumer_; // number of workers (threads)
|
||||
std::vector<std::string> selected_columns_; // columns which will be read
|
||||
std::map<string, uint64_t> column_schema_id_; // column-schema map
|
||||
std::vector<std::shared_ptr<ShardOperator>> operators_; // data operators, including shuffle, sample and category
|
||||
ShardTaskList tasks_; // shard task list
|
||||
std::mutex shard_locker_; // locker of shard
|
||||
|
||||
// flags
|
||||
bool all_in_index_ = true; // if all columns are stored in index-table
|
||||
bool interrupt_ = false; // reader interrupted
|
||||
|
||||
int64_t num_padded_; // number of padding samples
|
||||
|
||||
// Delivery/Iterator mode begin
|
||||
const std::string kThreadName = "THRD_ITER_"; // prefix of thread name
|
||||
std::vector<std::thread> thread_set_; // thread list
|
||||
int64_t num_rows_; // number of rows
|
||||
int64_t total_blob_size_; // total size of blob data
|
||||
std::mutex mtx_delivery_; // locker for delivery
|
||||
std::condition_variable cv_delivery_; // conditional variable for delivery
|
||||
std::condition_variable cv_iterator_; // conditional variable for iterator
|
||||
std::atomic<int> sample_id_position_; // index into the sample ids vector for the current sample id
|
||||
std::atomic<int> deliver_id_; // delivery ID which is picked up by iterator
|
||||
// map of delivery
|
||||
std::unordered_map<int, std::shared_ptr<std::vector<std::tuple<std::vector<uint8_t>, json>>>> delivery_map_;
|
||||
// Delivery/Iterator mode end
|
||||
|
||||
// all metadata in the index is not loaded during initialization
|
||||
bool lazy_load_;
|
||||
|
||||
// indicate shard_id : inc_count
|
||||
// 0 : 15 - shard0 has 15 samples
|
||||
// 1 : 41 - shard1 has 26 samples
|
||||
// 2 : 58 - shard2 has 17 samples
|
||||
std::vector<int64_t> shard_sample_count_;
|
||||
};
|
||||
} // namespace mindrecord
|
||||
} // namespace mindspore
|
||||
|
||||
#endif // MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_READER_H_
|
||||
|
|
@ -0,0 +1,67 @@
|
|||
/**
|
||||
* Copyright 2019-2021 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_SAMPLE_H_
|
||||
#define MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_SAMPLE_H_
|
||||
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
#include "minddata/mindrecord/include/shard_operator.h"
|
||||
#include "minddata/mindrecord/include/shard_shuffle.h"
|
||||
|
||||
namespace mindspore {
|
||||
namespace mindrecord {
|
||||
class __attribute__((visibility("default"))) ShardSample : public ShardOperator {
|
||||
public:
|
||||
explicit ShardSample(int64_t n);
|
||||
|
||||
ShardSample(int64_t num, int64_t den);
|
||||
|
||||
ShardSample(int64_t num, int64_t den, int64_t par, int64_t no_of_samples = 0, int64_t offset = -1);
|
||||
|
||||
ShardSample(const std::vector<int64_t> &indices);
|
||||
|
||||
ShardSample(const std::vector<int64_t> &indices, uint32_t seed);
|
||||
|
||||
~ShardSample() override{};
|
||||
|
||||
Status Execute(ShardTaskList &tasks) override;
|
||||
|
||||
Status UpdateTasks(ShardTaskList &tasks, int64_t taking);
|
||||
|
||||
Status SufExecute(ShardTaskList &tasks) override;
|
||||
|
||||
int64_t GetNumSamples(int64_t dataset_size, int64_t num_classes) override;
|
||||
|
||||
protected:
|
||||
int64_t numerator_;
|
||||
int64_t denominator_;
|
||||
int64_t partition_id_;
|
||||
int64_t no_of_samples_;
|
||||
std::shared_ptr<ShardShuffle> shuffle_op_;
|
||||
std::vector<int64_t> nums_per_shard_;
|
||||
|
||||
private:
|
||||
std::vector<int64_t> indices_;
|
||||
SamplerType sampler_type_;
|
||||
int64_t offset_;
|
||||
};
|
||||
} // namespace mindrecord
|
||||
} // namespace mindspore
|
||||
|
||||
#endif // MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_SAMPLE_H_
|
||||
|
|
@ -0,0 +1,81 @@
|
|||
/**
|
||||
* Copyright 2019 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_SCHEMA_H_
|
||||
#define MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_SCHEMA_H_
|
||||
|
||||
#include <fstream>
|
||||
#include <iostream>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include "minddata/mindrecord/include/common/shard_pybind.h"
|
||||
#include "minddata/mindrecord/include/common/shard_utils.h"
|
||||
#include "minddata/mindrecord/include/shard_error.h"
|
||||
#include "pybind11/pybind11.h"
|
||||
#include "utils/log_adapter.h"
|
||||
|
||||
namespace mindspore {
|
||||
namespace mindrecord {
|
||||
class __attribute__((visibility("default"))) Schema {
|
||||
public:
|
||||
~Schema() = default;
|
||||
|
||||
/// \brief obtain the json schema ,its description, its block fields
|
||||
/// \param[in] desc the description of the schema
|
||||
/// \param[in] schema the schema's json
|
||||
static std::shared_ptr<Schema> Build(std::string desc, const json &schema);
|
||||
|
||||
/// \brief compare two schema to judge if they are equal
|
||||
/// \param b another schema to be judged
|
||||
/// \return true if they are equal,false if not
|
||||
bool operator==(const Schema &b) const;
|
||||
|
||||
/// \brief get the schema and its description
|
||||
/// \return the json format of the schema and its description
|
||||
std::string GetDesc() const;
|
||||
|
||||
/// \brief get the schema and its description
|
||||
/// \return the json format of the schema and its description
|
||||
json GetSchema() const;
|
||||
|
||||
/// set the schema id
|
||||
/// \param[in] id the id need to be set
|
||||
void SetSchemaID(int64_t id);
|
||||
|
||||
/// get the schema id
|
||||
/// \return the int64 schema id
|
||||
int64_t GetSchemaID() const;
|
||||
|
||||
/// get the blob fields
|
||||
/// \return the vector<string> blob fields
|
||||
std::vector<std::string> GetBlobFields() const;
|
||||
|
||||
private:
|
||||
Schema() = default;
|
||||
static bool ValidateNumberShape(const json &it_value);
|
||||
static bool Validate(json schema);
|
||||
static std::vector<std::string> PopulateBlobFields(json schema);
|
||||
|
||||
std::string desc_;
|
||||
json schema_;
|
||||
std::vector<std::string> blob_fields_;
|
||||
int64_t schema_id_ = -1;
|
||||
};
|
||||
} // namespace mindrecord
|
||||
} // namespace mindspore
|
||||
|
||||
#endif // MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_SCHEMA_H_
|
||||
|
|
@ -0,0 +1,101 @@
|
|||
/**
|
||||
* Copyright 2019 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_SEGMENT_H_
|
||||
#define MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_SEGMENT_H_
|
||||
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <tuple>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
#include "minddata/mindrecord/include/shard_reader.h"
|
||||
|
||||
namespace mindspore {
|
||||
namespace mindrecord {
|
||||
using CATEGORY_INFO = std::vector<std::tuple<int, std::string, int>>;
|
||||
using PAGES = std::vector<std::tuple<std::vector<uint8_t>, json>>;
|
||||
using PAGES_LOAD = std::vector<std::tuple<std::vector<uint8_t>, pybind11::object>>;
|
||||
|
||||
class __attribute__((visibility("default"))) ShardSegment : public ShardReader {
|
||||
public:
|
||||
ShardSegment();
|
||||
|
||||
~ShardSegment() override = default;
|
||||
|
||||
/// \brief Get candidate category fields
|
||||
/// \return a list of fields names which are the candidates of category
|
||||
Status GetCategoryFields(std::shared_ptr<vector<std::string>> *fields_ptr);
|
||||
|
||||
/// \brief Set category field
|
||||
/// \param[in] category_field category name
|
||||
/// \return true if category name is existed
|
||||
Status SetCategoryField(std::string category_field);
|
||||
|
||||
/// \brief Thread-safe implementation of ReadCategoryInfo
|
||||
/// \return statistics data in json format with 2 field: "key" and "categories".
|
||||
/// The value of "categories" is a list. Each Element in list is {count, id， name}
|
||||
/// count: count of images in category
|
||||
/// id: internal unique identification, persistent
|
||||
/// name: category name
|
||||
/// example:
|
||||
/// { "key": "label",
|
||||
/// "categories": [ { "count": 3, "id": 0, "name": "sport", },
|
||||
/// { "count": 3, "id": 1, "name": "finance", } ] }
|
||||
Status ReadCategoryInfo(std::shared_ptr<std::string> *category_ptr);
|
||||
|
||||
/// \brief Thread-safe implementation of ReadAtPageById
|
||||
/// \param[in] category_id category ID
|
||||
/// \param[in] page_no page number
|
||||
/// \param[in] n_rows_of_page rows number in one page
|
||||
/// \return images array, image is a vector of uint8_t
|
||||
Status ReadAtPageById(int64_t category_id, int64_t page_no, int64_t n_rows_of_page,
|
||||
std::shared_ptr<std::vector<std::vector<uint8_t>>> *page_ptr);
|
||||
|
||||
/// \brief Thread-safe implementation of ReadAtPageByName
|
||||
/// \param[in] category_name category Name
|
||||
/// \param[in] page_no page number
|
||||
/// \param[in] n_rows_of_page rows number in one page
|
||||
/// \return images array, image is a vector of uint8_t
|
||||
Status ReadAtPageByName(std::string category_name, int64_t page_no, int64_t n_rows_of_page,
|
||||
std::shared_ptr<std::vector<std::vector<uint8_t>>> *pages_ptr);
|
||||
|
||||
Status ReadAllAtPageById(int64_t category_id, int64_t page_no, int64_t n_rows_of_page,
|
||||
std::shared_ptr<PAGES> *pages_ptr);
|
||||
|
||||
Status ReadAllAtPageByName(std::string category_name, int64_t page_no, int64_t n_rows_of_page,
|
||||
std::shared_ptr<PAGES> *pages_ptr);
|
||||
|
||||
std::pair<ShardType, std::vector<std::string>> GetBlobFields();
|
||||
|
||||
private:
|
||||
Status WrapCategoryInfo(std::shared_ptr<CATEGORY_INFO> *category_info_ptr);
|
||||
|
||||
std::string ToJsonForCategory(const std::vector<std::tuple<int, std::string, int>> &tri_vec);
|
||||
|
||||
std::string CleanUp(std::string fieldName);
|
||||
|
||||
Status PackImages(int group_id, int shard_id, std::vector<uint64_t> offset,
|
||||
std::shared_ptr<std::vector<uint8_t>> *images_ptr);
|
||||
|
||||
std::vector<std::string> candidate_category_fields_;
|
||||
std::string current_category_field_;
|
||||
const uint32_t kStartFieldId = 9;
|
||||
};
|
||||
} // namespace mindrecord
|
||||
} // namespace mindspore
|
||||
|
||||
#endif // MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_SEGMENT_H_
|
||||
|
|
@ -0,0 +1,48 @@
|
|||
/**
|
||||
* Copyright 2020-2021 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_SEQUENTIAL_SAMPLE_H_
|
||||
#define MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_SEQUENTIAL_SAMPLE_H_
|
||||
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
#include "minddata/mindrecord/include/shard_sample.h"
|
||||
|
||||
namespace mindspore {
|
||||
namespace mindrecord {
|
||||
class __attribute__((visibility("default"))) ShardSequentialSample : public ShardSample {
|
||||
public:
|
||||
ShardSequentialSample(int64_t n, int64_t offset);
|
||||
|
||||
ShardSequentialSample(float per, float per_offset);
|
||||
|
||||
~ShardSequentialSample() override{};
|
||||
|
||||
Status Execute(ShardTaskList &tasks) override;
|
||||
|
||||
int64_t GetNumSamples(int64_t dataset_size, int64_t num_classes) override;
|
||||
|
||||
private:
|
||||
int64_t offset_;
|
||||
float per_;
|
||||
float per_offset_;
|
||||
};
|
||||
} // namespace mindrecord
|
||||
} // namespace mindspore
|
||||
|
||||
#endif // MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_SEQUENTIAL_SAMPLE_H_
|
||||
|
|
@ -0,0 +1,57 @@
|
|||
/**
|
||||
* Copyright 2019-2021 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_SHUFFLE_H_
|
||||
#define MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_SHUFFLE_H_
|
||||
|
||||
#include <random>
|
||||
#include "minddata/mindrecord/include/shard_operator.h"
|
||||
|
||||
namespace mindspore {
|
||||
namespace mindrecord {
|
||||
class __attribute__((visibility("default"))) ShardShuffle : public ShardOperator {
|
||||
public:
|
||||
explicit ShardShuffle(uint32_t seed = 0, ShuffleType shuffle_type = kShuffleCategory);
|
||||
|
||||
ShardShuffle(uint32_t seed, int64_t no_of_samples, bool replacement, bool reshuffle_each_epoch,
|
||||
ShuffleType shuffle_type = kShuffleSample);
|
||||
|
||||
~ShardShuffle() override{};
|
||||
|
||||
Status Execute(ShardTaskList &tasks) override;
|
||||
|
||||
int64_t GetNumSamples(int64_t dataset_size, int64_t num_classes) override;
|
||||
|
||||
private:
|
||||
// Private helper function
|
||||
Status CategoryShuffle(ShardTaskList &tasks);
|
||||
|
||||
// Keep the file sequence the same but shuffle the data within each file
|
||||
Status ShuffleInfile(ShardTaskList &tasks);
|
||||
|
||||
// Shuffle the file sequence but keep the order of data within each file
|
||||
Status ShuffleFiles(ShardTaskList &tasks);
|
||||
|
||||
uint32_t shuffle_seed_;
|
||||
int64_t no_of_samples_;
|
||||
bool replacement_;
|
||||
bool reshuffle_each_epoch_;
|
||||
ShuffleType shuffle_type_;
|
||||
};
|
||||
} // namespace mindrecord
|
||||
} // namespace mindspore
|
||||
|
||||
#endif // MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_SHUFFLE_H_
|
||||
|
|
@ -0,0 +1,82 @@
|
|||
/**
|
||||
* Copyright 2019 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
#ifndef MINDSPORE_CCSRC_MINDDATA_MINDRECORD_STATISTICS_H
|
||||
#define MINDSPORE_CCSRC_MINDDATA_MINDRECORD_STATISTICS_H
|
||||
|
||||
#include <fstream>
|
||||
#include <iostream>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "minddata/mindrecord/include/common/shard_pybind.h"
|
||||
#include "minddata/mindrecord/include/common/shard_utils.h"
|
||||
#include "minddata/mindrecord/include/shard_error.h"
|
||||
#include "pybind11/pybind11.h"
|
||||
#include "utils/log_adapter.h"
|
||||
|
||||
namespace mindspore {
|
||||
namespace mindrecord {
|
||||
class __attribute__((visibility("default"))) Statistics {
|
||||
public:
|
||||
/// \brief save the statistic and its description
|
||||
/// \param[in] desc the statistic's description
|
||||
/// \param[in] statistics the statistic needs to be saved
|
||||
static std::shared_ptr<Statistics> Build(std::string desc, const json &statistics);
|
||||
|
||||
~Statistics() = default;
|
||||
|
||||
/// \brief compare two statistics to judge if they are equal
|
||||
/// \param b another statistics to be judged
|
||||
/// \return true if they are equal,false if not
|
||||
bool operator==(const Statistics &b) const;
|
||||
|
||||
/// \brief get the description
|
||||
/// \return the description
|
||||
std::string GetDesc() const;
|
||||
|
||||
/// \brief get the statistic
|
||||
/// \return json format of the statistic
|
||||
json GetStatistics() const;
|
||||
|
||||
/// \brief decode the bson statistics to json
|
||||
/// \param[in] encodedStatistics the bson type of statistics
|
||||
/// \return json type of statistic
|
||||
void SetStatisticsID(int64_t id);
|
||||
|
||||
/// \brief get the statistics id
|
||||
/// \return the int64 statistics id
|
||||
int64_t GetStatisticsID() const;
|
||||
|
||||
private:
|
||||
/// \brief validate the statistic
|
||||
/// \return true / false
|
||||
static bool Validate(const json &statistics);
|
||||
|
||||
static bool LevelRecursive(json level);
|
||||
|
||||
Statistics() = default;
|
||||
|
||||
std::string desc_;
|
||||
json statistics_;
|
||||
int64_t statistics_id_ = -1;
|
||||
};
|
||||
} // namespace mindrecord
|
||||
} // namespace mindspore
|
||||
|
||||
#endif // MINDSPORE_CCSRC_MINDDATA_MINDRECORD_STATISTICS_H
|
||||
|
|
@ -0,0 +1,132 @@
|
|||
/**
|
||||
* Copyright 2019-2021 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_TASK_H_
|
||||
#define MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_TASK_H_
|
||||
|
||||
#include <algorithm>
|
||||
#include <iostream>
|
||||
#include <string>
|
||||
#include <tuple>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
#include "minddata/mindrecord/include/common/shard_utils.h"
|
||||
|
||||
namespace mindspore {
|
||||
namespace mindrecord {
|
||||
|
||||
// The data struct is as below:
|
||||
// 1. TaskType: kCommonTask / kPaddedTask
|
||||
// 2. std::tuple<int, int> : shard_id, group_id(fast load) / sample_id(lazy load)
|
||||
// 3. std::vector<uint64_t>, json>> : [blob_start, blob_end], scalar_variable_fields
|
||||
using ShardTask = std::tuple<TaskType, std::tuple<int, int>, std::vector<uint64_t>, json>;
|
||||
|
||||
class __attribute__((visibility("default"))) ShardTaskList {
|
||||
public:
|
||||
ShardTaskList();
|
||||
|
||||
ShardTaskList(const ShardTaskList &task); // copy construction
|
||||
|
||||
ShardTaskList &operator=(const ShardTaskList &task); // assignment operator
|
||||
|
||||
~ShardTaskList() = default;
|
||||
|
||||
void InitSampleIds();
|
||||
|
||||
static void TaskListSwap(ShardTaskList &orig_tasks, ShardTaskList &new_tasks);
|
||||
|
||||
// Assigns the task based on task id
|
||||
inline void AssignTask(ShardTaskList &sourceTasks, int64_t id);
|
||||
|
||||
inline void InsertTask(TaskType task_type, int shard_id, int group_id, const std::vector<uint64_t> &offset,
|
||||
const json &label);
|
||||
|
||||
inline void InsertTask(const int64_t &i, TaskType task_type, int shard_id, int group_id,
|
||||
const std::vector<uint64_t> &offset, const json &label);
|
||||
|
||||
inline void InsertTask(ShardTask task);
|
||||
|
||||
inline void InsertTask(const int64_t &i, ShardTask task);
|
||||
|
||||
void MakePerm();
|
||||
|
||||
inline void InsertSampleId(int64_t id);
|
||||
|
||||
void PopBack();
|
||||
|
||||
int64_t Size() const;
|
||||
|
||||
int64_t SizeOfRows() const;
|
||||
|
||||
ShardTask &GetTaskByID(int64_t id);
|
||||
|
||||
ShardTask &GetRandomTask();
|
||||
|
||||
int64_t GetTaskSampleByID(int64_t id);
|
||||
|
||||
int64_t GetRandomTaskID();
|
||||
|
||||
static ShardTaskList Combine(std::vector<ShardTaskList> &category_tasks, bool replacement, int64_t num_elements,
|
||||
int64_t num_samples);
|
||||
|
||||
inline void ResizeTask(const int64_t &size);
|
||||
|
||||
uint32_t categories;
|
||||
|
||||
std::vector<int64_t> permutation_; // A list of ints used for shuffling sample ids
|
||||
|
||||
std::vector<int64_t> sample_ids_; // The list of actual ids that were sampled
|
||||
|
||||
std::vector<ShardTask> task_list_; // The full list of tasks
|
||||
};
|
||||
|
||||
inline void ShardTaskList::AssignTask(ShardTaskList &sourceTasks, int64_t id) {
|
||||
// Insert the sample id from the source into ourself by indexing at id position.
|
||||
// Important: The task list itself does not change.
|
||||
int64_t sample_id = sourceTasks.GetTaskSampleByID(id);
|
||||
MS_LOG(DEBUG) << "Insert sample id (" << sample_id << ") into task list from source task position: " << id;
|
||||
sample_ids_.push_back(sample_id);
|
||||
}
|
||||
|
||||
inline void ShardTaskList::InsertTask(TaskType task_type, int shard_id, int group_id,
|
||||
const std::vector<uint64_t> &offset, const json &label) {
|
||||
MS_LOG(DEBUG) << "Insert task into task list, shard_id: " << shard_id << ", group_id: " << group_id
|
||||
<< ", label: " << label.dump() << ", size of task_list_: " << task_list_.size() << ".";
|
||||
task_list_.emplace_back(task_type, std::make_tuple(shard_id, group_id), offset, label);
|
||||
}
|
||||
|
||||
inline void ShardTaskList::InsertTask(const int64_t &i, TaskType task_type, int shard_id, int group_id,
|
||||
const std::vector<uint64_t> &offset, const json &label) {
|
||||
MS_LOG(DEBUG) << "Insert task into task list, shard_id: " << shard_id << ", group_id: " << group_id
|
||||
<< ", label: " << label.dump() << ", size of task_list_: " << task_list_.size() << ".";
|
||||
task_list_[i] = {task_type, std::make_tuple(shard_id, group_id), offset, label};
|
||||
}
|
||||
|
||||
inline void ShardTaskList::InsertTask(ShardTask task) {
|
||||
MS_LOG(DEBUG) << "Insert task into task list, shard_id: " << std::get<0>(std::get<1>(task))
|
||||
<< ", group_id: " << std::get<1>(std::get<1>(task)) << ", label: " << std::get<3>(task).dump()
|
||||
<< ", size of task_list_: " << task_list_.size() << ".";
|
||||
|
||||
task_list_.push_back(std::move(task));
|
||||
}
|
||||
|
||||
inline void ShardTaskList::InsertTask(const int64_t &i, ShardTask task) { task_list_[i] = std::move(task); }
|
||||
|
||||
inline void ShardTaskList::ResizeTask(const int64_t &size) { task_list_.resize(size); }
|
||||
} // namespace mindrecord
|
||||
} // namespace mindspore
|
||||
|
||||
#endif // MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_TASK_H_
|
||||
|
|
@ -0,0 +1,256 @@
|
|||
/**
|
||||
* Copyright 2019 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_WRITER_H_
|
||||
#define MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_WRITER_H_
|
||||
|
||||
#include <libgen.h>
|
||||
#include <sys/file.h>
|
||||
#include <unistd.h>
|
||||
#include <algorithm>
|
||||
#include <array>
|
||||
#include <chrono>
|
||||
#include <exception>
|
||||
#include <fstream>
|
||||
#include <functional>
|
||||
#include <map>
|
||||
#include <memory>
|
||||
#include <mutex>
|
||||
#include <random>
|
||||
#include <string>
|
||||
#include <thread>
|
||||
#include <tuple>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
#include "minddata/mindrecord/include/common/shard_utils.h"
|
||||
#include "minddata/mindrecord/include/shard_column.h"
|
||||
#include "minddata/mindrecord/include/shard_error.h"
|
||||
#include "minddata/mindrecord/include/shard_header.h"
|
||||
#include "minddata/mindrecord/include/shard_index.h"
|
||||
#include "pybind11/pybind11.h"
|
||||
#include "pybind11/stl.h"
|
||||
#include "utils/log_adapter.h"
|
||||
|
||||
namespace mindspore {
|
||||
namespace mindrecord {
|
||||
class __attribute__((visibility("default"))) ShardWriter {
|
||||
public:
|
||||
ShardWriter();
|
||||
|
||||
~ShardWriter();
|
||||
|
||||
/// \brief Open file at the beginning 在开头打开文件
|
||||
/// \param[in] paths the file names list 文件名列表
|
||||
/// \param[in] append new data at the end of file if true, otherwise try to overwrite file 如果为 true，则文件末尾的新数据，否则尝试覆盖文件
|
||||
/// \param[in] overwrite a file with the same name if true 具有相同名称的文件（如果为 true）
|
||||
/// \return Status
|
||||
Status Open(const std::vector<std::string> &paths, bool append = false, bool overwrite = false);
|
||||
|
||||
/// \brief Open file at the ending 在结尾处打开文件
|
||||
/// \param[in] paths the file names list 文件名列表
|
||||
/// \return MSRStatus the status of MSRStatus MSR状态
|
||||
Status OpenForAppend(const std::string &path);
|
||||
|
||||
/// \brief Write header to disk 将标头写入磁盘
|
||||
/// \return MSRStatus the status of MSRStatus MSR状态
|
||||
Status Commit();
|
||||
|
||||
/// \brief Set file size 设置文件大小
|
||||
/// \param[in] header_size the size of header, only (1<<N) is accepted 标头的大小，仅接受 （1<<N）
|
||||
/// \return MSRStatus the status of MSRStatus MSR状态
|
||||
Status SetHeaderSize(const uint64_t &header_size);
|
||||
|
||||
/// \brief Set page size 设置页面大小
|
||||
/// \param[in] page_size the size of page, only (1<<N) is accepted 页面大小，仅接受 （1<<N）
|
||||
/// \return MSRStatus the status of MSRStatus MSR状态
|
||||
Status SetPageSize(const uint64_t &page_size);
|
||||
|
||||
/// \brief Set shard header 设置分片头
|
||||
/// \param[in] header_data the info of header 标题的信息
|
||||
/// WARNING, only called when file is empty 警告，仅在文件为空时调用
|
||||
/// \return MSRStatus the status of MSRStatus MSR状态
|
||||
Status SetShardHeader(std::shared_ptr<ShardHeader> header_data);
|
||||
|
||||
/// \brief write raw data by group size 按组大小写入原始数据
|
||||
/// \param[in] raw_data the vector of raw json data, vector format 原始 json 数据的向量，向量格式
|
||||
/// \param[in] blob_data the vector of image data 图像数据的向量
|
||||
/// \param[in] sign validate data or not
|
||||
/// \return MSRStatus the status of MSRStatus to judge if write successfully MSRStatus 判断写入是否成功的 MSRStatus 状态
|
||||
Status WriteRawData(std::map<uint64_t, std::vector<json>> &raw_data, vector<vector<uint8_t>> &blob_data,
|
||||
bool sign = true, bool parallel_writer = false);
|
||||
|
||||
/// \brief write raw data by group size for call from python 按组大小写入原始数据，以便从 python 调用
|
||||
/// \param[in] raw_data the vector of raw json data, python-handle format 原始 json 数据的向量，python 句柄格式
|
||||
/// \param[in] blob_data the vector of blob json data, python-handle format blob json 数据的向量，python-handle 格式
|
||||
/// \param[in] sign validate data or not 验证数据与否
|
||||
/// \return MSRStatus the status of MSRStatus to judge if write successfully MSRStatus 判断写入是否成功的 MSRStatus 状态
|
||||
Status WriteRawData(std::map<uint64_t, std::vector<py::handle>> &raw_data,
|
||||
std::map<uint64_t, std::vector<py::handle>> &blob_data, bool sign = true,
|
||||
bool parallel_writer = false);
|
||||
|
||||
Status MergeBlobData(const std::vector<string> &blob_fields,
|
||||
const std::map<std::string, std::unique_ptr<std::vector<uint8_t>>> &row_bin_data,
|
||||
std::shared_ptr<std::vector<uint8_t>> *output);
|
||||
|
||||
static Status Initialize(const std::unique_ptr<ShardWriter> *writer_ptr, const std::vector<std::string> &file_names);
|
||||
|
||||
private:
|
||||
/// \brief write shard header data to disk 将分片头数据写入磁盘
|
||||
Status WriteShardHeader();
|
||||
|
||||
/// \brief erase error data 擦除错误数据
|
||||
void DeleteErrorData(std::map<uint64_t, std::vector<json>> &raw_data, std::vector<std::vector<uint8_t>> &blob_data);
|
||||
|
||||
/// \brief populate error data 填充错误数据
|
||||
void PopulateMutexErrorData(const int &row, const std::string &message, std::map<int, std::string> &err_raw_data);
|
||||
|
||||
/// \brief check data 检查数据
|
||||
void CheckSliceData(int start_row, int end_row, json schema, const std::vector<json> &sub_raw_data,
|
||||
std::map<int, std::string> &err_raw_data);
|
||||
|
||||
/// \brief write shard header data to disk 将分片头数据写入磁盘
|
||||
Status ValidateRawData(std::map<uint64_t, std::vector<json>> &raw_data, std::vector<std::vector<uint8_t>> &blob_data,
|
||||
bool sign, std::shared_ptr<std::pair<int, int>> *count_ptr);
|
||||
|
||||
/// \brief fill data array in multiple thread run 在多线程运行中填充数据数组
|
||||
void FillArray(int start, int end, std::map<uint64_t, vector<json>> &raw_data,
|
||||
std::vector<std::vector<uint8_t>> &bin_data);
|
||||
|
||||
/// \brief serialized raw data 序列化的原始数据
|
||||
Status SerializeRawData(std::map<uint64_t, std::vector<json>> &raw_data, std::vector<std::vector<uint8_t>> &bin_data,
|
||||
uint32_t row_count);
|
||||
|
||||
/// \brief write all data parallel 并行写入所有数据
|
||||
Status ParallelWriteData(const std::vector<std::vector<uint8_t>> &blob_data,
|
||||
const std::vector<std::vector<uint8_t>> &bin_raw_data);
|
||||
|
||||
/// \brief write data shard by shard 逐个分片写入数据分片
|
||||
Status WriteByShard(int shard_id, int start_row, int end_row, const std::vector<std::vector<uint8_t>> &blob_data,
|
||||
const std::vector<std::vector<uint8_t>> &bin_raw_data);
|
||||
|
||||
/// \brief break image data up into multiple row groups 将图像数据分解为多个行组
|
||||
Status CutRowGroup(int start_row, int end_row, const std::vector<std::vector<uint8_t>> &blob_data,
|
||||
std::vector<std::pair<int, int>> &rows_in_group, const std::shared_ptr<Page> &last_raw_page,
|
||||
const std::shared_ptr<Page> &last_blob_page);
|
||||
|
||||
/// \brief append partial blob data to previous page 将部分 Blob 数据追加到上一页
|
||||
Status AppendBlobPage(const int &shard_id, const std::vector<std::vector<uint8_t>> &blob_data,
|
||||
const std::vector<std::pair<int, int>> &rows_in_group,
|
||||
const std::shared_ptr<Page> &last_blob_page);
|
||||
|
||||
/// \brief write new blob data page to disk 将新的 Blob 数据页写入磁盘
|
||||
Status NewBlobPage(const int &shard_id, const std::vector<std::vector<uint8_t>> &blob_data,
|
||||
const std::vector<std::pair<int, int>> &rows_in_group,
|
||||
const std::shared_ptr<Page> &last_blob_page);
|
||||
|
||||
/// \brief shift last row group to next raw page for new appending 将最后一行组移动到下一个原始页面以进行新的追加
|
||||
Status ShiftRawPage(const int &shard_id, const std::vector<std::pair<int, int>> &rows_in_group,
|
||||
std::shared_ptr<Page> &last_raw_page);
|
||||
|
||||
/// \brief write raw data page to disk 将原始数据页写入磁盘
|
||||
Status WriteRawPage(const int &shard_id, const std::vector<std::pair<int, int>> &rows_in_group,
|
||||
std::shared_ptr<Page> &last_raw_page, const std::vector<std::vector<uint8_t>> &bin_raw_data);
|
||||
|
||||
/// \brief generate empty raw data page 生成空的原始数据页面
|
||||
Status EmptyRawPage(const int &shard_id, std::shared_ptr<Page> &last_raw_page);
|
||||
|
||||
/// \brief append a row group at the end of raw page 在原始页面末尾追加行组
|
||||
Status AppendRawPage(const int &shard_id, const std::vector<std::pair<int, int>> &rows_in_group, const int &chunk_id,
|
||||
int &last_row_groupId, std::shared_ptr<Page> last_raw_page,
|
||||
const std::vector<std::vector<uint8_t>> &bin_raw_data);
|
||||
|
||||
/// \brief write blob chunk to disk 将 blob 块写入磁盘
|
||||
Status FlushBlobChunk(const std::shared_ptr<std::fstream> &out, const std::vector<std::vector<uint8_t>> &blob_data,
|
||||
const std::pair<int, int> &blob_row);
|
||||
|
||||
/// \brief write raw chunk to disk 将原始块写入磁盘
|
||||
Status FlushRawChunk(const std::shared_ptr<std::fstream> &out, const std::vector<std::pair<int, int>> &rows_in_group,
|
||||
const int &chunk_id, const std::vector<std::vector<uint8_t>> &bin_raw_data);
|
||||
|
||||
/// \brief break up into tasks by shard 按分片分解为任务
|
||||
std::vector<std::pair<int, int>> BreakIntoShards();
|
||||
|
||||
/// \brief calculate raw data size row by row 逐行计算原始数据大小
|
||||
Status SetRawDataSize(const std::vector<std::vector<uint8_t>> &bin_raw_data);
|
||||
|
||||
/// \brief calculate blob data size row by row 逐行计算 Blob 数据大小
|
||||
Status SetBlobDataSize(const std::vector<std::vector<uint8_t>> &blob_data);
|
||||
|
||||
/// \brief populate last raw page pointer 填充最后一个原始页面指针
|
||||
Status SetLastRawPage(const int &shard_id, std::shared_ptr<Page> &last_raw_page);
|
||||
|
||||
/// \brief populate last blob page pointer 填充最后一个 blob 页指针
|
||||
Status SetLastBlobPage(const int &shard_id, std::shared_ptr<Page> &last_blob_page);
|
||||
|
||||
/// \brief check the data by schema 按架构检查数据
|
||||
Status CheckData(const std::map<uint64_t, std::vector<json>> &raw_data);
|
||||
|
||||
/// \brief check the data and type 检查数据和类型
|
||||
Status CheckDataTypeAndValue(const std::string &key, const json &value, const json &data, const int &i,
|
||||
std::map<int, std::string> &err_raw_data);
|
||||
|
||||
/// \brief Lock writer and save pages info 锁定编写器并保存页面信息
|
||||
Status LockWriter(bool parallel_writer, std::unique_ptr<int> *fd_ptr);
|
||||
|
||||
/// \brief Unlock writer and save pages info 解锁作家并保存页面信息
|
||||
Status UnlockWriter(int fd, bool parallel_writer = false);
|
||||
|
||||
/// \brief Check raw data before writing 写入前检查原始数据
|
||||
Status WriteRawDataPreCheck(std::map<uint64_t, std::vector<json>> &raw_data, vector<vector<uint8_t>> &blob_data,
|
||||
bool sign, int *schema_count, int *row_count);
|
||||
|
||||
/// \brief Get full path from file name 从文件名获取完整路径
|
||||
Status GetFullPathFromFileName(const std::vector<std::string> &paths);
|
||||
|
||||
/// \brief Open files 打开文件
|
||||
Status OpenDataFiles(bool append, bool overwrite);
|
||||
|
||||
/// \brief Remove lock file 删除锁定文件
|
||||
Status RemoveLockFile();
|
||||
|
||||
/// \brief Remove lock file 删除锁定文件
|
||||
Status InitLockFile();
|
||||
|
||||
private:
|
||||
const std::string kLockFileSuffix = "_Locker";
|
||||
const std::string kPageFileSuffix = "_Pages";
|
||||
std::string lock_file_; // lock file for parallel run 锁定文件以进行并行运行
|
||||
std::string pages_file_; // temporary file of pages info for parallel run 用于并行运行的页面信息的临时文件
|
||||
|
||||
int shard_count_; // number of files 文件数
|
||||
uint64_t header_size_; // header size 页眉大小
|
||||
uint64_t page_size_; // page size 页面大小
|
||||
uint32_t row_count_; // count of rows 行数
|
||||
uint32_t schema_count_; // count of schemas 架构计数
|
||||
|
||||
std::vector<uint64_t> raw_data_size_; // Raw data size 原始数据大小
|
||||
std::vector<uint64_t> blob_data_size_; // Blob data size Blob 数据大小
|
||||
|
||||
std::vector<std::string> file_paths_; // file paths 文件路径
|
||||
std::vector<std::shared_ptr<std::fstream>> file_streams_; // file handles 文件句柄
|
||||
std::shared_ptr<ShardHeader> shard_header_; // shard header 分片头
|
||||
std::shared_ptr<ShardColumn> shard_column_; // shard columns 分片列
|
||||
|
||||
std::map<uint64_t, std::map<int, std::string>> err_mg_; // used for storing error raw_data info 用于存储错误raw_data信息
|
||||
|
||||
std::mutex check_mutex_; // mutex for data check 用于数据检查的互斥锁
|
||||
std::atomic<bool> flag_{false};
|
||||
std::atomic<int64_t> compression_size_;
|
||||
};
|
||||
} // namespace mindrecord
|
||||
} // namespace mindspore
|
||||
|
||||
#endif // MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_WRITER_H_
|
||||
|
|
@ -0,0 +1,62 @@
|
|||
/**
|
||||
* Copyright 2019-2021 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#include "minddata/mindrecord/include/shard_category.h"//按照路径寻找以下文件，导入到本文件
|
||||
|
||||
namespace mindspore {//创建名为mindspore的空间
|
||||
namespace mindrecord {//创建名为mindrecord的空间
|
||||
//引用ShardCategory空间中ShardCategory函数,输入三种参数调用指定函数
|
||||
//对参数categories、category_field_、num_elements_、num_categories_、replacement_进行进行参数初始化
|
||||
ShardCategory::ShardCategory(const std::vector<std::pair<std::string, std::string>> &categories, int64_t num_elements,
|
||||
bool replacement)
|
||||
: categories_(categories),
|
||||
category_field_(""),
|
||||
num_elements_(num_elements),
|
||||
num_categories_(0),
|
||||
replacement_(replacement) {}
|
||||
//引用ShardCategory空间中ShardCategory函数,输入四种参数调用指定函数
|
||||
//对参数categories、category_field_、num_elements_、num_categories_、replacement_进行进行参数初始化
|
||||
ShardCategory::ShardCategory(const std::string &category_field, int64_t num_elements, int64_t num_categories,
|
||||
bool replacement)
|
||||
: categories_({}),
|
||||
category_field_(category_field),
|
||||
num_elements_(num_elements),
|
||||
num_categories_(num_categories),
|
||||
replacement_(replacement) {}
|
||||
|
||||
Status ShardCategory::Execute(ShardTaskList &tasks) { return Status::OK(); }//在ShardCategory空间中创建Status型Execute函数,返回值为Status空间中的OK函数的返回值
|
||||
//在ShardCategory空间中创建int64_t型GetNumSamples函数,返回值为0或-1
|
||||
//判断dataset_size的值
|
||||
//若dataset_size的值为0,则返回dataset_size本身
|
||||
//若dataset_size的值大于0,则继续判断num_classes、num_categories_、num_elements_的值是否大于0。若均大于0,则修改num_classes的赋值,赋值为num_categories_和num_classes的最小值
|
||||
//继续判断num_classes是否为0,若为0则返回0。不为0则判断num_elements_是否大于int64_t类型最大值与num_classes的商,若是则返回-1。若均不符合则返回num_classes和num_elements_的乘积
|
||||
//若均不符合,则返回0
|
||||
int64_t ShardCategory::GetNumSamples(int64_t dataset_size, int64_t num_classes) {
|
||||
if (dataset_size == 0) return dataset_size;
|
||||
if (dataset_size > 0 && num_classes > 0 && num_categories_ > 0 && num_elements_ > 0) {
|
||||
num_classes = std::min(num_categories_, num_classes);
|
||||
if (num_classes == 0) {
|
||||
return 0;
|
||||
}
|
||||
if (num_elements_ > std::numeric_limits<int64_t>::max() / num_classes) {
|
||||
return -1;
|
||||
}
|
||||
return num_classes * num_elements_;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
} // namespace mindrecord
|
||||
} // namespace mindspore
|
||||
|
|
@ -0,0 +1,516 @@
|
|||
/**
|
||||
* Copyright 2020 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#include "minddata/mindrecord/include/shard_column.h"//按照路径寻找以下文件，导入到本文件,以下同理
|
||||
#include "utils/ms_utils.h"
|
||||
#include "minddata/mindrecord/include/common/shard_utils.h"
|
||||
#include "minddata/mindrecord/include/shard_error.h"
|
||||
|
||||
namespace mindspore {//创建名为mindspore的空间
|
||||
namespace mindrecord {//创建名为mindrecord的空间
|
||||
//引用ShardColumn空间中ShardColumn函数,输入两种参数调用指定函数
|
||||
//建立schema列表的表头
|
||||
ShardColumn::ShardColumn(const std::shared_ptr<ShardHeader> &shard_header, bool compress_integer) {
|
||||
auto first_schema = shard_header->GetSchemas()[0];
|
||||
json schema_json = first_schema->GetSchema();
|
||||
Init(schema_json, compress_integer);
|
||||
}
|
||||
//引用ShardColumn空间中Execute函数
|
||||
//引用Init函数进行操作
|
||||
ShardColumn::ShardColumn(const json &schema_json, bool compress_integer) { Init(schema_json, compress_integer); }
|
||||
//引用ShardColumn空间中Init函数
|
||||
void ShardColumn::Init(const json &schema_json, bool compress_integer) {
|
||||
auto schema = schema_json["schema"];
|
||||
auto blob_fields = schema_json["blob_fields"];
|
||||
//进入循环，遍历schema列表，记录key变量
|
||||
bool has_integer_array = false;
|
||||
for (json::iterator it = schema.begin(); it != schema.end(); ++it) {
|
||||
const std::string &column_name = it.key();
|
||||
column_name_.push_back(column_name);
|
||||
|
||||
json it_value = it.value();
|
||||
//判断shape是否为it_value列表的最末端，若不是则复制it_value列表加入column_shape_中，继续判断str_type是否是int32或int64类型，若是则记has_integer_array为真
|
||||
//若不是则将vec转换为int64并记入column_shape_列表中
|
||||
std::string str_type = it_value["type"];
|
||||
column_data_type_.push_back(ColumnDataTypeMap.at(str_type));
|
||||
if (it_value.find("shape") != it_value.end()) {
|
||||
std::vector<int64_t> vec(it_value["shape"].size());
|
||||
std::copy(it_value["shape"].begin(), it_value["shape"].end(), vec.begin());
|
||||
column_shape_.push_back(vec);
|
||||
if (str_type == "int32" || str_type == "int64") {
|
||||
has_integer_array = true;
|
||||
}
|
||||
} else {
|
||||
std::vector<int64_t> vec = {};
|
||||
column_shape_.push_back(vec);
|
||||
}
|
||||
}
|
||||
//创建column_name_id_列表
|
||||
for (uint64_t i = 0; i < column_name_.size(); i++) {
|
||||
column_name_id_[column_name_[i]] = i;
|
||||
}
|
||||
//创建blob_column_列表
|
||||
for (const auto &field : blob_fields) {
|
||||
blob_column_.push_back(field);
|
||||
}
|
||||
//创建blob_column_id_列表
|
||||
for (uint64_t i = 0; i < blob_column_.size(); i++) {
|
||||
blob_column_id_[blob_column_[i]] = i;
|
||||
}
|
||||
|
||||
has_compress_blob_ = (compress_integer && has_integer_array);
|
||||
num_blob_column_ = blob_column_.size();
|
||||
}
|
||||
//引用ShardColumn空间中GetColumnTypeByName函数
|
||||
Status ShardColumn::GetColumnTypeByName(const std::string &column_name, ColumnDataType *column_data_type,
|
||||
uint64_t *column_data_type_size, std::vector<int64_t> *column_shape,
|
||||
ColumnCategory *column_category) {
|
||||
RETURN_UNEXPECTED_IF_NULL(column_data_type);
|
||||
RETURN_UNEXPECTED_IF_NULL(column_data_type_size);
|
||||
RETURN_UNEXPECTED_IF_NULL(column_shape);
|
||||
RETURN_UNEXPECTED_IF_NULL(column_category);
|
||||
// Skip if column not found如果找不到列，则跳过
|
||||
*column_category = CheckColumnName(column_name);
|
||||
CHECK_FAIL_RETURN_UNEXPECTED(*column_category != ColumnNotFound,
|
||||
"[Internal ERROR] the type of column: " + column_name + " can not found.");
|
||||
|
||||
// Get data type and size获取数据类型和大小
|
||||
auto column_id = column_name_id_[column_name];
|
||||
*column_data_type = column_data_type_[column_id];
|
||||
*column_data_type_size = ColumnDataTypeSize[*column_data_type];
|
||||
*column_shape = column_shape_[column_id];
|
||||
return Status::OK();
|
||||
}
|
||||
//引用ShardColumn空间中GetColumnValueByName函数
|
||||
Status ShardColumn::GetColumnValueByName(const std::string &column_name, const std::vector<uint8_t> &columns_blob,
|
||||
const json &columns_json, const unsigned char **data,
|
||||
std::unique_ptr<unsigned char[]> *data_ptr, uint64_t *const n_bytes,
|
||||
ColumnDataType *column_data_type, uint64_t *column_data_type_size,
|
||||
std::vector<int64_t> *column_shape) {
|
||||
RETURN_UNEXPECTED_IF_NULL(column_data_type);
|
||||
RETURN_UNEXPECTED_IF_NULL(column_data_type_size);
|
||||
RETURN_UNEXPECTED_IF_NULL(column_shape);
|
||||
// Skip if column not found如果找不到列，则跳过
|
||||
auto column_category = CheckColumnName(column_name);
|
||||
CHECK_FAIL_RETURN_UNEXPECTED(column_category != ColumnNotFound,
|
||||
"[Internal ERROR] the type of column: " + column_name + " can not found.");
|
||||
// Get data type and size获取数据类型和大小
|
||||
auto column_id = column_name_id_[column_name];
|
||||
*column_data_type = column_data_type_[column_id];
|
||||
*column_data_type_size = ColumnDataTypeSize[*column_data_type];
|
||||
*column_shape = column_shape_[column_id];
|
||||
|
||||
// Retrieve value from json从json检索值
|
||||
if (column_category == ColumnInRaw) {
|
||||
RETURN_IF_NOT_OK(GetColumnFromJson(column_name, columns_json, data_ptr, n_bytes));
|
||||
*data = reinterpret_cast<const unsigned char *>(data_ptr->get());
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
// Retrieve value from blob从blob检索值
|
||||
RETURN_IF_NOT_OK(GetColumnFromBlob(column_name, columns_blob, data, data_ptr, n_bytes));
|
||||
if (*data == nullptr) {
|
||||
*data = reinterpret_cast<const unsigned char *>(data_ptr->get());
|
||||
}
|
||||
return Status::OK();
|
||||
}
|
||||
//引用ShardColumn空间中GetColumnFromJson函数
|
||||
Status ShardColumn::GetColumnFromJson(const std::string &column_name, const json &columns_json,
|
||||
std::unique_ptr<unsigned char[]> *data_ptr, uint64_t *n_bytes) {
|
||||
RETURN_UNEXPECTED_IF_NULL(n_bytes);
|
||||
RETURN_UNEXPECTED_IF_NULL(data_ptr);
|
||||
auto column_id = column_name_id_[column_name];
|
||||
auto column_data_type = column_data_type_[column_id];
|
||||
|
||||
// Initialize num bytes初始化（以字节为单位）
|
||||
*n_bytes = ColumnDataTypeSize[column_data_type];
|
||||
auto json_column_value = columns_json[column_name];
|
||||
CHECK_FAIL_RETURN_UNEXPECTED(json_column_value.is_string() || json_column_value.is_number(),
|
||||
"[Internal ERROR] the value of column: " + column_name +
|
||||
" should be string or number but got: " + json_column_value.dump());
|
||||
//通过column_data_type选择操作方法
|
||||
switch (column_data_type) {
|
||||
case ColumnFloat32: {
|
||||
return GetFloat<float>(data_ptr, json_column_value, false);
|
||||
}
|
||||
case ColumnFloat64: {
|
||||
return GetFloat<double>(data_ptr, json_column_value, true);
|
||||
}
|
||||
case ColumnInt32: {
|
||||
return GetInt<int32_t>(data_ptr, json_column_value);
|
||||
}
|
||||
case ColumnInt64: {
|
||||
return GetInt<int64_t>(data_ptr, json_column_value);
|
||||
}
|
||||
default: {
|
||||
// Convert string to c_str将字符串转换为c_str
|
||||
std::string tmp_string;
|
||||
if (json_column_value.is_string()) {
|
||||
tmp_string = json_column_value.get<string>();
|
||||
} else {
|
||||
tmp_string = json_column_value.dump();
|
||||
}
|
||||
*n_bytes = tmp_string.size();
|
||||
auto data = reinterpret_cast<const unsigned char *>(common::SafeCStr(tmp_string));
|
||||
*data_ptr = std::make_unique<unsigned char[]>(*n_bytes);
|
||||
for (uint32_t i = 0; i < *n_bytes; i++) {
|
||||
(*data_ptr)[i] = *(data + i);
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
return Status::OK();
|
||||
}
|
||||
//创建函数模板，引用ShardColumn空间中GetFloat函数
|
||||
template <typename T>
|
||||
Status ShardColumn::GetFloat(std::unique_ptr<unsigned char[]> *data_ptr, const json &json_column_value,
|
||||
bool use_double) {
|
||||
RETURN_UNEXPECTED_IF_NULL(data_ptr);
|
||||
std::unique_ptr<T[]> array_data = std::make_unique<T[]>(1);
|
||||
if (json_column_value.is_number()) {
|
||||
array_data[0] = json_column_value;
|
||||
} else {
|
||||
// Convert string to float将字符串转换为浮点
|
||||
try {
|
||||
if (use_double) {
|
||||
array_data[0] = json_column_value.get<double>();
|
||||
} else {
|
||||
array_data[0] = json_column_value.get<float>();
|
||||
}
|
||||
} catch (json::exception &e) {
|
||||
RETURN_STATUS_UNEXPECTED("[Internal ERROR] Failed to convert column value:" + json_column_value.dump() +
|
||||
" to type float, " + std::string(e.what()));
|
||||
}
|
||||
}
|
||||
//创建data_ptr列表
|
||||
auto data = reinterpret_cast<const unsigned char *>(array_data.get());
|
||||
*data_ptr = std::make_unique<unsigned char[]>(sizeof(T));
|
||||
for (uint32_t i = 0; i < sizeof(T); i++) {
|
||||
(*data_ptr)[i] = *(data + i);
|
||||
}
|
||||
return Status::OK();
|
||||
}
|
||||
//创建函数模板，引用ShardColumn空间中GetInt函数
|
||||
template <typename T>
|
||||
Status ShardColumn::GetInt(std::unique_ptr<unsigned char[]> *data_ptr, const json &json_column_value) {
|
||||
RETURN_UNEXPECTED_IF_NULL(data_ptr);
|
||||
std::unique_ptr<T[]> array_data = std::make_unique<T[]>(1);
|
||||
int64_t temp_value;
|
||||
bool less_than_zero = false;
|
||||
//判断json_column_value是否为integer类型，若是则令json_zero为0、temp_value等于json_column_value,并判断json_column_value是否小于0，若是则令less_than_zero为true
|
||||
//判断json_column_value是否为string类型，若是则令string_value为json_column_value
|
||||
if (json_column_value.is_number_integer()) {
|
||||
const json json_zero = 0;
|
||||
if (json_column_value < json_zero) {
|
||||
less_than_zero = true;
|
||||
}
|
||||
temp_value = json_column_value;
|
||||
} else if (json_column_value.is_string()) {
|
||||
std::string string_value = json_column_value;
|
||||
//设置异常捕捉器
|
||||
try {
|
||||
//判断string_value是否为空且string_value列表的第一位是否为‘-’,若是则给temp_value和less_than_zero赋值
|
||||
//若不是则直接给temp_value赋值
|
||||
if (!string_value.empty() && string_value[0] == '-') {
|
||||
temp_value = std::stoll(string_value);
|
||||
less_than_zero = true;
|
||||
} else {
|
||||
temp_value = static_cast<int64_t>(std::stoull(string_value));
|
||||
}
|
||||
} catch (std::invalid_argument &e) {//若问题类型为参数无效，则返回相应错误信息
|
||||
RETURN_STATUS_UNEXPECTED("[Internal ERROR] Failed to convert column value:" + string_value + " to type int, " +
|
||||
std::string(e.what()));
|
||||
} catch (std::out_of_range &e) {//若问题类型为超出范围，则返回相应错误信息
|
||||
RETURN_STATUS_UNEXPECTED("[Internal ERROR] Failed to convert column value:" + string_value + " to type int, " +
|
||||
std::string(e.what()));
|
||||
}
|
||||
} else {//若均不符合，则返回相应错误信息
|
||||
RETURN_STATUS_UNEXPECTED("[Internal ERROR] column value should be type string or number but got: " +
|
||||
json_column_value.dump());
|
||||
}
|
||||
//进行判断，若符合则返回相应错误信息
|
||||
if ((less_than_zero && temp_value < static_cast<int64_t>(std::numeric_limits<T>::min())) ||
|
||||
(!less_than_zero && static_cast<uint64_t>(temp_value) > static_cast<uint64_t>(std::numeric_limits<T>::max()))) {
|
||||
RETURN_STATUS_UNEXPECTED("[Internal ERROR] column value: " + std::to_string(temp_value) + " is out of range.");
|
||||
}
|
||||
array_data[0] = static_cast<T>(temp_value);
|
||||
//进入循环，建立data_ptr列表
|
||||
auto data = reinterpret_cast<const unsigned char *>(array_data.get());
|
||||
*data_ptr = std::make_unique<unsigned char[]>(sizeof(T));
|
||||
for (uint32_t i = 0; i < sizeof(T); i++) {
|
||||
(*data_ptr)[i] = *(data + i);
|
||||
}
|
||||
return Status::OK();
|
||||
}
|
||||
//引用ShardColumn空间中GetColumnFromBlob函数
|
||||
//判断已建立的column_name_id_列表和column_data_type_列表是否相同并进行操作
|
||||
Status ShardColumn::GetColumnFromBlob(const std::string &column_name, const std::vector<uint8_t> &columns_blob,
|
||||
const unsigned char **data, std::unique_ptr<unsigned char[]> *data_ptr,
|
||||
uint64_t *const n_bytes) {
|
||||
RETURN_UNEXPECTED_IF_NULL(data);
|
||||
uint64_t offset_address = 0;
|
||||
auto column_id = column_name_id_[column_name];
|
||||
RETURN_IF_NOT_OK(GetColumnAddressInBlock(column_id, columns_blob, n_bytes, &offset_address));
|
||||
auto column_data_type = column_data_type_[column_id];
|
||||
if (has_compress_blob_ && column_data_type == ColumnInt32) {
|
||||
RETURN_IF_NOT_OK(UncompressInt<int32_t>(column_id, data_ptr, columns_blob, n_bytes, offset_address));
|
||||
} else if (has_compress_blob_ && column_data_type == ColumnInt64) {
|
||||
RETURN_IF_NOT_OK(UncompressInt<int64_t>(column_id, data_ptr, columns_blob, n_bytes, offset_address));
|
||||
} else {
|
||||
*data = reinterpret_cast<const unsigned char *>(&(columns_blob[offset_address]));
|
||||
}
|
||||
|
||||
return Status::OK();
|
||||
}
|
||||
//引用ShardColumn空间中GetColumnName函数
|
||||
//判断it_column是佛偶为column_name_id_列表的结尾，若是则返回ColumnNotFound
|
||||
//若不是则返回ColumnInRaw或ColumnInBlob
|
||||
ColumnCategory ShardColumn::CheckColumnName(const std::string &column_name) {
|
||||
auto it_column = column_name_id_.find(column_name);
|
||||
if (it_column == column_name_id_.end()) {
|
||||
return ColumnNotFound;
|
||||
}
|
||||
auto it_blob = blob_column_id_.find(column_name);
|
||||
return it_blob == blob_column_id_.end() ? ColumnInRaw : ColumnInBlob;
|
||||
}
|
||||
//引用ShardColumn空间中CompressBlob函数
|
||||
std::vector<uint8_t> ShardColumn::CompressBlob(const std::vector<uint8_t> &blob, int64_t *compression_size) {
|
||||
// Skip if no compress columns如果没有压缩列，则跳过
|
||||
*compression_size = 0;
|
||||
if (!CheckCompressBlob()) {
|
||||
return blob;
|
||||
}
|
||||
|
||||
std::vector<uint8_t> dst_blob;
|
||||
uint64_t i_src = 0;
|
||||
for (int64_t i = 0; i < num_blob_column_; i++) {
|
||||
// Get column data type获取列数据类型
|
||||
auto src_data_type = column_data_type_[column_name_id_[blob_column_[i]]];
|
||||
auto int_type = src_data_type == ColumnInt32 ? kInt32Type : kInt64Type;
|
||||
|
||||
// Compress and return is blob has 1 column only压缩并返回blob只有1列
|
||||
if (num_blob_column_ == 1) {
|
||||
dst_blob = CompressInt(blob, int_type);
|
||||
*compression_size = static_cast<int64_t>(blob.size()) - static_cast<int64_t>(dst_blob.size());
|
||||
return dst_blob;
|
||||
}
|
||||
|
||||
// Just copy and continue if column dat type is not int32/int64如果列数据类型不是int32/int64，只需复制并继续
|
||||
uint64_t num_bytes = BytesBigToUInt64(blob, i_src, kInt64Type);
|
||||
if (src_data_type != ColumnInt32 && src_data_type != ColumnInt64) {
|
||||
dst_blob.insert(dst_blob.end(), blob.begin() + i_src, blob.begin() + i_src + kInt64Len + num_bytes);
|
||||
i_src += kInt64Len + num_bytes;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Get column slice in source blob获取源blob中的列切片
|
||||
std::vector<uint8_t> blob_slice(blob.begin() + i_src + kInt64Len, blob.begin() + i_src + kInt64Len + num_bytes);
|
||||
// Compress column压缩列
|
||||
auto dst_blob_slice = CompressInt(blob_slice, int_type);
|
||||
// Get new column size获取新列大小
|
||||
auto new_blob_size = UIntToBytesBig(dst_blob_slice.size(), kInt64Type);
|
||||
// Append new column size附加新列大小
|
||||
dst_blob.insert(dst_blob.end(), new_blob_size.begin(), new_blob_size.end());
|
||||
// Append new column data附加新列数据
|
||||
dst_blob.insert(dst_blob.end(), dst_blob_slice.begin(), dst_blob_slice.end());
|
||||
i_src += kInt64Len + num_bytes;
|
||||
}
|
||||
MS_LOG(DEBUG) << "Compress blob data from " << blob.size() << " to " << dst_blob.size() << ".";
|
||||
*compression_size = static_cast<int64_t>(blob.size()) - static_cast<int64_t>(dst_blob.size());
|
||||
return dst_blob;
|
||||
}
|
||||
//引用ShardColumn空间中CompressBlob函数
|
||||
vector<uint8_t> ShardColumn::CompressInt(const vector<uint8_t> &src_bytes, const IntegerType &int_type) {
|
||||
uint64_t i_size = kUnsignedOne << static_cast<uint8_t>(int_type);
|
||||
// Get number of elements获取元素数
|
||||
uint64_t src_n_int = src_bytes.size() / i_size;
|
||||
// Calculate bitmap size (bytes)计算位图大小（字节）
|
||||
uint64_t bitmap_size = (src_n_int + kNumDataOfByte - 1) / kNumDataOfByte;
|
||||
|
||||
// Initialize destination blob, more space than needed, will be resized初始化目标blob，超出所需空间，将调整大小
|
||||
vector<uint8_t> dst_bytes(kBytesOfColumnLen + bitmap_size + src_bytes.size(), 0);
|
||||
|
||||
// Write number of elements to destination blob将元素数写入目标blob
|
||||
vector<uint8_t> size_by_bytes = UIntToBytesBig(src_n_int, kInt32Type);
|
||||
for (uint64_t n = 0; n < kBytesOfColumnLen; n++) {
|
||||
dst_bytes[n] = size_by_bytes[n];
|
||||
}
|
||||
|
||||
// Write compressed int写入压缩int
|
||||
uint64_t i_dst = kBytesOfColumnLen + bitmap_size;
|
||||
for (uint64_t i = 0; i < src_n_int; i++) {
|
||||
// Initialize destination data type初始化目标数据类型
|
||||
IntegerType dst_int_type = kInt8Type;
|
||||
// Shift to next int position移到下一个int位置
|
||||
uint64_t pos = i * (kUnsignedOne << static_cast<uint8_t>(int_type));
|
||||
// Narrow down this int缩小这个整数
|
||||
int64_t i_n = BytesLittleToMinIntType(src_bytes, pos, int_type, &dst_int_type);
|
||||
|
||||
// Write this int to destination blob将此int写入目标blob
|
||||
uint64_t u_n = *reinterpret_cast<uint64_t *>(&i_n);
|
||||
auto temp_bytes = UIntToBytesLittle(u_n, dst_int_type);
|
||||
for (uint64_t j = 0; j < (kUnsignedOne << static_cast<uint8_t>(dst_int_type)); j++) {
|
||||
dst_bytes[i_dst++] = temp_bytes[j];
|
||||
}
|
||||
|
||||
// Update date type in bit map更新位图中的日期类型
|
||||
dst_bytes[i / kNumDataOfByte + kBytesOfColumnLen] |=
|
||||
(static_cast<uint8_t>(dst_int_type) << (kDataTypeBits * (kNumDataOfByte - kUnsignedOne - (i % kNumDataOfByte))));
|
||||
}
|
||||
// Resize destination blob调整目标blob的大小
|
||||
dst_bytes.resize(i_dst);
|
||||
MS_LOG(DEBUG) << "Compress blob field from " << src_bytes.size() << " to " << dst_bytes.size() << ".";
|
||||
return dst_bytes;
|
||||
}
|
||||
//引用ShardColumn空间中GetColumnAddressInBlock函数
|
||||
Status ShardColumn::GetColumnAddressInBlock(const uint64_t &column_id, const std::vector<uint8_t> &columns_blob,
|
||||
uint64_t *num_bytes, uint64_t *shift_idx) {
|
||||
RETURN_UNEXPECTED_IF_NULL(num_bytes);
|
||||
RETURN_UNEXPECTED_IF_NULL(shift_idx);
|
||||
//判断num_blob_column_是否为1，若是则用指针记录columns_blob列表的大小并返回
|
||||
if (num_blob_column_ == 1) {
|
||||
*num_bytes = columns_blob.size();
|
||||
*shift_idx = 0;
|
||||
return Status::OK();
|
||||
}
|
||||
auto blob_id = blob_column_id_[column_name_[column_id]];
|
||||
//进入循环，按步骤调用BytesBigToUInt64
|
||||
for (int32_t i = 0; i < blob_id; i++) {
|
||||
*shift_idx += kInt64Len + BytesBigToUInt64(columns_blob, *shift_idx, kInt64Type);
|
||||
}
|
||||
*num_bytes = BytesBigToUInt64(columns_blob, *shift_idx, kInt64Type);
|
||||
|
||||
(*shift_idx) += kInt64Len;
|
||||
|
||||
return Status::OK();
|
||||
}
|
||||
//创建函数模板，引用ShardColumn空间中UncompressInt函数
|
||||
template <typename T>
|
||||
Status ShardColumn::UncompressInt(const uint64_t &column_id, std::unique_ptr<unsigned char[]> *const data_ptr,
|
||||
const std::vector<uint8_t> &columns_blob, uint64_t *num_bytes, uint64_t shift_idx) {
|
||||
RETURN_UNEXPECTED_IF_NULL(data_ptr);
|
||||
RETURN_UNEXPECTED_IF_NULL(num_bytes);
|
||||
auto num_elements = BytesBigToUInt64(columns_blob, shift_idx, kInt32Type);
|
||||
*num_bytes = sizeof(T) * num_elements;
|
||||
|
||||
// Parse integer array解析整数数组
|
||||
uint64_t i_source = shift_idx + kBytesOfColumnLen + (num_elements + kNumDataOfByte - 1) / kNumDataOfByte;
|
||||
auto array_data = std::make_unique<T[]>(num_elements);
|
||||
|
||||
for (uint64_t i = 0; i < num_elements; i++) {
|
||||
uint8_t iBitMap = columns_blob[shift_idx + kBytesOfColumnLen + i / kNumDataOfByte];
|
||||
uint64_t i_type = (iBitMap >> ((kNumDataOfByte - 1 - (i % kNumDataOfByte)) * kDataTypeBits)) & kDataTypeBitMask;
|
||||
auto mr_int_type = static_cast<IntegerType>(i_type);
|
||||
int64_t i64 = BytesLittleToMinIntType(columns_blob, i_source, mr_int_type);
|
||||
i_source += (kUnsignedOne << i_type);
|
||||
array_data[i] = static_cast<T>(i64);
|
||||
}
|
||||
|
||||
auto data = reinterpret_cast<const unsigned char *>(array_data.get());
|
||||
*data_ptr = std::make_unique<unsigned char[]>(*num_bytes);
|
||||
// field is none. for example: numpy is null字段为无。例如：numpy为null
|
||||
if (*num_bytes == 0) {
|
||||
return Status::OK();
|
||||
}
|
||||
CHECK_FAIL_RETURN_UNEXPECTED(memcpy_s(data_ptr->get(), *num_bytes, data, *num_bytes) == 0,
|
||||
"[Internal ERROR] Failed to call securec func [memcpy_s]");
|
||||
return Status::OK();
|
||||
}
|
||||
//引用ShardColumn空间中BytesBigToUInt64函数
|
||||
//进入循环，计算result的值并返回
|
||||
uint64_t ShardColumn::BytesBigToUInt64(const std::vector<uint8_t> &bytes_array, const uint64_t &pos,
|
||||
const IntegerType &i_type) {
|
||||
uint64_t result = 0;
|
||||
for (uint64_t i = 0; i < (kUnsignedOne << static_cast<uint8_t>(i_type)); i++) {
|
||||
result = (result << kBitsOfByte) + bytes_array[pos + i];
|
||||
}
|
||||
return result;
|
||||
}
|
||||
//引用ShardColumn空间中UIntToBytesBig函数
|
||||
//进入循环，根据操作计算result的值并返回
|
||||
std::vector<uint8_t> ShardColumn::UIntToBytesBig(uint64_t value, const IntegerType &i_type) {
|
||||
uint64_t n_bytes = kUnsignedOne << static_cast<uint8_t>(i_type);
|
||||
std::vector<uint8_t> result(n_bytes, 0);
|
||||
for (uint64_t i = 0; i < n_bytes; i++) {
|
||||
result[n_bytes - 1 - i] = value & std::numeric_limits<uint8_t>::max();
|
||||
value >>= kBitsOfByte;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
//引用ShardColumn空间中UIntToBytesLittle函数
|
||||
//进入循环，根据操作计算result的值并返回
|
||||
std::vector<uint8_t> ShardColumn::UIntToBytesLittle(uint64_t value, const IntegerType &i_type) {
|
||||
uint64_t n_bytes = kUnsignedOne << static_cast<uint8_t>(i_type);
|
||||
std::vector<uint8_t> result(n_bytes, 0);
|
||||
for (uint64_t i = 0; i < n_bytes; i++) {
|
||||
result[i] = value & std::numeric_limits<uint8_t>::max();
|
||||
value >>= kBitsOfByte;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
//引用ShardColumn空间中BytesLittleToMinIntType函数
|
||||
int64_t ShardColumn::BytesLittleToMinIntType(const std::vector<uint8_t> &bytes_array, const uint64_t &pos,
|
||||
const IntegerType &src_i_type, IntegerType *dst_i_type) {
|
||||
uint64_t u_temp = 0;
|
||||
//进入循环，计算u_temp的值
|
||||
for (uint64_t i = 0; i < (kUnsignedOne << static_cast<uint8_t>(src_i_type)); i++) {
|
||||
u_temp = (u_temp << kBitsOfByte) +
|
||||
bytes_array[pos + (kUnsignedOne << static_cast<uint8_t>(src_i_type)) - kUnsignedOne - i];
|
||||
}
|
||||
//根据src_i_type的类型判断，并选择相应的处理
|
||||
int64_t i_out;
|
||||
switch (src_i_type) {
|
||||
case kInt8Type: {
|
||||
i_out = (int8_t)(u_temp & std::numeric_limits<uint8_t>::max());
|
||||
break;
|
||||
}
|
||||
case kInt16Type: {
|
||||
i_out = (int16_t)(u_temp & std::numeric_limits<uint16_t>::max());
|
||||
break;
|
||||
}
|
||||
case kInt32Type: {
|
||||
i_out = (int32_t)(u_temp & std::numeric_limits<uint32_t>::max());
|
||||
break;
|
||||
}
|
||||
case kInt64Type: {
|
||||
i_out = (int64_t)(u_temp & std::numeric_limits<uint64_t>::max());
|
||||
break;
|
||||
}
|
||||
default: {
|
||||
i_out = 0;
|
||||
}
|
||||
}
|
||||
//判断dst_i_type是否为假，若是则直接返回i_out
|
||||
if (!dst_i_type) {
|
||||
return i_out;
|
||||
}
|
||||
//判断i_out的取值，给指针dst_i_type赋值
|
||||
if (i_out >= static_cast<int64_t>(std::numeric_limits<int8_t>::min()) &&
|
||||
i_out <= static_cast<int64_t>(std::numeric_limits<int8_t>::max())) {
|
||||
*dst_i_type = kInt8Type;
|
||||
} else if (i_out >= static_cast<int64_t>(std::numeric_limits<int16_t>::min()) &&
|
||||
i_out <= static_cast<int64_t>(std::numeric_limits<int16_t>::max())) {
|
||||
*dst_i_type = kInt16Type;
|
||||
} else if (i_out >= static_cast<int64_t>(std::numeric_limits<int32_t>::min()) &&
|
||||
i_out <= static_cast<int64_t>(std::numeric_limits<int32_t>::max())) {
|
||||
*dst_i_type = kInt32Type;
|
||||
} else {
|
||||
*dst_i_type = kInt64Type;
|
||||
}
|
||||
return i_out;
|
||||
}
|
||||
} // namespace mindrecord
|
||||
} // namespace mindspore
|
||||
|
|
@ -0,0 +1,95 @@
|
|||
/**
|
||||
* Copyright 2020-2021 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#include "minddata/mindrecord/include/shard_distributed_sample.h"//按照路径寻找以下文件，导入到本文件
|
||||
|
||||
using mindspore::LogStream;//声明mindspore空间下的LogStream
|
||||
using mindspore::ExceptionType::NoExceptionType;//声明mindspore空间下ExceptionType类中的NoExceptionType
|
||||
using mindspore::MsLogLevel::ERROR;//声明mindspore空间下MsLogLevel类中的ERROR
|
||||
|
||||
namespace mindspore {//创建名为mindspore的空间
|
||||
namespace mindrecord {//创建名为mindrecord的空间
|
||||
//引用ShardDistributedSample空间中ShardDistributedSample函数,输入七种参数调用指定函数
|
||||
//对参数ShardSample、shuffle_、no_of_padded_samples_、first_epoch_、shuffle_op_进行进行参数初始化
|
||||
ShardDistributedSample::ShardDistributedSample(int num_shards, int shard_id, int64_t no_of_padded_samples, bool shuffle,
|
||||
uint32_t seed, int64_t no_of_samples, int64_t offset)
|
||||
: ShardSample(1, num_shards, shard_id, no_of_samples, offset),
|
||||
shuffle_(shuffle),
|
||||
no_of_padded_samples_(no_of_padded_samples),
|
||||
first_epoch_(true) {
|
||||
shuffle_op_ = std::make_shared<ShardShuffle>(seed, kShuffleSample);
|
||||
}
|
||||
//引用ShardDistributedSample空间中ShardDistributedSample函数,输入六种参数调用指定函数
|
||||
//对参数ShardDistributedSample进行进行参数初始化
|
||||
ShardDistributedSample::ShardDistributedSample(int num_shards, int shard_id, bool shuffle, uint32_t seed,
|
||||
int64_t no_of_samples, int64_t offset)
|
||||
: ShardDistributedSample(num_shards, shard_id, 0, shuffle, seed, no_of_samples, offset) {}
|
||||
//在ShardDistributedSample空间中创建int64_t型GetNumSamples函数,返回值为0或-1
|
||||
//判断no_of_padded_samples_的值
|
||||
//若no_of_padded_samples_的值小于等于0,令res变量等于0。若no_of_padded_samples_大于0,则返回0。
|
||||
//判断dataset_size与denominator_的模是否等于0,若相等则将dataset_size与denominator_和numerator_的乘积的商赋给res
|
||||
//若不相等,则将dataset_size与denominator_和numerator_的乘积的商+1后赋给res
|
||||
//返回no_of_samples_,no_of_samples_的取值取决于res是否等于0,若等于0则返回0,若不等于0则返回no_of_samples_与res的最小值
|
||||
//若no_of_padded_samples_的值大于0,将dataset_size和no_of_padded_samples_的和赋给padded_size
|
||||
//判断padded_size与denominator_的模是否为0,若是则返回padded_size与denominator_和numerator_的乘积的商,若不是则返回-1
|
||||
int64_t ShardDistributedSample::GetNumSamples(int64_t dataset_size, int64_t num_classes) {
|
||||
if (no_of_padded_samples_ <= 0) {
|
||||
int64_t res = 0;
|
||||
if (dataset_size % denominator_ == 0) {
|
||||
res = dataset_size / denominator_ * numerator_;
|
||||
} else {
|
||||
res = dataset_size / denominator_ * numerator_ + 1;
|
||||
}
|
||||
return no_of_samples_ == 0 ? res : std::min(no_of_samples_, res);
|
||||
} else {
|
||||
auto padded_size = dataset_size + no_of_padded_samples_;
|
||||
if (padded_size % denominator_ == 0) {
|
||||
return padded_size / denominator_ * numerator_;
|
||||
} else {
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
//在ShardDistributedSample空间中创建Status型PreExecute函数,返回值为Status变量
|
||||
//将tasks.Size()函数的返回值赋给total_no
|
||||
//判断no_of_padded_samples_是否大于0且first_epoch_是否为真,若是则调用CHECK_FAIL_RETURN_UNEXPECTED函数并输出非有效数据警告
|
||||
//若no_of_padded_samples_小于0,则判断first_epoch_是否为真,若是则将first_epoch_改为否,将tasks赋给task_。若不是则将task_赋给tasks
|
||||
//判断shuffle_是否为真，若是则调用shuffle_op_指针中SetShardSampleCount和UpdateShuffleMode的函数并调用RETURN_IF_NOT_OK函数
|
||||
//返回Status中Ok函数的返回值
|
||||
Status ShardDistributedSample::PreExecute(ShardTaskList &tasks) {
|
||||
auto total_no = tasks.Size();
|
||||
if (no_of_padded_samples_ > 0 && first_epoch_) {
|
||||
CHECK_FAIL_RETURN_UNEXPECTED(total_no % denominator_ == 0,
|
||||
"Invalid data, the size of dataset and padded samples: " + std::to_string(total_no) +
|
||||
" can not be divisible by the value of 'num_shards': " +
|
||||
std::to_string(denominator_) + ".\n Please adjust the value of 'num_padded'.");
|
||||
}
|
||||
if (first_epoch_) {
|
||||
first_epoch_ = false;
|
||||
task_ = tasks;
|
||||
} else {
|
||||
tasks = task_;
|
||||
}
|
||||
if (shuffle_ == true) {
|
||||
shuffle_op_->SetShardSampleCount(GetShardSampleCount());
|
||||
shuffle_op_->UpdateShuffleMode(GetShuffleMode());
|
||||
RETURN_IF_NOT_OK((*shuffle_op_)(tasks));
|
||||
}
|
||||
return Status::OK();
|
||||
}
|
||||
} // namespace mindrecord
|
||||
} // namespace mindspore
|
||||
|
|
@ -0,0 +1,724 @@
|
|||
/**
|
||||
* Copyright 2019 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#include "minddata/mindrecord/include/shard_header.h"//按照路径寻找以下文件，导入到本文件
|
||||
|
||||
#include <map>//
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include "utils/file_utils.h"//按照路径寻找以下文件，导入到本文件
|
||||
#include "utils/ms_utils.h"
|
||||
#include "minddata/mindrecord/include/shard_error.h"
|
||||
#include "minddata/mindrecord/include/shard_page.h"
|
||||
|
||||
using mindspore::LogStream;//声明mindspore空间下的LogStream
|
||||
using mindspore::ExceptionType::NoExceptionType;//声明mindspore空间下ExceptionType类中的NoExceptionType
|
||||
using mindspore::MsLogLevel::ERROR;//声明mindspore空间下MsLogLevel类中的ERROR
|
||||
|
||||
namespace mindspore {//创建名为mindspore的空间
|
||||
namespace mindrecord {//创建名为mindrecord的空间
|
||||
//在atomic空间中创建bool型thread_status函数
|
||||
std::atomic<bool> thread_status(false);
|
||||
//引用ShardHeader空间中ShardHeader函数,输入三种参数调用指定函数
|
||||
//对参数shard_count_、header_size_、page_size_、compression_size_进行进行参数初始化
|
||||
//给index_变量赋值，值为访问Index空间的数量
|
||||
ShardHeader::ShardHeader() : shard_count_(0), header_size_(0), page_size_(0), compression_size_(0) {
|
||||
index_ = std::make_shared<Index>();
|
||||
}
|
||||
//在ShardHeader空间中创建InitializeHeader函数
|
||||
Status ShardHeader::InitializeHeader(const std::vector<json> &headers, bool load_dataset) {
|
||||
shard_count_ = headers.size();
|
||||
int shard_index = 0;
|
||||
bool first = true;
|
||||
//进入循环，判断first是否为真，若是则进行header列表的创建
|
||||
for (const auto &header : headers) {
|
||||
if (first) {
|
||||
first = false;
|
||||
RETURN_IF_NOT_OK(ParseSchema(header["schema"]));
|
||||
RETURN_IF_NOT_OK(ParseIndexFields(header["index_fields"]));
|
||||
RETURN_IF_NOT_OK(ParseStatistics(header["statistics"]));
|
||||
ParseShardAddress(header["shard_addresses"]);
|
||||
header_size_ = header["header_size"].get<uint64_t>();
|
||||
page_size_ = header["page_size"].get<uint64_t>();
|
||||
compression_size_ = header.contains("compression_size") ? header["compression_size"].get<uint64_t>() : 0;
|
||||
}
|
||||
RETURN_IF_NOT_OK(ParsePage(header["page"], shard_index, load_dataset));
|
||||
shard_index++;
|
||||
}
|
||||
return Status::OK();
|
||||
}
|
||||
//在ShardHeader空间中创建CheckFileStatus函数
|
||||
Status ShardHeader::CheckFileStatus(const std::string &path) {
|
||||
auto realpath = FileUtils::GetRealPath(path.c_str());
|
||||
//捕捉错误，并输出错误信息
|
||||
CHECK_FAIL_RETURN_UNEXPECTED(
|
||||
realpath.has_value(),
|
||||
"Invalid file, failed to get the realpath of mindrecord files. Please check file path: " + path);
|
||||
std::ifstream fin(realpath.value(), std::ios::in | std::ios::binary);
|
||||
CHECK_FAIL_RETURN_UNEXPECTED(fin.is_open(),
|
||||
"Invalid file, failed to open files for loading mindrecord files. Please check file "
|
||||
"path, permission and open file limit: " +
|
||||
path);
|
||||
// fetch file size获取文件大小
|
||||
auto &io_seekg = fin.seekg(0, std::ios::end);
|
||||
if (!io_seekg.good() || io_seekg.fail() || io_seekg.bad()) {
|
||||
fin.close();
|
||||
RETURN_STATUS_UNEXPECTED("[Internal ERROR] failed to seekg file, file path: " + path);
|
||||
}
|
||||
//捕捉错误，并输出错误信息
|
||||
size_t file_size = fin.tellg();
|
||||
if (file_size < kMinFileSize) {
|
||||
fin.close();
|
||||
RETURN_STATUS_UNEXPECTED("Invalid file, the size of mindrecord file: " + std::to_string(file_size) +
|
||||
" is smaller than the lower limit: " + std::to_string(kMinFileSize) +
|
||||
".\n Please check file path: " + path +
|
||||
" and use 'FileWriter' to generate valid mindrecord files.");
|
||||
}
|
||||
fin.close();
|
||||
return Status::OK();
|
||||
}
|
||||
//在ShardHeader空间中创建ValidateHeader函数
|
||||
Status ShardHeader::ValidateHeader(const std::string &path, std::shared_ptr<json> *header_ptr) {
|
||||
RETURN_UNEXPECTED_IF_NULL(header_ptr);
|
||||
RETURN_IF_NOT_OK(CheckFileStatus(path));
|
||||
//
|
||||
auto realpath = FileUtils::GetRealPath(path.c_str());
|
||||
CHECK_FAIL_RETURN_UNEXPECTED(
|
||||
realpath.has_value(),
|
||||
"Invalid file, failed to get the realpath of mindrecord files. Please check file path: " + path);
|
||||
|
||||
// read header size读取标头大小
|
||||
json json_header;
|
||||
std::ifstream fin(realpath.value(), std::ios::in | std::ios::binary);
|
||||
CHECK_FAIL_RETURN_UNEXPECTED(fin.is_open(),
|
||||
"Invalid file, failed to open files for loading mindrecord files. Please check file "
|
||||
"path, permission and open file limit: " +
|
||||
path);
|
||||
//判断io_rea的相关参数，若符合则关闭fin，并返回错误信息
|
||||
uint64_t header_size = 0;
|
||||
auto &io_read = fin.read(reinterpret_cast<char *>(&header_size), kInt64Len);
|
||||
if (!io_read.good() || io_read.fail() || io_read.bad()) {
|
||||
fin.close();
|
||||
RETURN_STATUS_UNEXPECTED("[Internal ERROR] failed to read file, file path: " + path);
|
||||
}
|
||||
//判断header列表的大小是否大于header应有的大小
|
||||
//若符合则关闭fin并返回错误信息
|
||||
if (header_size > kMaxHeaderSize) {
|
||||
fin.close();
|
||||
RETURN_STATUS_UNEXPECTED(
|
||||
"Invalid file, the size of mindrecord file header is larger than the upper limit. \nPlease use 'FileWriter' to "
|
||||
"generate valid mindrecord files.");
|
||||
}
|
||||
|
||||
// read header content读取标题内容
|
||||
std::vector<uint8_t> header_content(header_size);
|
||||
auto &io_read_content = fin.read(reinterpret_cast<char *>(&header_content[0]), header_size);
|
||||
if (!io_read_content.good() || io_read_content.fail() || io_read_content.bad()) {
|
||||
fin.close();
|
||||
RETURN_STATUS_UNEXPECTED("[Internal ERROR] Failed to read file, file path: " + path);
|
||||
}
|
||||
|
||||
fin.close();
|
||||
std::string raw_header_content = std::string(header_content.begin(), header_content.end());
|
||||
// parse json content解析json内容
|
||||
try {
|
||||
json_header = json::parse(raw_header_content);
|
||||
} catch (json::parse_error &e) {
|
||||
RETURN_STATUS_UNEXPECTED("[Internal ERROR] Failed to parse the metadata in JSON format in the mindrecord files: " +
|
||||
std::string(e.what()));
|
||||
}
|
||||
*header_ptr = std::make_shared<json>(json_header);
|
||||
return Status::OK();
|
||||
}
|
||||
//引用ShardHeader空间中BuildSingleHeader函数
|
||||
//创建singleheader列表
|
||||
Status ShardHeader::BuildSingleHeader(const std::string &file_path, std::shared_ptr<json> *header_ptr) {
|
||||
RETURN_UNEXPECTED_IF_NULL(header_ptr);
|
||||
std::shared_ptr<json> raw_header;
|
||||
RETURN_IF_NOT_OK(ValidateHeader(file_path, &raw_header));
|
||||
uint64_t compression_size =
|
||||
raw_header->contains("compression_size") ? (*raw_header)["compression_size"].get<uint64_t>() : 0;
|
||||
json header = {{"shard_addresses", (*raw_header)["shard_addresses"]},
|
||||
{"header_size", (*raw_header)["header_size"]},
|
||||
{"page_size", (*raw_header)["page_size"]},
|
||||
{"compression_size", compression_size},
|
||||
{"index_fields", (*raw_header)["index_fields"]},
|
||||
{"blob_fields", (*raw_header)["schema"][0]["blob_fields"]},
|
||||
{"schema", (*raw_header)["schema"][0]["schema"]},
|
||||
{"version", (*raw_header)["version"]}};
|
||||
*header_ptr = std::make_shared<json>(header);
|
||||
return Status::OK();
|
||||
}
|
||||
//引用ShardHeader空间中BuildDataset函数
|
||||
//创建dataset列表
|
||||
Status ShardHeader::BuildDataset(const std::vector<std::string> &file_paths, bool load_dataset) {
|
||||
uint32_t thread_num = std::thread::hardware_concurrency();
|
||||
if (thread_num == 0) {
|
||||
thread_num = kThreadNumber;
|
||||
}
|
||||
uint32_t work_thread_num = 0;
|
||||
uint32_t shard_count = file_paths.size();
|
||||
int group_num = ceil(shard_count * 1.0 / thread_num);
|
||||
std::vector<std::thread> thread_set(thread_num);
|
||||
std::vector<json> headers(shard_count);
|
||||
for (uint32_t x = 0; x < thread_num; ++x) {
|
||||
int start_num = x * group_num;
|
||||
int end_num = ((x + 1) * group_num > shard_count) ? shard_count : (x + 1) * group_num;
|
||||
if (start_num >= end_num) {
|
||||
continue;
|
||||
}
|
||||
|
||||
thread_set[x] =
|
||||
std::thread(&ShardHeader::GetHeadersOneTask, this, start_num, end_num, std::ref(headers), file_paths);
|
||||
work_thread_num++;
|
||||
}
|
||||
|
||||
for (uint32_t x = 0; x < work_thread_num; ++x) {
|
||||
thread_set[x].join();
|
||||
}
|
||||
if (thread_status) {
|
||||
thread_status = false;
|
||||
RETURN_STATUS_UNEXPECTED("[Internal ERROR] Error raised in GetHeadersOneTask function.");
|
||||
}
|
||||
RETURN_IF_NOT_OK(InitializeHeader(headers, load_dataset));
|
||||
return Status::OK();
|
||||
}
|
||||
//引用ShardHeader空间中GetHeadersOneTask函数
|
||||
void ShardHeader::GetHeadersOneTask(int start, int end, std::vector<json> &headers,
|
||||
const vector<string> &realAddresses) {
|
||||
if (thread_status || end > realAddresses.size()) {
|
||||
return;
|
||||
}
|
||||
for (int x = start; x < end; ++x) {
|
||||
std::shared_ptr<json> header;
|
||||
auto status = ValidateHeader(realAddresses[x], &header);
|
||||
if (status.IsError()) {
|
||||
thread_status = true;
|
||||
return;
|
||||
}
|
||||
(*header)["shard_addresses"] = realAddresses;
|
||||
if (std::find(kSupportedVersion.begin(), kSupportedVersion.end(), (*header)["version"]) ==
|
||||
kSupportedVersion.end()) {
|
||||
MS_LOG(ERROR) << "Invalid file, the version of mindrecord files" << (*header)["version"].dump()
|
||||
<< " is not supported.\nPlease use 'FileWriter' to generate valid mindrecord files.";
|
||||
thread_status = true;
|
||||
return;
|
||||
}
|
||||
headers[x] = *header;
|
||||
}
|
||||
}
|
||||
//引用ShardHeader空间中InitByFiles函数
|
||||
Status ShardHeader::InitByFiles(const std::vector<std::string> &file_paths) {
|
||||
std::vector<std::string> file_names(file_paths.size());
|
||||
std::transform(file_paths.begin(), file_paths.end(), file_names.begin(), [](std::string fp) -> std::string {
|
||||
std::shared_ptr<std::string> fn;
|
||||
return GetFileName(fp, &fn).IsOk() ? *fn : "";
|
||||
});
|
||||
|
||||
shard_addresses_ = std::move(file_names);
|
||||
shard_count_ = file_paths.size();
|
||||
CHECK_FAIL_RETURN_UNEXPECTED(shard_count_ != 0 && (shard_count_ <= kMaxShardCount),
|
||||
"[Internal ERROR] 'shard_count_': " + std::to_string(shard_count_) +
|
||||
"is not in range (0, " + std::to_string(kMaxShardCount) + "].");
|
||||
pages_.resize(shard_count_);
|
||||
return Status::OK();
|
||||
}
|
||||
//引用ShardHeader空间中ParseIndexFields函数
|
||||
//进入循环遍历index_field列表，获得parsed_index_fields列表
|
||||
Status ShardHeader::ParseIndexFields(const json &index_fields) {
|
||||
std::vector<std::pair<uint64_t, std::string>> parsed_index_fields;
|
||||
for (auto &index_field : index_fields) {
|
||||
auto schema_id = index_field["schema_id"].get<uint64_t>();
|
||||
std::string field_name = index_field["index_field"].get<std::string>();
|
||||
std::pair<uint64_t, std::string> parsed_index_field(schema_id, field_name);
|
||||
parsed_index_fields.push_back(parsed_index_field);
|
||||
}
|
||||
RETURN_IF_NOT_OK(AddIndexFields(parsed_index_fields));
|
||||
return Status::OK();
|
||||
}
|
||||
//引用ShardHeader空间中ParsePage函数
|
||||
Status ShardHeader::ParsePage(const json &pages, int shard_index, bool load_dataset) {
|
||||
// set shard_index when load_dataset is false当load_dataset为false时设置shard_index
|
||||
CHECK_FAIL_RETURN_UNEXPECTED(shard_count_ <= kMaxFileCount,
|
||||
"Invalid file, the number of mindrecord files: " + std::to_string(shard_count_) +
|
||||
"is not in range (0, " + std::to_string(kMaxFileCount) +
|
||||
"].\nPlease use 'FileWriter' to generate fewer mindrecord files.");
|
||||
//判断pages_列表是否为空，若是则更改pages_列表的大小
|
||||
if (pages_.empty()) {
|
||||
pages_.resize(shard_count_);
|
||||
}
|
||||
//进入循环遍历pages_列表，设置pages_列表内容
|
||||
for (auto &page : pages) {
|
||||
int page_id = page["page_id"];
|
||||
int shard_id = page["shard_id"];
|
||||
std::string page_type = page["page_type"];
|
||||
int page_type_id = page["page_type_id"];
|
||||
auto start_row_id = page["start_row_id"].get<uint64_t>();
|
||||
auto end_row_id = page["end_row_id"].get<uint64_t>();
|
||||
|
||||
std::vector<std::pair<int, uint64_t>> row_group_ids(page["row_group_ids"].size());
|
||||
std::transform(page["row_group_ids"].begin(), page["row_group_ids"].end(), row_group_ids.begin(),
|
||||
[](json rg) { return std::make_pair(rg["id"], rg["offset"].get<uint64_t>()); });
|
||||
|
||||
auto page_size = page["page_size"].get<uint64_t>();
|
||||
|
||||
std::shared_ptr<Page> parsed_page = std::make_shared<Page>(page_id, shard_id, page_type, page_type_id, start_row_id,
|
||||
end_row_id, row_group_ids, page_size);
|
||||
if (load_dataset == true) {
|
||||
pages_[shard_id].push_back(std::move(parsed_page));
|
||||
} else {
|
||||
pages_[shard_index].push_back(std::move(parsed_page));
|
||||
}
|
||||
}
|
||||
return Status::OK();
|
||||
}
|
||||
//引用ShardHeader空间中ParseStatistics函数
|
||||
Status ShardHeader::ParseStatistics(const json &statistics) {
|
||||
//进入循环，遍历statistics列表，若有错误输出错误信息
|
||||
for (auto &statistic : statistics) {
|
||||
CHECK_FAIL_RETURN_UNEXPECTED(
|
||||
statistic.find("desc") != statistic.end() && statistic.find("statistics") != statistic.end(),
|
||||
"[Internal ERROR] Failed to deserialize statistics: " + statistics.dump());
|
||||
std::string statistic_description = statistic["desc"].get<std::string>();
|
||||
json statistic_body = statistic["statistics"];
|
||||
std::shared_ptr<Statistics> parsed_statistic = Statistics::Build(statistic_description, statistic_body);
|
||||
RETURN_UNEXPECTED_IF_NULL(parsed_statistic);
|
||||
AddStatistic(parsed_statistic);
|
||||
}
|
||||
return Status::OK();
|
||||
}
|
||||
//引用ShardHeader空间中ParseShardAddress函数
|
||||
Status ShardHeader::ParseSchema(const json &schemas) {
|
||||
for (auto &schema : schemas) {
|
||||
// change how we get schemaBody once design is finalized设计完成后，更改获取schemaBody的方式
|
||||
CHECK_FAIL_RETURN_UNEXPECTED(schema.find("desc") != schema.end() && schema.find("blob_fields") != schema.end() &&
|
||||
schema.find("schema") != schema.end(),
|
||||
"[Internal ERROR] Failed to deserialize schema: " + schema.dump());
|
||||
std::string schema_description = schema["desc"].get<std::string>();
|
||||
std::vector<std::string> blob_fields = schema["blob_fields"].get<std::vector<std::string>>();
|
||||
json schema_body = schema["schema"];
|
||||
std::shared_ptr<Schema> parsed_schema = Schema::Build(schema_description, schema_body);
|
||||
RETURN_UNEXPECTED_IF_NULL(parsed_schema);
|
||||
AddSchema(parsed_schema);
|
||||
}
|
||||
return Status::OK();
|
||||
}
|
||||
//引用ShardHeader空间中ParseShardAddress函数
|
||||
//复制address列表到shard_addresses_
|
||||
void ShardHeader::ParseShardAddress(const json &address) {
|
||||
std::copy(address.begin(), address.end(), std::back_inserter(shard_addresses_));
|
||||
}
|
||||
//引用ShardHeader空间中SerializeHeader函数
|
||||
std::vector<std::string> ShardHeader::SerializeHeader() {
|
||||
std::vector<std::string> header;
|
||||
auto index = SerializeIndexFields();
|
||||
auto stats = SerializeStatistics();
|
||||
auto schema = SerializeSchema();
|
||||
auto pages = SerializePage();
|
||||
auto address = SerializeShardAddress();
|
||||
//判断shard_count_是否符合条件，若符合则返回string类型
|
||||
if (shard_count_ > static_cast<int>(pages.size())) {
|
||||
return std::vector<string>{};
|
||||
}
|
||||
//判断shard_count_是否小于shard_count_最大值，若是则进入循环，编辑s字符串，最后返回字符串信息
|
||||
if (shard_count_ <= kMaxShardCount) {
|
||||
for (int shardId = 0; shardId < shard_count_; shardId++) {
|
||||
string s;
|
||||
s += "{\"header_size\":" + std::to_string(header_size_) + ",";
|
||||
s += "\"index_fields\":" + index + ",";
|
||||
s += "\"page\":" + pages[shardId] + ",";
|
||||
s += "\"page_size\":" + std::to_string(page_size_) + ",";
|
||||
s += "\"compression_size\":" + std::to_string(compression_size_) + ",";
|
||||
s += "\"schema\":" + schema + ",";
|
||||
s += "\"shard_addresses\":" + address + ",";
|
||||
s += "\"shard_id\":" + std::to_string(shardId) + ",";
|
||||
s += "\"statistics\":" + stats + ",";
|
||||
s += "\"version\":\"" + std::string(kVersion) + "\"";
|
||||
s += "}";
|
||||
header.emplace_back(s);
|
||||
}
|
||||
}
|
||||
return header;
|
||||
}
|
||||
//引用ShardHeader空间中SerializeIndexFields函数
|
||||
//遍历fields列表，返回schema_id、index_field的值
|
||||
std::string ShardHeader::SerializeIndexFields() {
|
||||
json j;
|
||||
auto fields = index_->GetFields();
|
||||
(void)std::transform(fields.begin(), fields.end(), std::back_inserter(j),
|
||||
[](const std::pair<uint64_t, std::string> &field) -> json {
|
||||
return {{"schema_id", field.first}, {"index_field", field.second}};
|
||||
});
|
||||
return j.dump();
|
||||
}
|
||||
//引用ShardHeader空间中SerializePage函数
|
||||
//遍历shard_pages列表，返回GetPage函数的值
|
||||
std::vector<std::string> ShardHeader::SerializePage() {
|
||||
std::vector<string> pages;
|
||||
for (auto &shard_pages : pages_) {
|
||||
json j;
|
||||
(void)std::transform(shard_pages.begin(), shard_pages.end(), std::back_inserter(j),
|
||||
[](const std::shared_ptr<Page> &p) { return p->GetPage(); });
|
||||
pages.emplace_back(j.dump());
|
||||
}
|
||||
return pages;
|
||||
}
|
||||
//引用ShardHeader空间中SerializeStatistics函数
|
||||
//遍历statistics_列表，返回GetStatistics函数的值
|
||||
std::string ShardHeader::SerializeStatistics() {
|
||||
json j;
|
||||
(void)std::transform(statistics_.begin(), statistics_.end(), std::back_inserter(j),
|
||||
[](const std::shared_ptr<Statistics> &stats) { return stats->GetStatistics(); });
|
||||
return j.dump();
|
||||
}
|
||||
//引用ShardHeader空间中SerializeSchema函数
|
||||
//遍历schema_列表，返回GetSchema函数的值
|
||||
std::string ShardHeader::SerializeSchema() {
|
||||
json j;
|
||||
(void)std::transform(schema_.begin(), schema_.end(), std::back_inserter(j),
|
||||
[](const std::shared_ptr<Schema> &schema) { return schema->GetSchema(); });
|
||||
return j.dump();
|
||||
}
|
||||
//引用ShardHeader空间中SerializeShardAddress函数
|
||||
//进入循环，一次根据地址获得文件名并判断是否为空
|
||||
std::string ShardHeader::SerializeShardAddress() {
|
||||
json j;
|
||||
std::shared_ptr<std::string> fn_ptr;
|
||||
for (const auto &addr : shard_addresses_) {
|
||||
(void)GetFileName(addr, &fn_ptr);
|
||||
(void)j.emplace_back(*fn_ptr);
|
||||
}
|
||||
return j.dump();
|
||||
}
|
||||
//引用ShardHeader空间中GetPage函数
|
||||
//判断shard_id、page_id是否在标准范围内，若是则获取pages_列表中的对应信息，若不是则返回错误信息
|
||||
Status ShardHeader::GetPage(const int &shard_id, const int &page_id, std::shared_ptr<Page> *page_ptr) {
|
||||
RETURN_UNEXPECTED_IF_NULL(page_ptr);
|
||||
if (shard_id < static_cast<int>(pages_.size()) && page_id < static_cast<int>(pages_[shard_id].size())) {
|
||||
*page_ptr = pages_[shard_id][page_id];
|
||||
return Status::OK();
|
||||
}
|
||||
page_ptr = nullptr;
|
||||
RETURN_STATUS_UNEXPECTED("[Internal ERROR] Failed to get Page, 'page_id': " + std::to_string(page_id));
|
||||
}
|
||||
//引用ShardHeader空间中SetPage函数
|
||||
//判断shard_id、page_id是否在标准范围内，若是则在pages_列表中加入新值，若不是则返回错误信息
|
||||
Status ShardHeader::SetPage(const std::shared_ptr<Page> &new_page) {
|
||||
int shard_id = new_page->GetShardID();
|
||||
int page_id = new_page->GetPageID();
|
||||
if (shard_id < static_cast<int>(pages_.size()) && page_id < static_cast<int>(pages_[shard_id].size())) {
|
||||
pages_[shard_id][page_id] = new_page;
|
||||
return Status::OK();
|
||||
}
|
||||
RETURN_STATUS_UNEXPECTED("[Internal ERROR] Failed to set Page, 'page_id': " + std::to_string(page_id));
|
||||
}
|
||||
//引用ShardHeader空间中AddPage函数
|
||||
//判断shard_id、page_id是否在标准范围内，若是则在pages_列表中添加page，若不是则返回错误信息
|
||||
Status ShardHeader::AddPage(const std::shared_ptr<Page> &new_page) {
|
||||
int shard_id = new_page->GetShardID();
|
||||
int page_id = new_page->GetPageID();
|
||||
if (shard_id < static_cast<int>(pages_.size()) && page_id == static_cast<int>(pages_[shard_id].size())) {
|
||||
pages_[shard_id].push_back(new_page);
|
||||
return Status::OK();
|
||||
}
|
||||
RETURN_STATUS_UNEXPECTED("[Internal ERROR] Failed to add Page, 'page_id': " + std::to_string(page_id));
|
||||
}
|
||||
//引用ShardHeader空间中GetLastPageId函数
|
||||
//判断shard_id是否在范围内，若是则返回0，若不是则返回pages_列表的大小-1的值
|
||||
int64_t ShardHeader::GetLastPageId(const int &shard_id) {
|
||||
if (shard_id >= static_cast<int>(pages_.size())) {
|
||||
return 0;
|
||||
}
|
||||
return pages_[shard_id].size() - 1;
|
||||
}
|
||||
//引用ShardHeader空间中GetLastPageIdByType函数
|
||||
int ShardHeader::GetLastPageIdByType(const int &shard_id, const std::string &page_type) {
|
||||
//判断shard_id是否在范围内，若是则返回0
|
||||
if (shard_id >= static_cast<int>(pages_.size())) {
|
||||
return 0;
|
||||
}
|
||||
int last_page_id = -1;
|
||||
//进入循环，若pages_列表中存在page_type则给last_page_id 赋值
|
||||
for (uint64_t i = pages_[shard_id].size(); i >= 1; i--) {
|
||||
if (pages_[shard_id][i - 1]->GetPageType() == page_type) {
|
||||
last_page_id = pages_[shard_id][i - 1]->GetPageID();
|
||||
return last_page_id;
|
||||
}
|
||||
}
|
||||
return last_page_id;
|
||||
}
|
||||
//引用ShardHeader空间中GetPageByGroupId函数
|
||||
//若发现错误则输出错误信息
|
||||
//进入循环，建立page_ptr链表
|
||||
Status ShardHeader::GetPageByGroupId(const int &group_id, const int &shard_id, std::shared_ptr<Page> *page_ptr) {
|
||||
RETURN_UNEXPECTED_IF_NULL(page_ptr);
|
||||
CHECK_FAIL_RETURN_UNEXPECTED(shard_id < static_cast<int>(pages_.size()),
|
||||
"[Internal ERROR] 'shard_id': " + std::to_string(shard_id) +
|
||||
" should be smaller than the size of 'pages_': " + std::to_string(pages_.size()) +
|
||||
".");
|
||||
for (uint64_t i = pages_[shard_id].size(); i >= 1; i--) {
|
||||
auto page = pages_[shard_id][i - 1];
|
||||
if (page->GetPageType() == kPageTypeBlob && page->GetPageTypeID() == group_id) {
|
||||
*page_ptr = std::make_shared<Page>(*page);
|
||||
return Status::OK();
|
||||
}
|
||||
}
|
||||
page_ptr = nullptr;
|
||||
RETURN_STATUS_UNEXPECTED("[Internal ERROR] Failed to get Page, 'group_id': " + std::to_string(group_id));
|
||||
}
|
||||
//引用ShardHeader空间中AddSchema函数
|
||||
int ShardHeader::AddSchema(std::shared_ptr<Schema> schema) {
|
||||
//判断schema列表是否为空，若是则返回错误信息
|
||||
if (schema == nullptr) {
|
||||
MS_LOG(ERROR) << "[Internal ERROR] The pointer of schema is NULL.";
|
||||
return -1;
|
||||
}
|
||||
//继续判断schema空间中的存储空间是否为空，若是则返回错误信息
|
||||
if (!schema_.empty()) {
|
||||
MS_LOG(ERROR) << "The schema is added repeatedly. Please remove the redundant 'add_schema' function.";
|
||||
return -1;
|
||||
}
|
||||
//判断schema_id是否为-1，若是则给schema_id再赋值，后加入到schema列表
|
||||
int64_t schema_id = schema->GetSchemaID();
|
||||
if (schema_id == -1) {
|
||||
schema_id = schema_.size();
|
||||
schema->SetSchemaID(schema_id);
|
||||
}
|
||||
schema_.push_back(schema);
|
||||
return schema_id;
|
||||
}
|
||||
//引用ShardHeader空间中AddStatistic函数
|
||||
//判断statistic是否为真，若是则对statistic_id赋值，并将值列入statistic列表
|
||||
void ShardHeader::AddStatistic(std::shared_ptr<Statistics> statistic) {
|
||||
if (statistic) {
|
||||
int64_t statistics_id = statistic->GetStatisticsID();
|
||||
if (statistics_id == -1) {
|
||||
statistics_id = statistics_.size();
|
||||
statistic->SetStatisticsID(statistics_id);
|
||||
}
|
||||
statistics_.push_back(statistic);
|
||||
}
|
||||
}
|
||||
//引用ShardHeader空间中InitIndexPtr函数
|
||||
//判断index_是否为真，若是则进行操作
|
||||
std::shared_ptr<Index> ShardHeader::InitIndexPtr() {
|
||||
std::shared_ptr<Index> index = index_;
|
||||
if (!index_) {
|
||||
index = std::make_shared<Index>();
|
||||
index_ = index;
|
||||
}
|
||||
return index;
|
||||
}
|
||||
//引用ShardHeader空间中CheckIndexField函数,输入参数n调用指定函数
|
||||
Status ShardHeader::CheckIndexField(const std::string &field, const json &schema) {
|
||||
// check field name is or is not valid检查字段名称无效
|
||||
CHECK_FAIL_RETURN_UNEXPECTED(schema.find(field) != schema.end(),
|
||||
"Invalid input, 'index_fields': " + field + " can not found in schema: " +
|
||||
schema.dump() + ".\n Please use 'add_index' function to add proper 'index_fields'.");
|
||||
CHECK_FAIL_RETURN_UNEXPECTED(schema[field]["type"] != "Bytes",
|
||||
"Invalid input, type of 'index_fields': " + field +
|
||||
" is bytes and can not set as an 'index_fields'.\n Please use 'add_index' function to "
|
||||
"add the other 'index_fields'.");
|
||||
CHECK_FAIL_RETURN_UNEXPECTED(schema.find(field) == schema.end() || schema[field].find("shape") == schema[field].end(),
|
||||
"Invalid input, type of 'index_fields': " + field +
|
||||
" is array and can not set as an 'index_fields'.\n Please use 'add_index' function to "
|
||||
"add the other 'index_fields'.");
|
||||
return Status::OK();
|
||||
}
|
||||
//引用ShardHeader空间中AddIndexFields函数,输入参数n调用指定函数
|
||||
Status ShardHeader::AddIndexFields(const std::vector<std::string> &fields) {
|
||||
//判断fields是否为空
|
||||
if (fields.empty()) {
|
||||
return Status::OK();
|
||||
}
|
||||
CHECK_FAIL_RETURN_UNEXPECTED(!GetSchemas().empty(),
|
||||
"Invalid data, schema is empty. Please use 'add_schema' function to add schema first.");
|
||||
// create index Object创建索引对象
|
||||
std::shared_ptr<Index> index = InitIndexPtr();
|
||||
for (const auto &schemaPtr : schema_) {
|
||||
std::shared_ptr<Schema> schema_ptr;
|
||||
RETURN_IF_NOT_OK(GetSchemaByID(schemaPtr->GetSchemaID(), &schema_ptr));
|
||||
json schema = schema_ptr->GetSchema().at("schema");
|
||||
// checkout and add fields for each schema签出并为每个架构添加字段
|
||||
std::set<std::string> field_set;
|
||||
for (const auto &item : index->GetFields()) {
|
||||
field_set.insert(item.second);
|
||||
}
|
||||
for (const auto &field : fields) {
|
||||
CHECK_FAIL_RETURN_UNEXPECTED(
|
||||
field_set.find(field) == field_set.end(),
|
||||
"The 'index_fields': " + field + " is added repeatedly. Please remove the redundant 'add_index' function.");
|
||||
// check field name is or is not valid检查字段名称无效
|
||||
RETURN_IF_NOT_OK(CheckIndexField(field, schema));
|
||||
field_set.insert(field);
|
||||
// add field into index将字段添加到索引中
|
||||
index.get()->AddIndexField(schemaPtr->GetSchemaID(), field);
|
||||
}
|
||||
}
|
||||
index_ = index;
|
||||
return Status::OK();
|
||||
}
|
||||
//引用ShardHeader空间中GetAllSchemaID函数,输入参数n调用指定函数
|
||||
Status ShardHeader::GetAllSchemaID(std::set<uint64_t> &bucket_count) {
|
||||
// get all schema id获取所有架构id
|
||||
for (const auto &schema : schema_) {
|
||||
auto schema_id = schema->GetSchemaID();
|
||||
CHECK_FAIL_RETURN_UNEXPECTED(bucket_count.find(schema_id) == bucket_count.end(),
|
||||
"[Internal ERROR] duplicate schema exist, schema id: " + std::to_string(schema_id));
|
||||
bucket_count.insert(schema_id);
|
||||
}
|
||||
return Status::OK();
|
||||
}
|
||||
//引用ShardHeader空间中AddIndexFields函数,输入参数n调用指定函数
|
||||
Status ShardHeader::AddIndexFields(std::vector<std::pair<uint64_t, std::string>> fields) {
|
||||
//判断fields是否为空
|
||||
if (fields.empty()) {
|
||||
return Status::OK();
|
||||
}
|
||||
// create index Object创建索引对象
|
||||
std::shared_ptr<Index> index = InitIndexPtr();
|
||||
// get all schema id获取所有架构id
|
||||
std::set<uint64_t> bucket_count;
|
||||
RETURN_IF_NOT_OK(GetAllSchemaID(bucket_count));
|
||||
// check and add fields for each schema检查并添加每个模式的字段
|
||||
std::set<std::pair<uint64_t, std::string>> field_set;
|
||||
for (const auto &item : index->GetFields()) {
|
||||
field_set.insert(item);
|
||||
}
|
||||
for (const auto &field : fields) {
|
||||
CHECK_FAIL_RETURN_UNEXPECTED(field_set.find(field) == field_set.end(),
|
||||
"The 'index_fields': " + field.second +
|
||||
" is added repeatedly. Please remove the redundant 'add_index' function.");
|
||||
uint64_t schema_id = field.first;
|
||||
std::string field_name = field.second;
|
||||
|
||||
// check schemaId is or is not valid check schemaId无效
|
||||
CHECK_FAIL_RETURN_UNEXPECTED(bucket_count.find(schema_id) != bucket_count.end(),
|
||||
"[Internal ERROR] 'schema_id': " + std::to_string(schema_id) + " can not found.");
|
||||
// check field name is or is not valid check field name无效
|
||||
std::shared_ptr<Schema> schema_ptr;
|
||||
RETURN_IF_NOT_OK(GetSchemaByID(schema_id, &schema_ptr));
|
||||
json schema = schema_ptr->GetSchema().at("schema");
|
||||
CHECK_FAIL_RETURN_UNEXPECTED(schema.find(field_name) != schema.end(),
|
||||
"Invalid input, 'index_fields': " + field_name + " can not found in schema: " +
|
||||
schema.dump() + ".\n Please use 'add_index' function to add proper 'index_fields'.");
|
||||
RETURN_IF_NOT_OK(CheckIndexField(field_name, schema));
|
||||
field_set.insert(field);
|
||||
// add field into index将字段添加到索引中
|
||||
index->AddIndexField(schema_id, field_name);
|
||||
}
|
||||
index_ = index;
|
||||
return Status::OK();
|
||||
}
|
||||
//引用ShardHeader空间中GetShardAddressByID函数,输入参数n调用指定函数
|
||||
//判断shard_id是否大于shard_addresses_列表的大小，若是则返回“”，若不是则返回shard_addresses_列表中at函数的返回值
|
||||
std::string ShardHeader::GetShardAddressByID(int64_t shard_id) {
|
||||
if (shard_id >= shard_addresses_.size()) {
|
||||
return "";
|
||||
}
|
||||
return shard_addresses_.at(shard_id);
|
||||
}
|
||||
//引用ShardHeader空间中GetSchemas函数,返回schema_的值
|
||||
std::vector<std::shared_ptr<Schema>> ShardHeader::GetSchemas() { return schema_; }
|
||||
//引用ShardHeader空间中GetStatistics函数,返回statistics_的值
|
||||
std::vector<std::shared_ptr<Statistics>> ShardHeader::GetStatistics() { return statistics_; }
|
||||
//引用ShardHeader空间中GetFields函数,返回index_中GetFields的函数返回值
|
||||
std::vector<std::pair<uint64_t, std::string>> ShardHeader::GetFields() { return index_->GetFields(); }
|
||||
//引用ShardHeader空间中GetIndex函数,返回index_的值
|
||||
std::shared_ptr<Index> ShardHeader::GetIndex() { return index_; }
|
||||
//引用ShardHeader空间中GetSchemaByID函数
|
||||
//判断schema列表中id是否在标准范围内
|
||||
Status ShardHeader::GetSchemaByID(int64_t schema_id, std::shared_ptr<Schema> *schema_ptr) {
|
||||
RETURN_UNEXPECTED_IF_NULL(schema_ptr);
|
||||
int64_t schema_size = schema_.size();
|
||||
CHECK_FAIL_RETURN_UNEXPECTED(schema_id >= 0 && schema_id < schema_size,
|
||||
"[Internal ERROR] 'schema_id': " + std::to_string(schema_id) + " is not in range [0, " +
|
||||
std::to_string(schema_size) + ").");
|
||||
*schema_ptr = schema_.at(schema_id);
|
||||
return Status::OK();
|
||||
}
|
||||
//引用ShardHeader空间中GetStatisticByID函数
|
||||
//判断statistic列表中id的值是否在标准范围内
|
||||
Status ShardHeader::GetStatisticByID(int64_t statistic_id, std::shared_ptr<Statistics> *statistics_ptr) {
|
||||
RETURN_UNEXPECTED_IF_NULL(statistics_ptr);
|
||||
int64_t statistics_size = statistics_.size();
|
||||
CHECK_FAIL_RETURN_UNEXPECTED(statistic_id >= 0 && statistic_id < statistics_size,
|
||||
"[Internal ERROR] 'statistic_id': " + std::to_string(statistic_id) +
|
||||
" is not in range [0, " + std::to_string(statistics_size) + ").");
|
||||
*statistics_ptr = statistics_.at(statistic_id);
|
||||
return Status::OK();
|
||||
}
|
||||
//在ShardHeader空间中创建PagesToFile函数
|
||||
Status ShardHeader::PagesToFile(const std::string dump_file_name) {
|
||||
auto realpath = FileUtils::GetRealPath(dump_file_name.c_str());
|
||||
CHECK_FAIL_RETURN_UNEXPECTED(realpath.has_value(),
|
||||
"[Internal ERROR] Failed to get the realpath of Pages file, path: " + dump_file_name);
|
||||
// write header content to file, dump whatever is in the file before将头内容写入文件，转储之前文件中的任何内容
|
||||
std::ofstream page_out_handle(realpath.value(), std::ios_base::trunc | std::ios_base::out);
|
||||
CHECK_FAIL_RETURN_UNEXPECTED(page_out_handle.good(),
|
||||
"[Internal ERROR] Failed to open Pages file, path: " + dump_file_name);
|
||||
auto pages = SerializePage();
|
||||
for (const auto &shard_pages : pages) {
|
||||
page_out_handle << shard_pages << "\n";
|
||||
}
|
||||
page_out_handle.close();
|
||||
return Status::OK();
|
||||
}
|
||||
//在ShardHeader空间中创建FileToPages函数
|
||||
Status ShardHeader::FileToPages(const std::string dump_file_name) {
|
||||
for (auto &v : pages_) { // clean pages清理页面
|
||||
v.clear();
|
||||
}
|
||||
auto realpath = FileUtils::GetRealPath(dump_file_name.c_str());
|
||||
CHECK_FAIL_RETURN_UNEXPECTED(realpath.has_value(),
|
||||
"[Internal ERROR] Failed to get the realpath of Pages file, path: " + dump_file_name);
|
||||
// attempt to open the file contains the page in json试图打开包含json页面的文件
|
||||
std::ifstream page_in_handle(realpath.value());
|
||||
CHECK_FAIL_RETURN_UNEXPECTED(page_in_handle.good(),
|
||||
"[Internal ERROR] Pages file does not exist, path: " + dump_file_name);
|
||||
std::string line;
|
||||
while (std::getline(page_in_handle, line)) {
|
||||
RETURN_IF_NOT_OK(ParsePage(json::parse(line), -1, true));
|
||||
}
|
||||
page_in_handle.close();
|
||||
return Status::OK();
|
||||
}
|
||||
//在ShardHeader空间中创建Initialize函数
|
||||
Status ShardHeader::Initialize(const std::shared_ptr<ShardHeader> *header_ptr, const json &schema,
|
||||
const std::vector<std::string> &index_fields, std::vector<std::string> &blob_fields,
|
||||
uint64_t &schema_id) {
|
||||
RETURN_UNEXPECTED_IF_NULL(header_ptr);
|
||||
auto schema_ptr = Schema::Build("mindrecord", schema);
|
||||
CHECK_FAIL_RETURN_UNEXPECTED(schema_ptr != nullptr, "[Internal ERROR] Failed to build schema: " + schema.dump() +
|
||||
"." + "Check the [ERROR] logs before for more details.");
|
||||
schema_id = (*header_ptr)->AddSchema(schema_ptr);
|
||||
// create index创建索引
|
||||
std::vector<std::pair<uint64_t, std::string>> id_index_fields;
|
||||
if (!index_fields.empty()) {
|
||||
(void)transform(index_fields.begin(), index_fields.end(), std::back_inserter(id_index_fields),
|
||||
[schema_id](const std::string &el) { return std::make_pair(schema_id, el); });
|
||||
RETURN_IF_NOT_OK((*header_ptr)->AddIndexFields(id_index_fields));
|
||||
}
|
||||
|
||||
auto build_schema_ptr = (*header_ptr)->GetSchemas()[0];
|
||||
blob_fields = build_schema_ptr->GetBlobFields();
|
||||
return Status::OK();
|
||||
}
|
||||
} // namespace mindrecord
|
||||
} // namespace mindspore
|
||||
|
|
@ -0,0 +1,33 @@
|
|||
/**
|
||||
* Copyright 2019 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#include "minddata/mindrecord/include/shard_index.h"//按照路径寻找以下文件，导入到本文件
|
||||
|
||||
namespace mindspore {//创建名为mindspore的空间
|
||||
namespace mindrecord {//创建名为mindrecord的空间
|
||||
// table name for index索引的表名
|
||||
const char TABLENAME[] = "index_table";
|
||||
//调用Index类中Index函数给成员变量database_name_、table_name_赋值
|
||||
Index::Index() : database_name_(""), table_name_(TABLENAME) {}
|
||||
//创建Index空间下void型AddIndexField函数,调用fields_类中emplace_back函数
|
||||
void Index::AddIndexField(const int64_t &schemaId, const std::string &field) {
|
||||
fields_.emplace_back(pair<int64_t, string>(schemaId, field));
|
||||
}
|
||||
|
||||
// Get attribute list获取属性列表
|
||||
std::vector<std::pair<uint64_t, std::string>> Index::GetFields() { return fields_; }
|
||||
} // namespace mindrecord
|
||||
} // namespace mindspore
|
||||
|
|
@ -0,0 +1,58 @@
|
|||
/**
|
||||
* Copyright 2019 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#include "minddata/mindrecord/include/shard_page.h"//按照路径寻找以下文件，导入到本文件,以下同理
|
||||
#include "pybind11/pybind11.h"
|
||||
|
||||
namespace mindspore {//创建名为mindspore的空间
|
||||
namespace mindrecord {//创建名为mindrecord的空间
|
||||
//调用json Page类下GetPage函数,进行变量初始化
|
||||
//判断函数row_group_ids_.size()的返回值是否为0，若是则将row_groups数组中"id"和"offset"的值设为0，若不是则分别设为rg.first和rg.second
|
||||
//返回str_page
|
||||
json Page::GetPage() const {
|
||||
json str_page;
|
||||
str_page["page_id"] = page_id_;
|
||||
str_page["shard_id"] = shard_id_;
|
||||
str_page["page_type"] = page_type_;
|
||||
str_page["page_type_id"] = page_type_id_;
|
||||
str_page["start_row_id"] = start_row_id_;
|
||||
str_page["end_row_id"] = end_row_id_;
|
||||
if (row_group_ids_.size() == 0) {
|
||||
json row_groups = json({});
|
||||
row_groups["id"] = 0;
|
||||
row_groups["offset"] = 0;
|
||||
str_page["row_group_ids"].push_back(row_groups);
|
||||
} else {
|
||||
for (const auto &rg : row_group_ids_) {
|
||||
json row_groups = json({});
|
||||
row_groups["id"] = rg.first;
|
||||
row_groups["offset"] = rg.second;
|
||||
str_page["row_group_ids"].push_back(row_groups);
|
||||
}
|
||||
}
|
||||
str_page["page_size"] = page_size_;
|
||||
return str_page;
|
||||
}
|
||||
//调用Page类下void型DeleteLastGroupId函数
|
||||
//判断row_group_ids_.empty()函数的返回值是否为0，若是则修改page_size_的值为row_group_ids_.back().second，调用row_group_ids_.pop_back函数
|
||||
void Page::DeleteLastGroupId() {
|
||||
if (!row_group_ids_.empty()) {
|
||||
page_size_ = row_group_ids_.back().second;
|
||||
row_group_ids_.pop_back();
|
||||
}
|
||||
}
|
||||
} // namespace mindrecord
|
||||
} // namespace mindspore
|
||||
|
|
@ -0,0 +1,53 @@
|
|||
/**
|
||||
* Copyright 2019-2021 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#include "minddata/mindrecord/include/shard_pk_sample.h"//按照路径寻找以下文件，导入到本文件
|
||||
|
||||
using mindspore::LogStream;//声明mindspore空间下的LogStream
|
||||
using mindspore::ExceptionType::NoExceptionType;//声明mindspore空间下ExceptionType类中的NoExceptionType
|
||||
using mindspore::MsLogLevel::ERROR;//声明mindspore空间下MsLogLevel类中的ERROR
|
||||
|
||||
namespace mindspore {//创建名为mindspore的空间
|
||||
namespace mindrecord {//创建名为mindrecord的空间
|
||||
//引用ShardPkSample空间中ShardPkSample函数,输入三种参数调用指定函数
|
||||
//对参数ShardCategory、shuffle_、num_samples_进行进行参数初始化
|
||||
ShardPkSample::ShardPkSample(const std::string &category_field, int64_t num_elements, int64_t num_samples)
|
||||
: ShardCategory(category_field, num_elements, std::numeric_limits<int64_t>::max(), true),
|
||||
shuffle_(false),
|
||||
num_samples_(num_samples) {}
|
||||
//引用ShardPkSample空间中ShardPkSample函数,输入四种参数调用指定函数
|
||||
//对参数ShardCategory进行进行参数初始化
|
||||
ShardPkSample::ShardPkSample(const std::string &category_field, int64_t num_elements, int64_t num_categories,
|
||||
int64_t num_samples)
|
||||
: ShardCategory(category_field, num_elements, num_categories, true), shuffle_(false), num_samples_(num_samples) {}
|
||||
//引用ShardPkSample空间中ShardPkSample函数,输入四种参数调用指定函数
|
||||
//对参数ShardCategory、shuffle_op_进行进行参数初始化
|
||||
ShardPkSample::ShardPkSample(const std::string &category_field, int64_t num_elements, int64_t num_categories,
|
||||
uint32_t seed, int64_t num_samples)
|
||||
: ShardCategory(category_field, num_elements, num_categories, true), shuffle_(true), num_samples_(num_samples) {
|
||||
shuffle_op_ = std::make_shared<ShardShuffle>(seed, kShuffleSample); // 进行重新排序和替换
|
||||
}
|
||||
//在ShardDistributedSample空间中创建Status型PreExecute函数,返回值为Status变量
|
||||
//判断shuffle_是否为真,若是则调用RETURN_IF_NOT_OK函数
|
||||
//返回Status中Ok函数的返回值
|
||||
Status ShardPkSample::SufExecute(ShardTaskList &tasks) {
|
||||
if (shuffle_ == true) {
|
||||
RETURN_IF_NOT_OK((*shuffle_op_)(tasks));
|
||||
}
|
||||
return Status::OK();
|
||||
}
|
||||
} // namespace mindrecord
|
||||
} // namespace mindspore
|
||||
|
|
@ -0,0 +1,203 @@
|
|||
/**
|
||||
* Copyright 2019-2021 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#include "minddata/mindrecord/include/shard_sample.h"//按照路径寻找以下文件，导入到本文件
|
||||
|
||||
using mindspore::LogStream;//声明mindspore空间下的LogStream
|
||||
using mindspore::ExceptionType::NoExceptionType;//声明mindspore空间下ExceptionType类中的NoExceptionType
|
||||
using mindspore::MsLogLevel::ERROR;//声明mindspore空间下MsLogLevel类中的ERROR
|
||||
|
||||
namespace mindspore {//创建名为mindspore的空间
|
||||
namespace mindrecord {//创建名为mindrecord的空间
|
||||
//引用ShardSample空间中ShardSample函数,输入参数n调用指定函数
|
||||
//对参数numerator_、denominator_、partition_id_、no_of_samples_、indices_、sampler_type_、offset_进行参数初始化
|
||||
ShardSample::ShardSample(int64_t n)
|
||||
: numerator_(0),
|
||||
denominator_(0),
|
||||
partition_id_(0),
|
||||
no_of_samples_(n),
|
||||
indices_({}),
|
||||
sampler_type_(kCustomTopNSampler),
|
||||
offset_(-1) {}
|
||||
//引用ShardSample空间中ShardSample函数,输入两种参数调用指定函数
|
||||
//对参数numerator_、denominator_、partition_id_、no_of_samples_、indices_、sampler_type_、offset_进行参数初始化
|
||||
ShardSample::ShardSample(int64_t num, int64_t den)
|
||||
: numerator_(num),
|
||||
denominator_(den),
|
||||
partition_id_(0),
|
||||
no_of_samples_(0),
|
||||
indices_({}),
|
||||
sampler_type_(kCustomTopPercentSampler),
|
||||
offset_(-1) {}
|
||||
//引用ShardSample空间中ShardSample函数,输入五种参数调用指定函数
|
||||
//对参数numerator_、denominator_、partition_id_、no_of_samples_、indices_、sampler_type_、offset_进行参数初始化
|
||||
ShardSample::ShardSample(int64_t num, int64_t den, int64_t par, int64_t no_of_samples, int64_t offset)
|
||||
: numerator_(num),
|
||||
denominator_(den),
|
||||
partition_id_(par),
|
||||
no_of_samples_(no_of_samples),
|
||||
indices_({}),
|
||||
sampler_type_(kCustomTopPercentSampler),
|
||||
offset_(offset) {}
|
||||
//引用ShardSample空间中ShardSample函数,输入参数indices调用指定函数
|
||||
//对参数numerator_、denominator_、partition_id_、no_of_samples_、indices_、sampler_type_、offset_进行参数初始化
|
||||
ShardSample::ShardSample(const std::vector<int64_t> &indices)
|
||||
: numerator_(0),
|
||||
denominator_(0),
|
||||
partition_id_(0),
|
||||
no_of_samples_(0),
|
||||
indices_(indices),
|
||||
sampler_type_(kSubsetSampler) {}
|
||||
//引用ShardSample空间中ShardSample函数,输入参数indices调用指定函数，此函数继承ShardSample函数
|
||||
//对参数sampler_type_、shuffle_op_进行参数初始化
|
||||
ShardSample::ShardSample(const std::vector<int64_t> &indices, uint32_t seed) : ShardSample(indices) {
|
||||
sampler_type_ = kSubsetRandomSampler;
|
||||
shuffle_op_ = std::make_shared<ShardShuffle>(seed);
|
||||
}
|
||||
//在ShardCategory空间中创建int64_t型GetNumSamples函数,返回值为int64_t型参数
|
||||
//判断sampler_type_与kCustomTopNSampler是否赋值相同，若是则返回no_of_samples_
|
||||
//判断sampler_type_与kCustomTopPercentSampler是否复制相同，若是则再次判断dataset_size与denominator_的模是否为0，若是则返回dataset_size与denominator_和numerator_的乘积的商
|
||||
//若不是则返回dataset_size与denominator_和numerator_的乘积的商+1的值
|
||||
//判断sampler_type_与kSubsetRandomSampler的值或者sampler_type_与kSubsetSampler的值是否相等，若任一等式成立则返回indices_.size函数的返回值
|
||||
//若均不符合条件，最后返回0
|
||||
int64_t ShardSample::GetNumSamples(int64_t dataset_size, int64_t num_classes) {
|
||||
if (sampler_type_ == kCustomTopNSampler) {
|
||||
return no_of_samples_;
|
||||
}
|
||||
|
||||
if (sampler_type_ == kCustomTopPercentSampler) {
|
||||
if (dataset_size % denominator_ == 0) {
|
||||
return dataset_size / denominator_ * numerator_;
|
||||
} else {
|
||||
return dataset_size / denominator_ * numerator_ + 1;
|
||||
}
|
||||
}
|
||||
if (sampler_type_ == kSubsetRandomSampler || sampler_type_ == kSubsetSampler) {
|
||||
return indices_.size();
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
//在ShardDistributedSample空间中创建Status型PreExecute函数,返回值为Status变量
|
||||
//判断tasks.permutation_.empty函数的返回值是否为真，若是则将tasks.sample_ids_.size()的返回值赋给total_no并调用CHECK_FAIL_RETURN_UNEXPECTED函数
|
||||
//判断sampler_type_与kSubsetRandomSampler的值或者sampler_type_与kSubsetSampler的值是否相等，若任一等式成立则进入循环，修改index的赋值并调用new_tasks.AssignTask函数
|
||||
//若两等式均不成立，则判断nums_per_shard_.empty函数的返回值是否为真，若是则进入循环，调用new_tasks.AssignTask函数并累加count变量，直至no_of_samples_不等于0且count与no_of_samples_相等均成立时跳出循环
|
||||
//若不是，则进入另一循环，循环次数与上式不同，循环运行内容相同
|
||||
//若tasks.permutation_.empty函数的返回值为假，则将tasks.sample_ids_.size()的返回值赋给total_no并调用CHECK_FAIL_RETURN_UNEXPECTED函数，直接进入执行相同操作的循环，次数不同
|
||||
//在输出返回值前调用ShardTaskList类中的TaskListSwap函数
|
||||
//返回Status中Ok函数的返回值
|
||||
Status ShardSample::UpdateTasks(ShardTaskList &tasks, int64_t taking) {
|
||||
if (tasks.permutation_.empty()) {
|
||||
ShardTaskList new_tasks;
|
||||
auto total_no = tasks.sample_ids_.size();
|
||||
CHECK_FAIL_RETURN_UNEXPECTED(total_no > 0,
|
||||
"[Internal ERROR] 'total_no' should be positive but got: " + std::to_string(total_no));
|
||||
if (sampler_type_ == kSubsetRandomSampler || sampler_type_ == kSubsetSampler) {
|
||||
for (int64_t i = 0; i < indices_.size(); ++i) {
|
||||
int64_t index = ((indices_[i] % total_no) + total_no) % total_no;
|
||||
new_tasks.AssignTask(tasks, index); // different mod result between c and python c和python有不同mod结果
|
||||
}
|
||||
} else {
|
||||
int64_t count = 0;
|
||||
if (nums_per_shard_.empty()) {
|
||||
for (int64_t i = partition_id_ * taking; i < (partition_id_ + 1) * taking; i++) {
|
||||
if (no_of_samples_ != 0 && count == no_of_samples_) break;
|
||||
new_tasks.AssignTask(tasks, i % total_no); // rounding up. if overflow, go back to start四舍五入。如果溢出，返回开始
|
||||
count++;
|
||||
}
|
||||
} else {
|
||||
// Get samples within a specific range获取特定范围内的样本
|
||||
int64_t i = partition_id_ - 1 >= 0 ? nums_per_shard_[partition_id_ - 1] : 0;
|
||||
for (; i < nums_per_shard_[partition_id_]; i++) {
|
||||
if (no_of_samples_ != 0 && count == no_of_samples_) break;
|
||||
new_tasks.AssignTask(tasks, i % total_no);
|
||||
count++;
|
||||
}
|
||||
}
|
||||
}
|
||||
ShardTaskList::TaskListSwap(tasks, new_tasks);
|
||||
} else {
|
||||
ShardTaskList new_tasks;
|
||||
int64_t total_no = tasks.permutation_.size();
|
||||
CHECK_FAIL_RETURN_UNEXPECTED(total_no > 0,
|
||||
"[Internal ERROR] 'total_no' should be positive but got: " + std::to_string(total_no));
|
||||
int64_t cnt = 0;
|
||||
for (int64_t i = partition_id_ * taking; i < (partition_id_ + 1) * taking; i++) {
|
||||
if (no_of_samples_ != 0 && cnt == no_of_samples_) break;
|
||||
new_tasks.AssignTask(tasks, tasks.permutation_[i % total_no]);
|
||||
cnt++;
|
||||
}
|
||||
ShardTaskList::TaskListSwap(tasks, new_tasks);
|
||||
}
|
||||
return Status::OK();
|
||||
}
|
||||
//在ShardDistributedSample空间中创建Status型PreExecute函数,返回值为Status变量
|
||||
//判断offset_是否为-1，若不是则进入循环，对samples_per_buffer_、remainder进行赋值并对remainder、offset_进行判断后调整samples_per_buffer_的赋值，最终调用nums_per_shard_.push_back函数
|
||||
//判断sampler_type_与kCustomTopNSampler是否相等，若相等则对no_of_samples_、taking = no_of_samples_赋值
|
||||
//若不相等则判断sampler_type_ 与kSubsetRandomSampler是否相等或sampler_type_与kSubsetSampler是否相等，其中任一成立即调用CHECK_FAIL_RETURN_UNEXPECTED函数并输出非法输入警告
|
||||
//若均不符合则判断numerator_、denominator_是否均大于0且numerator_是否小于等于denominator_ ，若是则继续判断numerator_是否等于1且denominator_是否大于1，若是则对taking赋值
|
||||
//若不是则对taking进行其他操作的赋值
|
||||
//若不符合第一条件，则调用RETURN_STATUS_UNEXPECTED函数输出标准变量不符合警告
|
||||
//返回UpdateTasks函数的返回值
|
||||
Status ShardSample::Execute(ShardTaskList &tasks) {
|
||||
if (offset_ != -1) {
|
||||
int64_t old_v = 0;
|
||||
int64_t num_rows_ = tasks.sample_ids_.size();
|
||||
for (int64_t x = 0; x < denominator_; x++) {
|
||||
int64_t samples_per_buffer_ = (num_rows_ + offset_) / denominator_;
|
||||
int64_t remainder = (num_rows_ + offset_) % denominator_;
|
||||
if (x < remainder) samples_per_buffer_++;
|
||||
if (x < offset_) samples_per_buffer_--;
|
||||
old_v += samples_per_buffer_;
|
||||
// nums_per_shard_ is used to save the current shard's ending index nums_per_shard用于保存当前碎片的结束索引
|
||||
nums_per_shard_.push_back(old_v);
|
||||
}
|
||||
}
|
||||
int no_of_categories = static_cast<int>(tasks.categories);
|
||||
int64_t total_no = tasks.sample_ids_.size();
|
||||
int64_t taking = 0;
|
||||
if (sampler_type_ == kCustomTopNSampler) { // non sharding case constructor #1非分片case构造函数#1
|
||||
no_of_samples_ = std::min(no_of_samples_, total_no);
|
||||
taking = no_of_samples_ - no_of_samples_ % no_of_categories;
|
||||
} else if (sampler_type_ == kSubsetRandomSampler || sampler_type_ == kSubsetSampler) {
|
||||
CHECK_FAIL_RETURN_UNEXPECTED(static_cast<int64_t>(indices_.size()) <= total_no,
|
||||
"Invalid input, indices size: " + std::to_string(indices_.size()) +
|
||||
" should be less than or equal to database size: " + std::to_string(total_no) + ".");
|
||||
} else { // constructor TopPercent顶部百分比
|
||||
if (numerator_ > 0 && denominator_ > 0 && numerator_ <= denominator_) {
|
||||
if (numerator_ == 1 && denominator_ > 1) { // sharding分片
|
||||
taking = (total_no + denominator_ - 1) / denominator_;
|
||||
} else { // non sharding不分片
|
||||
taking = total_no * numerator_ / denominator_;
|
||||
taking -= (taking % no_of_categories);
|
||||
}
|
||||
} else {
|
||||
RETURN_STATUS_UNEXPECTED("[Internal ERROR] 'numerator_': " + std::to_string(numerator_) +
|
||||
" should be positive and less than denominator_: " + std::to_string(denominator_) + ".");
|
||||
}
|
||||
}
|
||||
return UpdateTasks(tasks, taking);
|
||||
}
|
||||
//在ShardDistributedSample空间中创建Status型PreExecute函数,返回值为Status变量
|
||||
//判断sampler_type_和kSubsetRandomSampler的值是否相等，若是则调用RETURN_IF_NOT_OK函数
|
||||
//返回Status中Ok函数的返回值
|
||||
Status ShardSample::SufExecute(ShardTaskList &tasks) {
|
||||
if (sampler_type_ == kSubsetRandomSampler) {
|
||||
RETURN_IF_NOT_OK((*shuffle_op_)(tasks));
|
||||
}
|
||||
return Status::OK();
|
||||
}
|
||||
} // namespace mindrecord
|
||||
} // namespace mindspore
|
||||
|
|
@ -0,0 +1,179 @@
|
|||
/**
|
||||
* Copyright 2019 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#include "minddata/mindrecord/include/shard_schema.h"//按照路径寻找以下文件，导入到本文件,以下同理
|
||||
#include "utils/ms_utils.h"
|
||||
|
||||
using mindspore::LogStream;//声明mindspore空间下的LogStream
|
||||
using mindspore::ExceptionType::NoExceptionType;//声明mindspore空间下ExceptionType类中的NoExceptionType
|
||||
using mindspore::MsLogLevel::ERROR;//声明mindspore空间下MsLogLevel类中的ERROR
|
||||
|
||||
namespace mindspore {//创建名为mindspore的空间
|
||||
namespace mindrecord {//创建名为mindrecord的空间
|
||||
//建立存储空间的函数
|
||||
//引用Schema空间中的Build函数，返回空间指针
|
||||
std::shared_ptr<Schema> Schema::Build(std::string desc, const json &schema) {
|
||||
// validate check验证检查
|
||||
if (!Validate(schema)) {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
std::vector<std::string> blob_fields = PopulateBlobFields(schema);
|
||||
Schema object_schema;
|
||||
object_schema.desc_ = std::move(desc);
|
||||
object_schema.blob_fields_ = std::move(blob_fields);
|
||||
object_schema.schema_ = schema;
|
||||
object_schema.schema_id_ = -1;
|
||||
return std::make_shared<Schema>(object_schema);
|
||||
}
|
||||
//引用Schema空间中的GetDesc函数，返回desc_参数
|
||||
std::string Schema::GetDesc() const { return desc_; }
|
||||
//引用Schema空间中GetSchema函数，返回str_schema数组
|
||||
//str_schema数组中储存这Schema的基本数值
|
||||
json Schema::GetSchema() const {
|
||||
json str_schema;
|
||||
str_schema["desc"] = desc_;
|
||||
str_schema["schema"] = schema_;
|
||||
str_schema["blob_fields"] = blob_fields_;
|
||||
return str_schema;
|
||||
}
|
||||
//引用Schema空间中SetSchemaID函数
|
||||
//使schema_id_储存相应的id数值
|
||||
void Schema::SetSchemaID(int64_t id) { schema_id_ = id; }
|
||||
//引用Schema空间中GetSchemaID函数
|
||||
//获得schema_id_的值
|
||||
int64_t Schema::GetSchemaID() const { return schema_id_; }
|
||||
//引用Schema空间中GetGetBlobFields函数
|
||||
//获得blob_fields_的值
|
||||
std::vector<std::string> Schema::GetBlobFields() const { return blob_fields_; }
|
||||
//引用Schema空间中的PopulateBlobFields函数
|
||||
//依次对比schema数组中各项的size、shape、type是否符合要求，若符合则将该项的key存储进blob_fields
|
||||
//最终返回blob_fields
|
||||
std::vector<std::string> Schema::PopulateBlobFields(json schema) {
|
||||
std::vector<std::string> blob_fields;
|
||||
for (json::iterator it = schema.begin(); it != schema.end(); ++it) {
|
||||
json it_value = it.value();
|
||||
if ((it_value.size() == kInt2 && it_value.find("shape") != it_value.end()) || it_value["type"] == "bytes") {
|
||||
blob_fields.emplace_back(it.key());
|
||||
}
|
||||
}
|
||||
return blob_fields;
|
||||
}
|
||||
//引用Schema空间中的ValidateNumberShape函数
|
||||
bool Schema::ValidateNumberShape(const json &it_value) {
|
||||
//判断传入的数据是否是数组的结尾，若是则输出错误信息，并返回false
|
||||
if (it_value.find("shape") == it_value.end()) {
|
||||
MS_LOG(ERROR) << "Invalid schema, 'shape' object can not found in " << it_value.dump()
|
||||
<< ". Please check the input schema.";
|
||||
return false;
|
||||
}
|
||||
//给shape变量赋值，值为输入数据的类型
|
||||
//判断其值是否在标准范围内，若不是则输出错误信息，并返回false
|
||||
auto shape = it_value["shape"];
|
||||
if (!shape.is_array()) {
|
||||
MS_LOG(ERROR) << "Invalid schema, the value of 'shape' should be list format but got: " << it_value["shape"]
|
||||
<< ". Please check the input schema.";
|
||||
return false;
|
||||
}
|
||||
//给num_negtive_one变量赋值为0
|
||||
//进入循环并不断给i进行赋值，值为不同的shape
|
||||
//若出现i等于0或i小于-1的情况，输出错误信息，并返回false
|
||||
//若出现i等于-1的情况，则num_negtive_one+1进行计数
|
||||
int num_negtive_one = 0;
|
||||
for (const auto &i : shape) {
|
||||
if (i == 0 || i < -1) {
|
||||
MS_LOG(ERROR) << "Invalid schema, the element of 'shape' value should be -1 or greater than 0 but got: " << i
|
||||
<< ". Please check the input schema.";
|
||||
return false;
|
||||
}
|
||||
if (i == -1) {
|
||||
num_negtive_one++;
|
||||
}
|
||||
}
|
||||
//判断num_negtive_one是否大于1，若是则输出错误信息，并返回false
|
||||
if (num_negtive_one > 1) {
|
||||
MS_LOG(ERROR) << "Invalid schema, only 1 variable dimension(-1) allowed in 'shape' value but got: "
|
||||
<< it_value["shape"] << ". Please check the input schema.";
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
////引用Schema空间中的Validate函数
|
||||
bool Schema::Validate(json schema) {
|
||||
//判断schema空间是否为空，若是则输出错误信息，并返回false
|
||||
if (schema.empty()) {
|
||||
MS_LOG(ERROR) << "Invalid schema, schema is empty. Please check the input schema.";
|
||||
return false;
|
||||
}
|
||||
//进入循环，依次判断schema空间中所有数据是否符合标准，若不是则输出错误信息，并返回false
|
||||
for (json::iterator it = schema.begin(); it != schema.end(); ++it) {
|
||||
// make sure schema key name must be composed of '0-9' or 'a-z' or 'A-Z' or '_'确保架构密钥名称必须由“0-9”、“a-z”、“a-z”或“_”组成
|
||||
if (!ValidateFieldName(it.key())) {
|
||||
MS_LOG(ERROR) << "Invalid schema, field name: " << it.key()
|
||||
<< "is not composed of '0-9' or 'a-z' or 'A-Z' or '_'. Please rename the field name in schema.";
|
||||
return false;
|
||||
}
|
||||
//确保数据的type存在
|
||||
json it_value = it.value();
|
||||
if (it_value.find("type") == it_value.end()) {
|
||||
MS_LOG(ERROR) << "Invalid schema, 'type' object can not found in field " << it_value.dump()
|
||||
<< ". Please add the 'type' object for field in schema.";
|
||||
return false;
|
||||
}
|
||||
//确保数据的type合法
|
||||
if (kFieldTypeSet.find(it_value["type"]) == kFieldTypeSet.end()) {
|
||||
MS_LOG(ERROR) << "Invalid schema, the value of 'type': " << it_value["type"]
|
||||
<< " is not supported.\nPlease modify the value of 'type' to 'int32', 'int64', 'float32', "
|
||||
"'float64', 'string', 'bytes' in schema.";
|
||||
return false;
|
||||
}
|
||||
//确保数据的合法数据运行后报错
|
||||
if (it_value.size() == kInt1) {
|
||||
continue;
|
||||
}
|
||||
//确保schema空间中存储空间为合法空间
|
||||
if (it_value["type"] == "bytes" || it_value["type"] == "string") {
|
||||
MS_LOG(ERROR)
|
||||
<< "Invalid schema, no other field can be added when the value of 'type' is 'string' or 'types' but got: "
|
||||
<< it_value.dump() << ". Please remove other fields in schema.";
|
||||
return false;
|
||||
}
|
||||
//确保schema空间中存储空间的type和shape属性完整
|
||||
if (it_value.size() != kInt2) {
|
||||
MS_LOG(ERROR) << "Invalid schema, the fields should be 'type' or 'type' and 'shape' but got: " << it_value.dump()
|
||||
<< ". Please check the schema.";
|
||||
return false;
|
||||
}
|
||||
//确保数据的shape属性符合条件
|
||||
if (!ValidateNumberShape(it_value)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
//重载mindrecord空间中的Schema中的b
|
||||
//加入判断：判断此空间下GetDesc函数和GetSchema函数的返回值是否与b的相应函数的返回值相同
|
||||
//任意等式不成立，则返回false，反之返回true
|
||||
bool Schema::operator==(const mindrecord::Schema &b) const {
|
||||
if (this->GetDesc() != b.GetDesc() || this->GetSchema() != b.GetSchema()) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
} // namespace mindrecord
|
||||
} // namespace mindspore
|
||||
|
|
@ -0,0 +1,81 @@
|
|||
/**
|
||||
* Copyright 2020-2021 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#include "minddata/mindrecord/include/shard_sequential_sample.h"//按照路径寻找以下文件，导入到本文件
|
||||
|
||||
using mindspore::LogStream;//声明mindspore空间下的LogStream
|
||||
using mindspore::ExceptionType::NoExceptionType;//声明mindspore空间下ExceptionType类中的NoExceptionType
|
||||
using mindspore::MsLogLevel::ERROR;//声明mindspore空间下MsLogLevel类中的ERROR
|
||||
|
||||
namespace mindspore {//创建名为mindspore的空间
|
||||
namespace mindrecord {//创建名为mindrecord的空间
|
||||
//引用ShardSequentialSample空间中ShardSequentialSample函数,输入两种参数调用指定函数
|
||||
//对参数ShardSample、offset_、per_、per_offset_进行参数初始化
|
||||
ShardSequentialSample::ShardSequentialSample(int64_t n, int64_t offset)
|
||||
: ShardSample(n), offset_(offset), per_(0.0f), per_offset_(0.0f) {}
|
||||
//引用ShardSequentialSample空间中ShardSequentialSample函数,输入两种参数调用指定函数
|
||||
//对参数ShardSample、offset_、per_、per_offset_进行参数初始化
|
||||
ShardSequentialSample::ShardSequentialSample(float per, float per_offset)
|
||||
: ShardSample(0), offset_(0), per_(per), per_offset_(per_offset) {}
|
||||
//引用ShardSequentialSample空间中GetNumSamples函数
|
||||
//判断no_of_samples_是否等于0且per_是否在-kEpsilon到kEpsilon区间内，若是则返回dataset_size
|
||||
//判断per_是否在kEpsilon到1.0f区间内，若是则返回dataset_size与kEpsilon的乘积
|
||||
//若均不符合则返回dataset_size和no_of_samples_之间较小的值
|
||||
int64_t ShardSequentialSample::GetNumSamples(int64_t dataset_size, int64_t num_classes) {
|
||||
if (no_of_samples_ == 0 && (per_ >= -kEpsilon && per_ <= kEpsilon)) {
|
||||
return dataset_size;
|
||||
}
|
||||
if (per_ > kEpsilon && per_ <= 1.0f) {
|
||||
return dataset_size * kEpsilon;
|
||||
}
|
||||
return std::min(static_cast<int64_t>(no_of_samples_), dataset_size);
|
||||
}
|
||||
//引用ShardSequentialSample空间中Execute函数
|
||||
Status ShardSequentialSample::Execute(ShardTaskList &tasks) {
|
||||
int64_t taking;
|
||||
int64_t total_no = static_cast<int64_t>(tasks.sample_ids_.size());
|
||||
//判断no_of_samples_是否等于0且per_是否在-kEpsilon到kEpsilon区间内，若是则给taking赋值，值为total_no
|
||||
//判断per_是否在kEpsilon到1.0f区间内，若是则给taking赋值，值为total_no与kEpsilon的乘积
|
||||
//若均不符合则返回total_no和no_of_samples_之间较小的值
|
||||
if (no_of_samples_ == 0 && (per_ >= -kEpsilon && per_ <= kEpsilon)) {
|
||||
taking = total_no;
|
||||
} else if (per_ > kEpsilon && per_ <= 1.0f) {
|
||||
taking = total_no * kEpsilon;
|
||||
} else {
|
||||
taking = std::min(static_cast<int64_t>(no_of_samples_), total_no);
|
||||
}
|
||||
//判断tasks中permutation_是否为空，若是则给total_no赋值，值为tasks.Size()，并进入循环，依次给new_tasks赋值
|
||||
//引用ShardTaskList空间中的TaskListSwap函数
|
||||
if (tasks.permutation_.empty()) {
|
||||
ShardTaskList new_tasks;
|
||||
total_no = static_cast<int64_t>(tasks.Size());
|
||||
for (int64_t i = offset_; i < taking + offset_; ++i) {
|
||||
new_tasks.AssignTask(tasks, i % total_no);
|
||||
}
|
||||
ShardTaskList::TaskListSwap(tasks, new_tasks);
|
||||
} else { // shuffled洗牌
|
||||
ShardTaskList new_tasks;
|
||||
total_no = static_cast<int64_t>(tasks.permutation_.size());
|
||||
for (int64_t i = offset_; i < taking + offset_; ++i) {
|
||||
new_tasks.AssignTask(tasks, tasks.permutation_[i % total_no]);
|
||||
}
|
||||
ShardTaskList::TaskListSwap(tasks, new_tasks);
|
||||
}
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
} // namespace mindrecord
|
||||
} // namespace mindspore
|
||||
|
|
@ -0,0 +1,215 @@
|
|||
/**
|
||||
* Copyright 2019-2021 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#include "minddata/mindrecord/include/shard_shuffle.h"//按照路径寻找以下文件，导入到本文件
|
||||
|
||||
#include <algorithm>//
|
||||
|
||||
namespace mindspore {//创建名为mindspore的空间
|
||||
namespace mindrecord {//创建名为mindrecord的空间
|
||||
//引用ShardShuffle空间中ShardShuffle函数,输入两种参数调用指定函数
|
||||
//对参数shuffle_seed_、no_of_samples_、replacement_、reshuffle_each_epoch_、shuffle_type_进行进行参数初始化
|
||||
ShardShuffle::ShardShuffle(uint32_t seed, ShuffleType shuffle_type)
|
||||
: shuffle_seed_(seed),
|
||||
no_of_samples_(0),
|
||||
replacement_(false),
|
||||
reshuffle_each_epoch_(true),
|
||||
shuffle_type_(shuffle_type) {}
|
||||
//引用ShardShuffle空间中ShardShuffle函数,输入五种参数调用指定函数
|
||||
//对参数shuffle_seed_、no_of_samples_、replacement_、reshuffle_each_epoch_、shuffle_type_进行进行参数初始化
|
||||
ShardShuffle::ShardShuffle(uint32_t seed, int64_t no_of_samples, bool replacement, bool reshuffle_each_epoch,
|
||||
ShuffleType shuffle_type)
|
||||
: shuffle_seed_(seed),
|
||||
no_of_samples_(no_of_samples),
|
||||
replacement_(replacement),
|
||||
reshuffle_each_epoch_(reshuffle_each_epoch),
|
||||
shuffle_type_(shuffle_type) {}
|
||||
//在ShardShuffle空间中创建GetNumSamples函数
|
||||
//判断replacement_是否为真，若是则返回dataset_size或no_of_samples_
|
||||
int64_t ShardShuffle::GetNumSamples(int64_t dataset_size, int64_t num_classes) {
|
||||
if (replacement_) {
|
||||
return no_of_samples_ == 0 ? dataset_size : no_of_samples_;
|
||||
}
|
||||
return no_of_samples_ == 0 ? dataset_size : std::min(dataset_size, no_of_samples_);
|
||||
}
|
||||
//在ShardShuffle空间中创建CategoryShuffle函数
|
||||
Status ShardShuffle::CategoryShuffle(ShardTaskList &tasks) {
|
||||
int64_t individual_size = tasks.sample_ids_.size() / tasks.categories;
|
||||
std::vector<std::vector<int64_t>> new_permutations(tasks.categories, std::vector<int64_t>(individual_size));
|
||||
//进入循环，反复调用shuffle
|
||||
for (int64_t i = 0; i < tasks.categories; i++) {
|
||||
for (int64_t j = 0; j < individual_size; j++) new_permutations[i][j] = j;
|
||||
std::shuffle(new_permutations[i].begin(), new_permutations[i].end(), std::default_random_engine(shuffle_seed_));
|
||||
}
|
||||
tasks.permutation_.clear();
|
||||
//进入循环，创建tasks.permutation_列表
|
||||
for (int64_t j = 0; j < individual_size; j++) {
|
||||
for (int64_t i = 0; i < tasks.categories; i++) {
|
||||
tasks.permutation_.push_back(new_permutations[i][j] * tasks.categories + i);
|
||||
}
|
||||
}
|
||||
//进入循环，遍历new_tasks列表
|
||||
ShardTaskList new_tasks;
|
||||
for (int64_t i = 0; i < individual_size; ++i) {
|
||||
new_tasks.AssignTask(tasks, tasks.permutation_[i]);
|
||||
}
|
||||
ShardTaskList::TaskListSwap(tasks, new_tasks);
|
||||
|
||||
return Status::OK();
|
||||
}
|
||||
//在ShardShuffle空间中创建ShuffleFiles函数
|
||||
Status ShardShuffle::ShuffleFiles(ShardTaskList &tasks) {
|
||||
//判断no_of_samples_是否为0，若是则赋值tasks的大小
|
||||
if (no_of_samples_ == 0) {
|
||||
no_of_samples_ = tasks.Size();
|
||||
}
|
||||
CHECK_FAIL_RETURN_UNEXPECTED(
|
||||
no_of_samples_ > 0, "Invalid input, 'num_samples' should be positive but got: " + std::to_string(no_of_samples_));
|
||||
auto shard_sample_cout = GetShardSampleCount();
|
||||
|
||||
// shuffle the files index洗牌文件索引
|
||||
std::vector<int64_t> shuffle_files;
|
||||
for (int64_t i = 0; i < shard_sample_cout.size(); i++) {
|
||||
shuffle_files.push_back(i);
|
||||
}
|
||||
std::shuffle(shuffle_files.begin(), shuffle_files.end(), std::default_random_engine(shuffle_seed_));
|
||||
|
||||
// reconstruct the permutation between files重建文件之间的排列
|
||||
// -- before --
|
||||
// file1: [0, 1, 2]
|
||||
// file2: [3, 4, 5, 6]
|
||||
// file3: [7, 8]
|
||||
// file4: [9, 10]
|
||||
// files: [file1, file2, file3, file4]
|
||||
// permutation: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
|
||||
// -- after --
|
||||
// files: [file4, file1, file3, file2]
|
||||
// permutation : [9, 10, 0, 1, 2, 7, 8, 3, 4, 5, 6]
|
||||
auto original_permutation = tasks.permutation_;
|
||||
int64_t whole_index = 0;
|
||||
//进入循环，遍历shuffle_files列表并进行复制
|
||||
for (int64_t i = 0; i < shuffle_files.size(); i++) {
|
||||
int64_t start_index = 0;
|
||||
int64_t current_size = 0;
|
||||
if (shuffle_files[i] == 0) {
|
||||
start_index = 0;
|
||||
current_size = shard_sample_cout[shuffle_files[i]];
|
||||
} else {
|
||||
start_index = shard_sample_cout[shuffle_files[i] - 1];
|
||||
current_size = shard_sample_cout[shuffle_files[i]] - start_index;
|
||||
}
|
||||
std::copy(original_permutation.begin() + start_index, original_permutation.begin() + start_index + current_size,
|
||||
tasks.permutation_.begin() + whole_index);
|
||||
whole_index += current_size;
|
||||
}
|
||||
//进入循环，遍历new_tasks列表
|
||||
auto total_no = tasks.Size();
|
||||
int64_t samples_to_assign =
|
||||
(no_of_samples_ > 0 && no_of_samples_ < total_no) ? no_of_samples_ : tasks.sample_ids_.size();
|
||||
ShardTaskList new_tasks;
|
||||
for (int64_t i = 0; i < samples_to_assign; ++i) {
|
||||
new_tasks.AssignTask(tasks, tasks.permutation_[i]);
|
||||
}
|
||||
ShardTaskList::TaskListSwap(tasks, new_tasks);
|
||||
return Status::OK();
|
||||
}
|
||||
//在ShardShuffle空间中创建ShuffleInFiles函数
|
||||
Status ShardShuffle::ShuffleInfile(ShardTaskList &tasks) {
|
||||
//判断no_of_samples_是否为0，若是则赋值tasks的大小
|
||||
if (no_of_samples_ == 0) {
|
||||
no_of_samples_ = tasks.Size();
|
||||
}
|
||||
CHECK_FAIL_RETURN_UNEXPECTED(
|
||||
no_of_samples_ > 0, "Invalid input, 'num_samples' should be positive but got: " + std::to_string(no_of_samples_));
|
||||
// reconstruct the permutation in file重建文件中的排列
|
||||
// -- before --
|
||||
// file1: [0, 1, 2]
|
||||
// file2: [3, 4, 5, 6]
|
||||
// file3: [7, 8]
|
||||
// file4: [9, 10]
|
||||
// files: [file1, file2, file3, file4]
|
||||
// permutation: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
|
||||
// -- after --
|
||||
// permutation: [2, 0, 1, 4, 6, 3, 5, 8, 7, 9, 10]
|
||||
auto shard_sample_cout = GetShardSampleCount();
|
||||
int64_t start_index = 0;
|
||||
//进入循环，计算出start_index的值
|
||||
for (int64_t i = 0; i < shard_sample_cout.size(); i++) {
|
||||
auto current_size = shard_sample_cout[i] - start_index;
|
||||
std::shuffle(tasks.permutation_.begin() + start_index, tasks.permutation_.begin() + start_index + current_size,
|
||||
std::default_random_engine(shuffle_seed_));
|
||||
start_index = shard_sample_cout[i];
|
||||
}
|
||||
auto total_no = tasks.Size();
|
||||
ShardTaskList new_tasks;
|
||||
int64_t samples_to_assign =
|
||||
(no_of_samples_ > 0 && no_of_samples_ < total_no) ? no_of_samples_ : tasks.sample_ids_.size();
|
||||
//进入循环，遍历new_tasks列表
|
||||
for (int64_t i = 0; i < samples_to_assign; ++i) {
|
||||
new_tasks.AssignTask(tasks, tasks.permutation_[i]);
|
||||
}
|
||||
ShardTaskList::TaskListSwap(tasks, new_tasks);
|
||||
return Status::OK();
|
||||
}
|
||||
//在ShardShuffle空间中创建Execute函数
|
||||
Status ShardShuffle::Execute(ShardTaskList &tasks) {
|
||||
//
|
||||
if (reshuffle_each_epoch_) {
|
||||
shuffle_seed_++;
|
||||
}
|
||||
CHECK_FAIL_RETURN_UNEXPECTED(tasks.categories >= 1,
|
||||
"[Internal ERROR] task categories should be greater than or equal to 1 but got: " +
|
||||
std::to_string(tasks.categories));
|
||||
if (shuffle_type_ == kShuffleSample) { // shuffle each sample洗牌每个样品
|
||||
if (tasks.permutation_.empty() == true) {
|
||||
tasks.MakePerm();
|
||||
}
|
||||
if (GetShuffleMode() == dataset::ShuffleMode::kGlobal) {
|
||||
if (replacement_ == true) {
|
||||
ShardTaskList new_tasks;
|
||||
if (no_of_samples_ == 0) {
|
||||
no_of_samples_ = tasks.sample_ids_.size();
|
||||
}
|
||||
CHECK_FAIL_RETURN_UNEXPECTED(no_of_samples_ > 0, "Invalid input, 'num_samples' should be positive but got: " +
|
||||
std::to_string(no_of_samples_));
|
||||
for (uint32_t i = 0; i < no_of_samples_; ++i) {
|
||||
new_tasks.AssignTask(tasks, tasks.GetRandomTaskID());
|
||||
}
|
||||
|
||||
ShardTaskList::TaskListSwap(tasks, new_tasks);
|
||||
} else {
|
||||
std::shuffle(tasks.permutation_.begin(), tasks.permutation_.end(), std::default_random_engine(shuffle_seed_));
|
||||
auto total_no = tasks.Size();
|
||||
ShardTaskList new_tasks;
|
||||
int64_t samples_to_assign =
|
||||
(no_of_samples_ > 0 && no_of_samples_ < total_no) ? no_of_samples_ : tasks.sample_ids_.size();
|
||||
for (int64_t i = 0; i < samples_to_assign; ++i) {
|
||||
new_tasks.AssignTask(tasks, tasks.permutation_[i]);
|
||||
}
|
||||
ShardTaskList::TaskListSwap(tasks, new_tasks);
|
||||
}
|
||||
} else if (GetShuffleMode() == dataset::ShuffleMode::kInfile) {
|
||||
RETURN_IF_NOT_OK(ShuffleInfile(tasks));
|
||||
} else if (GetShuffleMode() == dataset::ShuffleMode::kFiles) {
|
||||
RETURN_IF_NOT_OK(ShuffleFiles(tasks));
|
||||
}
|
||||
} else { // shuffle unit like: (a1, b1, c1),(a2, b2, c2),..., (an, bn, cn)分组洗牌
|
||||
return this->CategoryShuffle(tasks);
|
||||
}
|
||||
return Status::OK();
|
||||
}
|
||||
} // namespace mindrecord
|
||||
} // namespace mindspore
|
||||
|
|
@ -0,0 +1,109 @@
|
|||
/**
|
||||
* Copyright 2019 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#include "minddata/mindrecord/include/shard_statistics.h"//按照路径寻找以下文件，导入到本文件,以下同理
|
||||
#include "pybind11/pybind11.h"
|
||||
|
||||
using mindspore::LogStream;//声明mindspore空间下的LogStream
|
||||
using mindspore::ExceptionType::NoExceptionType;//声明mindspore空间下ExceptionType类中的NoExceptionType
|
||||
using mindspore::MsLogLevel::ERROR;//声明mindspore空间下MsLogLevel类中的ERROR
|
||||
|
||||
namespace mindspore {//创建名为mindspore的空间
|
||||
namespace mindrecord {//创建名为mindrecord的空间
|
||||
//建立存储空间的函数
|
||||
//引用Schema空间中的Build函数，返回空间指针
|
||||
std::shared_ptr<Statistics> Statistics::Build(std::string desc, const json &statistics) {
|
||||
// validate check验证检查
|
||||
if (!Validate(statistics)) {
|
||||
return nullptr;
|
||||
}
|
||||
Statistics object_statistics;
|
||||
object_statistics.desc_ = std::move(desc);
|
||||
object_statistics.statistics_ = statistics;
|
||||
object_statistics.statistics_id_ = -1;
|
||||
return std::make_shared<Statistics>(object_statistics);
|
||||
}
|
||||
//引用Statistics空间中的GetDesc函数，返回desc_参数
|
||||
std::string Statistics::GetDesc() const { return desc_; }
|
||||
//引用Statistics空间中GetStatistics函数，返回str_schema数组
|
||||
//str_statistics数组中储存这statistics的基本数值
|
||||
json Statistics::GetStatistics() const {
|
||||
json str_statistics;
|
||||
str_statistics["desc"] = desc_;
|
||||
str_statistics["statistics"] = statistics_;
|
||||
return str_statistics;
|
||||
}
|
||||
//引用Statistics空间中SetStatisticsID函数
|
||||
//使statistics_id_储存相应的id数值
|
||||
void Statistics::SetStatisticsID(int64_t id) { statistics_id_ = id; }
|
||||
//引用Statistics空间中GetStatisticsID函数
|
||||
//获得statistics_id_的值
|
||||
int64_t Statistics::GetStatisticsID() const { return statistics_id_; }
|
||||
//引用Statistics空间中的Validate函数
|
||||
bool Statistics::Validate(const json &statistics) {
|
||||
//判断数据是否符合标准，若不符则输出错误信息，并返回false
|
||||
if (statistics.size() != kInt1) {
|
||||
MS_LOG(ERROR) << "Invalid data, 'statistics' is empty.";
|
||||
return false;
|
||||
}
|
||||
//判断数据是否是statistics空间的结尾，若是则输出错误信息，并返回false
|
||||
if (statistics.find("level") == statistics.end()) {
|
||||
MS_LOG(ERROR) << "Invalid data, 'level' object can not found in statistic";
|
||||
return false;
|
||||
}
|
||||
//若以上都不符合则返回LevelRecursive函数
|
||||
return LevelRecursive(statistics["level"]);
|
||||
}
|
||||
//引用Statistics空间中的LevelRecursive函数
|
||||
bool Statistics::LevelRecursive(json level) {
|
||||
bool ini = true;
|
||||
//进入循环，遍历level数组
|
||||
for (json::iterator it = level.begin(); it != level.end(); ++it) {
|
||||
json a = it.value();
|
||||
//判断该空间大小是否符合标准2，若符合则判断该空间的key和count是否与无数据空间一致，若是则返回错误信息并返回false
|
||||
//若空间大小不符合标准2，则判断空间大小是否符合标准3，若符合则判断该空间key、count、level是否与无数据空间一致，若是则返回错误信息并返回false，若不符合则给ini变量赋值
|
||||
//若均不符合，则返回错误信息，返回false
|
||||
//最后返回ini的值
|
||||
if (a.size() == kInt2) {
|
||||
if ((a.find("key") == a.end()) || (a.find("count") == a.end())) {
|
||||
MS_LOG(ERROR) << "Invalid data, the node field is 2, but 'key'/'count' object does not existed";
|
||||
return false;
|
||||
}
|
||||
} else if (a.size() == kInt3) {
|
||||
if ((a.find("key") == a.end()) || (a.find("count") == a.end()) || a.find("level") == a.end()) {
|
||||
MS_LOG(ERROR) << "Invalid data, the node field is 3, but 'key'/'count'/'level' object does not existed";
|
||||
return false;
|
||||
} else {
|
||||
ini = LevelRecursive(a.at("level"));
|
||||
}
|
||||
} else {
|
||||
MS_LOG(ERROR) << "Invalid data, the node field is not equal to 2 or 3";
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return ini;
|
||||
}
|
||||
//重载mindrecord空间中的Statistics中的b
|
||||
//加入判断：判断此空间下GetStatistics函数和GetStatistics函数的返回值是否与b的相应函数的返回值相同
|
||||
//任意等式不成立，则返回false，反之返回true
|
||||
bool Statistics::operator==(const Statistics &b) const {
|
||||
if (this->GetStatistics() != b.GetStatistics()) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
} // namespace mindrecord
|
||||
} // namespace mindspore
|
||||
|
|
@ -0,0 +1,162 @@
|
|||
/**
|
||||
* Copyright 2019-2021 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#include "minddata/dataset/util/random.h"//按照路径寻找以下文件，导入到本文件,以下同理
|
||||
#include "minddata/mindrecord/include/shard_task_list.h"
|
||||
#include "utils/ms_utils.h"
|
||||
#include "minddata/mindrecord/include/common/shard_utils.h"
|
||||
|
||||
using mindspore::LogStream;//声明mindspore空间下的LogStream
|
||||
using mindspore::ExceptionType::NoExceptionType;//声明mindspore空间下ExceptionType类中的NoExceptionType
|
||||
using mindspore::MsLogLevel::DEBUG;//声明mindspore空间下MsLogLevel类中的ERROR
|
||||
|
||||
namespace mindspore {//创建名为mindspore的空间
|
||||
namespace mindrecord {//创建名为mindrecord的空间
|
||||
//引用ShardTaskList空间中ShardTaskList函数,不输入参数调用指定函数
|
||||
//对参数categories进行参数初始化
|
||||
ShardTaskList::ShardTaskList() : categories(1) {}
|
||||
//引用ShardTaskList空间中ShardTaskList函数,输入一种参数调用指定函数
|
||||
//对参数categories、permutation_、sample_ids_、task_list_进行参数初始化
|
||||
ShardTaskList::ShardTaskList(const ShardTaskList &other)
|
||||
: categories(other.categories),
|
||||
permutation_(other.permutation_),
|
||||
sample_ids_(other.sample_ids_),
|
||||
task_list_(other.task_list_) {}
|
||||
//重载ShardTaskList空间中的TaskList中的other
|
||||
//交换多种值的内容
|
||||
ShardTaskList &ShardTaskList::operator=(const ShardTaskList &other) {
|
||||
ShardTaskList tmp(other);
|
||||
std::swap(categories, tmp.categories);
|
||||
permutation_.swap(tmp.permutation_);
|
||||
sample_ids_.swap(tmp.sample_ids_);
|
||||
task_list_.swap(tmp.task_list_);
|
||||
return *this;
|
||||
}
|
||||
//引用ShardTaskList空间中InitSampleIds函数
|
||||
//进入循环，创建列表
|
||||
void ShardTaskList::InitSampleIds() {
|
||||
// no-op if there already exists sample ids. Do not clobber previous list如果已经存在示例id，则无操作。不要破坏上一个列表
|
||||
if (sample_ids_.empty()) {
|
||||
sample_ids_ = std::vector<int64_t>(task_list_.size());
|
||||
for (auto i = 0; i < task_list_.size(); i++) {
|
||||
sample_ids_[i] = i;
|
||||
}
|
||||
}
|
||||
}
|
||||
//引用ShardTaskList空间中MakePerm函数
|
||||
//创建permutation_列表
|
||||
void ShardTaskList::MakePerm() {
|
||||
int64_t perm_size = sample_ids_.size();
|
||||
permutation_ = std::vector<int64_t>(perm_size);
|
||||
for (int64_t i = 0; i < perm_size; i++) {
|
||||
permutation_[i] = i;
|
||||
}
|
||||
}
|
||||
//引用ShardTaskList空间中TaskListSwap函数
|
||||
// Swap the new_tasks with orig_tasks将新任务与orig_tasks交换
|
||||
void ShardTaskList::TaskListSwap(ShardTaskList &orig_tasks, ShardTaskList &new_tasks) {
|
||||
// When swapping, if the orig_tasks contains fields that need to be preserved after the swap, then swapping with a交换时，如果orig_tasks包含交换后需要保留的字段，则使用
|
||||
// new_tasks that does not have those fields will result in clobbering/losing the data after the swap.没有这些字段的new_tasks将导致交换后数据丢失。
|
||||
// The task_list_ should not be lost/clobbered.task_list_不应丢失/丢失。
|
||||
// This function can be called in the middle of mindrecord's epoch, when orig_tasks.task_list_ is still being这个函数可以在mindrecord的时代中期调用，当orig_任务时。task_list_仍在
|
||||
// used by mindrecord op's worker threads. So don't touch its task_list_ since this field should be preserved anyways.
|
||||
//由mindrecord op的工作线程使用。因此，不要触摸其task_list_，因为无论如何都应该保留此字段。
|
||||
std::swap(orig_tasks.categories, new_tasks.categories);
|
||||
std::swap(orig_tasks.permutation_, new_tasks.permutation_);
|
||||
std::swap(orig_tasks.sample_ids_, new_tasks.sample_ids_);
|
||||
}
|
||||
//引用ShardTaskList空间中PopBack函数
|
||||
//调用task_list_类中pop_back函数
|
||||
void ShardTaskList::PopBack() { task_list_.pop_back(); }
|
||||
//引用ShardTaskList空间中Size函数
|
||||
//返回task_list_空间中size函数的返回值
|
||||
int64_t ShardTaskList::Size() const { return static_cast<int64_t>(task_list_.size()); }
|
||||
//引用ShardTaskList空间中SizeOfRows函数
|
||||
int64_t ShardTaskList::SizeOfRows() const {
|
||||
//判断task_list_的长度是否为0，若是则返回0
|
||||
if (task_list_.size() == 0) return static_cast<int64_t>(0);
|
||||
|
||||
// 1 task is 1 page1个任务是1页
|
||||
const size_t kBlobInfoIndex = 2;
|
||||
auto sum_num_rows = [](int64_t x, ShardTask y) { return x + std::get<kBlobInfoIndex>(y)[0]; };
|
||||
int64_t nRows = std::accumulate(task_list_.begin(), task_list_.end(), 0, sum_num_rows);
|
||||
return nRows;
|
||||
}
|
||||
//引用ShardTaskList空间中GetTaskByID函数
|
||||
//返回task_list_数组中id的返回值
|
||||
ShardTask &ShardTaskList::GetTaskByID(int64_t id) { return task_list_[id]; }
|
||||
//引用ShardTaskList空间中GetTaskSampleByID函数
|
||||
//返回sample_ids_数组中id的返回值
|
||||
int64_t ShardTaskList::GetTaskSampleByID(int64_t id) { return sample_ids_[id]; }
|
||||
//引用ShardTaskList空间中GetRandomTaskID函数
|
||||
int64_t ShardTaskList::GetRandomTaskID() {
|
||||
std::mt19937 gen = mindspore::dataset::GetRandomDevice();
|
||||
std::uniform_int_distribution<> dis(0, sample_ids_.size() - 1);
|
||||
return dis(gen);
|
||||
}
|
||||
//引用ShardTaskList空间中GetRandomTask函数
|
||||
ShardTask &ShardTaskList::GetRandomTask() {
|
||||
std::mt19937 gen = mindspore::dataset::GetRandomDevice();
|
||||
std::uniform_int_distribution<> dis(0, task_list_.size() - 1);
|
||||
return task_list_[dis(gen)];
|
||||
}
|
||||
//引用ShardTaskList空间中Combine函数
|
||||
ShardTaskList ShardTaskList::Combine(std::vector<ShardTaskList> &category_tasks, bool replacement, int64_t num_elements,
|
||||
int64_t num_samples) {
|
||||
ShardTaskList res;
|
||||
//判断category_tasks是否为空，若是则返回res的值
|
||||
if (category_tasks.empty()) return res;
|
||||
auto total_categories = category_tasks.size();
|
||||
res.categories = static_cast<int64_t>(total_categories);
|
||||
//判断resplacement是否为false，若是则判断category_tasks列表中的最小值
|
||||
//在0至category_tasks最小值的区间内反复调用InsertTask函数进行数据处理，直到num_samples等于0且count等于num_samples
|
||||
//若resplacement不为0，则判断category_tasks列表中的最大值
|
||||
//在0至category_tasks最小值的区间内反复调用InsertTask函数进行数据处理，直到num_samples等于0且count等于num_samples
|
||||
if (replacement == false) {
|
||||
auto minTasks = category_tasks[0].Size();
|
||||
for (int64_t i = 1; i < total_categories; i++) {
|
||||
minTasks = std::min(minTasks, category_tasks[i].Size());
|
||||
}
|
||||
int64_t count = 0;
|
||||
for (int64_t task_no = 0; task_no < minTasks; task_no++) {
|
||||
for (int64_t i = 0; i < total_categories; i++) {
|
||||
if (num_samples != 0 && count == num_samples) break;
|
||||
res.InsertTask(std::move(category_tasks[i].GetTaskByID(task_no)));
|
||||
count++;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
auto maxTasks = category_tasks[0].Size();
|
||||
for (int64_t i = 1; i < total_categories; i++) {
|
||||
maxTasks = std::max(maxTasks, category_tasks[i].Size());
|
||||
}
|
||||
if (num_elements != std::numeric_limits<int64_t>::max()) {
|
||||
maxTasks = static_cast<decltype(maxTasks)>(num_elements);
|
||||
}
|
||||
int64_t count = 0;
|
||||
for (int64_t i = 0; i < total_categories; i++) {
|
||||
for (int64_t j = 0; j < maxTasks; j++) {
|
||||
if (num_samples != 0 && count == num_samples) break;
|
||||
res.InsertTask(category_tasks[i].GetRandomTask());
|
||||
count++;
|
||||
}
|
||||
}
|
||||
}
|
||||
//返回res的值
|
||||
return res;
|
||||
}
|
||||
} // namespace mindrecord
|
||||
} // namespace mindspore
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
if(ENABLE_D OR ENABLE_ACL)
|
||||
file(GLOB_RECURSE _TRANSFORM_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc")
|
||||
list(REMOVE_ITEM _TRANSFORM_SRC_LIST "graph_ir/op_declare/hcom_ops_declare.cc")
|
||||
set_property(SOURCE ${_TRANSFORM_SRC_LIST} PROPERTY COMPILE_DEFINITIONS
|
||||
SUBMODULE_ID=mindspore::SubModuleId::SM_GE_ADPT)
|
||||
add_library(_mindspore_transform_graph_ir_obj OBJECT ${_TRANSFORM_SRC_LIST})
|
||||
|
||||
endif()
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
/**
|
||||
* Copyright 2019 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_ALL_OPS_H_
|
||||
#define MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_ALL_OPS_H_
|
||||
|
||||
// old
|
||||
#include "ops/all_ops.h"
|
||||
#endif // MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_ALL_OPS_H_
|
||||
|
|
@ -0,0 +1,130 @@
|
|||
/**
|
||||
* Copyright 2019 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#include "transform/graph_ir/op_declare/array_ops_declare.h"
|
||||
#include <vector>
|
||||
|
||||
namespace mindspore::transform {//创建名为transform的空间，使其位于mindspore空间下
|
||||
// const
|
||||
//部分语句与下面语句块作用类似，此处为对变量const进行调整
|
||||
INPUT_MAP(Const) = EMPTY_INPUT_MAP;//将const与标准进行比较，使原以input_map_为的key变为空
|
||||
ATTR_MAP(Const) = {{"value", ATTR_DESC(value, AnyTraits<AnyValue>())}};/*
|
||||
将value处理并存入对应ATTR_DESC结构体的相应变量中
|
||||
将value内容转为字符串变量并存储至结构体的name变量中
|
||||
引用Operator空间并将指针所指的类转为ATTR_DESC结构体
|
||||
//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
//存入MaxPool对应空间并用attr_map_指针保存
|
||||
//其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
将Const的类型与标准进行对比，后进行空间调整，并用指针attr_map_为key存储相应内容
|
||||
*/
|
||||
OUTPUT_MAP(Const) = {{0, OUTPUT_DESC(y)}};//用新建的OUTPUT结构体并与标准比较其容量，以output_map_为变量的key
|
||||
|
||||
// Constant
|
||||
//此处为对变量Constant进行调整
|
||||
INPUT_MAP(Constant) = EMPTY_INPUT_MAP;
|
||||
ATTR_MAP(Constant) = {{"value", ATTR_DESC(value, AnyTraits<AnyValue>())}};
|
||||
OUTPUT_MAP(Constant) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(Constant, kNameConst, ADPT_DESC(Constant, Const))//将Constant处理并存入对应REG_ADPT_DESC结构体的相应变量中
|
||||
//将Constant内容转为字符串变量并存储至结构体的name变量中
|
||||
//引用Operator空间并将指针所指的类转为结构体
|
||||
|
||||
// ScalarSummary
|
||||
INPUT_MAP(Summary) = {{2, INPUT_DESC(x)}};
|
||||
ATTR_MAP(Summary) = EMPTY_ATTR_MAP;
|
||||
#ifndef ENABLE_SECURITY
|
||||
REG_ADPT_DESC(ScalarSummary, prim::kPrimScalarSummary->name(), ADPT_DESC(Summary))
|
||||
REG_ADPT_DESC(ImageSummary, prim::kPrimImageSummary->name(), ADPT_DESC(Summary))
|
||||
REG_ADPT_DESC(TensorSummary, prim::kPrimTensorSummary->name(), ADPT_DESC(Summary))
|
||||
REG_ADPT_DESC(HistogramSummary, prim::kPrimHistogramSummary->name(), ADPT_DESC(Summary))
|
||||
#endif
|
||||
REG_ADPT_DESC(Debug, prim::kPrimDebug->name(), ADPT_DESC(Summary))
|
||||
|
||||
// Data
|
||||
INPUT_MAP(Data) = EMPTY_INPUT_MAP;
|
||||
ATTR_MAP(Data) = EMPTY_ATTR_MAP;
|
||||
REG_ADPT_DESC(Data, kNameParam, ADPT_DESC(Data))
|
||||
|
||||
// Shape
|
||||
INPUT_MAP(Shape) = {{1, INPUT_DESC(x)}};
|
||||
ATTR_MAP(Shape) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(Shape) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(Shape, kNameShape, ADPT_DESC(Shape))
|
||||
|
||||
// Reshape
|
||||
INPUT_MAP(Reshape) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(shape)}};
|
||||
ATTR_MAP(Reshape) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(Reshape) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(Reshape, kNameReshape, ADPT_DESC(Reshape))
|
||||
REG_ADPT_DESC(FlattenGrad, kNameFlattenGrad, ADPT_DESC(Reshape))
|
||||
|
||||
// TransShape
|
||||
INPUT_MAP(TransShape) = {{1, INPUT_DESC(x)}};
|
||||
INPUT_ATTR_MAP(TransShape) = {{2, ATTR_DESC(outShape, AnyTraits<int64_t>(), AnyTraits<std::vector<int64_t>>())}};
|
||||
ATTR_MAP(TransShape) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(TransShape) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(TransShape, kNameTransShape, ADPT_DESC(TransShape))
|
||||
|
||||
// MirrorPad
|
||||
INPUT_MAP(MirrorPad) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(paddings)}};
|
||||
ATTR_MAP(MirrorPad) = {{"mode", ATTR_DESC(mode, AnyTraits<std::string>())}};
|
||||
OUTPUT_MAP(MirrorPad) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(MirrorPad, kNameMirrorPad, ADPT_DESC(MirrorPad))
|
||||
|
||||
// MirrorPadGrad
|
||||
INPUT_MAP(MirrorPadGrad) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(paddings)}};
|
||||
ATTR_MAP(MirrorPadGrad) = {{"mode", ATTR_DESC(mode, AnyTraits<std::string>())}};
|
||||
OUTPUT_MAP(MirrorPadGrad) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(MirrorPadGrad, kNameMirrorPadGrad, ADPT_DESC(MirrorPadGrad))
|
||||
|
||||
// ExpandDims
|
||||
INPUT_MAP(ExpandDims) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(axis)}};
|
||||
ATTR_MAP(ExpandDims) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(ExpandDims) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(ExpandDims, kNameExpandDims, ADPT_DESC(ExpandDims))
|
||||
|
||||
// Squeeze
|
||||
INPUT_MAP(Squeeze) = {{1, INPUT_DESC(x)}};
|
||||
ATTR_MAP(Squeeze) = {{"axis", ATTR_DESC(axis, AnyTraits<int64_t>(), AnyTraits<std::vector<int64_t>>())}};
|
||||
OUTPUT_MAP(Squeeze) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(Squeeze, prim::kPrimSqueeze->name(), ADPT_DESC(Squeeze))
|
||||
|
||||
// ReverseSequence
|
||||
INPUT_MAP(ReverseSequence) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(seq_lengths)}};
|
||||
ATTR_MAP(ReverseSequence) = {{"seq_dim", ATTR_DESC(seq_dim, AnyTraits<int64_t>())},
|
||||
{"batch_dim", ATTR_DESC(batch_dim, AnyTraits<int64_t>())}};
|
||||
OUTPUT_MAP(ReverseSequence) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(ReverseSequence, kNameReverseSequence, ADPT_DESC(ReverseSequence))
|
||||
|
||||
// EditDistance
|
||||
INPUT_MAP(EditDistance) = {{1, INPUT_DESC(hypothesis_indices)}, {2, INPUT_DESC(hypothesis_values)},
|
||||
{3, INPUT_DESC(hypothesis_shape)}, {4, INPUT_DESC(truth_indices)},
|
||||
{5, INPUT_DESC(truth_values)}, {6, INPUT_DESC(truth_shape)}};
|
||||
ATTR_MAP(EditDistance) = {{"normalize", ATTR_DESC(normalize, AnyTraits<bool>())}};
|
||||
OUTPUT_MAP(EditDistance) = {{0, OUTPUT_DESC(output)}};
|
||||
REG_ADPT_DESC(EditDistance, kNameEditDistance, ADPT_DESC(EditDistance))
|
||||
|
||||
// NonZero
|
||||
INPUT_MAP(NonZero) = {{1, INPUT_DESC(x)}};
|
||||
ATTR_MAP(NonZero) = {{"transpose", ATTR_DESC(transpose, AnyTraits<bool>())}};
|
||||
OUTPUT_MAP(NonZero) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(NonZero, kNameNonZero, ADPT_DESC(NonZero))
|
||||
|
||||
// Unsqueeze
|
||||
INPUT_MAP(Unsqueeze) = {{1, INPUT_DESC(x)}};
|
||||
ATTR_MAP(Unsqueeze) = {{"axis", ATTR_DESC(axes, AnyTraits<int64_t>(), AnyTraits<std::vector<int64_t>>())}};
|
||||
OUTPUT_MAP(Unsqueeze) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(Unsqueeze, kNameUnsqueeze, ADPT_DESC(Unsqueeze))
|
||||
} // namespace mindspore::transform
|
||||
|
|
@ -0,0 +1,71 @@
|
|||
/**
|
||||
* Copyright 2019-2021 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_ARRAY_OPS_DECLARE_H_
|
||||
#define MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_ARRAY_OPS_DECLARE_H_
|
||||
|
||||
#include <string>
|
||||
#include "utils/hash_map.h"
|
||||
#include "transform/graph_ir/op_declare/op_declare_macro.h"
|
||||
#include "ops/array_ops.h"
|
||||
|
||||
namespace mindspore::transform {
|
||||
DECLARE_OP_ADAPTER(Shape)//将shape收录后与标准进行比较，进行空间调整，用input_map_为key储存
|
||||
DECLARE_OP_USE_OUTPUT(Shape)//将shape收录后与标准进行比较，进行空间调整，用output_map_为key储存
|
||||
//下同，根据引入的不同变量进行对不同变量的操作
|
||||
|
||||
DECLARE_OP_ADAPTER(Reshape)
|
||||
DECLARE_OP_USE_OUTPUT(Reshape)
|
||||
|
||||
DECLARE_OP_ADAPTER(TransShape)
|
||||
DECLARE_OP_USE_INPUT_ATTR(TransShape)//将TransShape收录后与标准进行比较，进行空间调整，用input_map_为key储存
|
||||
DECLARE_OP_USE_OUTPUT(TransShape)
|
||||
|
||||
DECLARE_OP_ADAPTER(MirrorPad)
|
||||
DECLARE_OP_USE_OUTPUT(MirrorPad)
|
||||
|
||||
DECLARE_OP_ADAPTER(MirrorPadGrad)
|
||||
DECLARE_OP_USE_OUTPUT(MirrorPadGrad)
|
||||
|
||||
DECLARE_OP_ADAPTER(ExpandDims)
|
||||
DECLARE_OP_USE_OUTPUT(ExpandDims)
|
||||
|
||||
DECLARE_OP_ADAPTER(Squeeze)
|
||||
DECLARE_OP_USE_OUTPUT(Squeeze)
|
||||
|
||||
DECLARE_OP_ADAPTER(Constant)
|
||||
DECLARE_OP_USE_OUTPUT(Constant)
|
||||
|
||||
DECLARE_OP_ADAPTER(Summary)
|
||||
|
||||
DECLARE_OP_ADAPTER(Const)
|
||||
DECLARE_OP_USE_OUTPUT(Const)
|
||||
|
||||
DECLARE_OP_ADAPTER(Data)
|
||||
|
||||
DECLARE_OP_ADAPTER(ReverseSequence)
|
||||
DECLARE_OP_USE_OUTPUT(ReverseSequence)
|
||||
|
||||
DECLARE_OP_ADAPTER(EditDistance)
|
||||
DECLARE_OP_USE_OUTPUT(EditDistance)
|
||||
|
||||
DECLARE_OP_ADAPTER(NonZero)
|
||||
DECLARE_OP_USE_OUTPUT(NonZero)
|
||||
|
||||
DECLARE_OP_ADAPTER(Unsqueeze)
|
||||
DECLARE_OP_USE_OUTPUT(Unsqueeze)
|
||||
} // namespace mindspore::transform
|
||||
#endif // MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_ARRAY_OPS_DECLARE_H_
|
||||
|
|
@ -0,0 +1,34 @@
|
|||
/**
|
||||
* Copyright 2019 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#include "transform/graph_ir/op_declare/control_flow_ops_declare.h"
|
||||
|
||||
namespace mindspore::transform {
|
||||
// Merge
|
||||
INPUT_MAP(Merge) = EMPTY_INPUT_MAP;//将Merge与标准进行比较，使原以input_map_为的key变为空
|
||||
DYN_INPUT_MAP(Merge) = {{1, DYN_INPUT_DESC(x)}};//将Merge与标准进行比较，以dyn_input_map_为key并构造储存了name变量x的DYN_INPUT_DESC结构体并与标准比较其容量
|
||||
ATTR_MAP(Merge) = EMPTY_ATTR_MAP;//将Merge与标准进行比较，使原以input_map_为的key变为空
|
||||
OUTPUT_MAP(Merge) = {{0, OUTPUT_DESC(y)}, {1, OUTPUT_DESC(value_index)}};//将Merge与标准进行比较，并分别将两组变量与标准比较其容量
|
||||
REG_ADPT_DESC(Merge, kNameMerge, ADPT_DESC(Merge))//将Merge处理并存入对应OutputDesc结构体的相应变量中
|
||||
//将Merge内容转为字符串变量并存储至结构体的name变量中
|
||||
//引用Operator空间并将指针所指的类转为OutputDesc结构体
|
||||
|
||||
// Switch
|
||||
INPUT_MAP(Switch) = {{1, INPUT_DESC(data)}, {2, INPUT_DESC(pred)}};
|
||||
OUTPUT_MAP(Switch) = {{0, OUTPUT_DESC(output_false)}, {1, OUTPUT_DESC(output_true)}};
|
||||
ATTR_MAP(Switch) = EMPTY_ATTR_MAP;
|
||||
REG_ADPT_DESC(Switch, kNameGeSwitch, ADPT_DESC(Switch))
|
||||
} // namespace mindspore::transform
|
||||
|
|
@ -0,0 +1,46 @@
|
|||
/**
|
||||
* Copyright 2019-2021 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_CONTROL_FLOW_OPS_DECLARE_H_
|
||||
#define MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_CONTROL_FLOW_OPS_DECLARE_H_
|
||||
|
||||
#include <string>
|
||||
#include "utils/hash_map.h"
|
||||
#include "transform/graph_ir/op_declare/op_declare_macro.h"
|
||||
#include "ops/control_flow_ops.h"
|
||||
|
||||
namespace mindspore::transform {
|
||||
DECLARE_OP_ADAPTER(Merge)/*
|
||||
函数功能为将收录内容的类型与标准进行对比，后进行空间调整，并用指针input_map_为key存储相应内容
|
||||
具体实现过程：
|
||||
判断int与类InputDesc的容量是否小于标准+判断int与类InputDesc是否为可移动构造类型+判断int与类InputDesc是否拥有移动赋值运算符，获得IsFlat
|
||||
判断T类型是否为void，若是则返回类型Key，若不是则根据IsFlat的真假返回Key或Key const，并与T类型交换最后返回其值。
|
||||
判断完成后，若结果为真，将范围在4~ 16384的内容从空间分配器中删去
|
||||
若结果为假，建立新的分配器并删除原分配的空间（链表），建立新的内存块并加入到新的分配器中。
|
||||
统计可用元素数量，通过已分配空间计算需要分配的内存大小
|
||||
为新数据创建链表并加入至分配的空间中，元素字节强制对齐。
|
||||
并在建成的堆中分配具体空间
|
||||
使用op_adapter_base.h下ge空间中op类下变量T，利用op_adapter.h中OpAdapter中分配的初始指针作为key储存元素
|
||||
string与AttrDesc进行相同操作，用指针attr_map_为key存储相应内容
|
||||
*/
|
||||
//此处变量为Merge，最终用input_map_为key储存相应内容
|
||||
DECLARE_OP_USE_DYN_INPUT(Merge)//函数功能为将Merge的类型与标准进行对比，后进行空间调整，并用指针input_map_为key存储相应内容
|
||||
DECLARE_OP_USE_OUTPUT(Merge)//函数功能为将Merge的类型与标准进行对比，后进行空间调整，并用指针output_map_为key存储相应内容
|
||||
|
||||
DECLARE_OP_ADAPTER(Switch)
|
||||
DECLARE_OP_USE_OUTPUT(Switch)
|
||||
} // namespace mindspore::transform
|
||||
#endif // MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_CONTROL_FLOW_OPS_DECLARE_H_
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,50 @@
|
|||
/**
|
||||
* Copyright 2019 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#include "transform/graph_ir/op_declare/ctc_ops_declare.h"
|
||||
|
||||
namespace mindspore::transform {
|
||||
// CTCLoss
|
||||
INPUT_MAP(CTCLoss) = {{1, INPUT_DESC(inputs)},
|
||||
{2, INPUT_DESC(labels_indices)},
|
||||
{3, INPUT_DESC(labels_values)},
|
||||
{4, INPUT_DESC(sequence_length)}};//收录四组变量并分别与标准比较其容量并返回对应的值，以input_map_为key储存相应内容
|
||||
ATTR_MAP(CTCLoss) = {
|
||||
{"preprocess_collapse_repeated", ATTR_DESC(preprocess_collapse_repeated, AnyTraits<bool>())},
|
||||
{"ctc_merge_repeated", ATTR_DESC(ctc_merge_repeated, AnyTraits<bool>())},
|
||||
{"ignore_longer_outputs_than_inputs", ATTR_DESC(ignore_longer_outputs_than_inputs, AnyTraits<bool>())}};//收录四组变量并分别与标准比较其容量并返回对应的值，以attr_map_为key储存相应内容
|
||||
/*
|
||||
将ignore_longer_outputs_than_inputs处理并存入对应ATTR_DESC结构体的相应变量中
|
||||
将ignore_longer_outputs_than_inputs内容转为字符串变量并存储至结构体的name变量中
|
||||
引用Operator空间并将指针所指的类转为ATTR_DESC结构体
|
||||
//AnyTraits作用:对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
存入MaxPool对应空间并用attr_map_指针保存
|
||||
其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
*/
|
||||
OUTPUT_MAP(CTCLoss) = {{0, OUTPUT_DESC(loss)}, {1, OUTPUT_DESC(gradient)}};//将收录的两组数据的类型与标准进行对比，后进行空间调整，并用指针output_map_为key存储相应内容
|
||||
REG_ADPT_DESC(CTCLoss, kNameCTCLoss, ADPT_DESC(CTCLoss))//将CTCLoss处理并存入对应REG_ADPT_Desc结构体的相应变量中
|
||||
//将name变量内容转为字符串变量并存储至结构体的name变量中
|
||||
//引用Operator空间并将指针所指的类转为REG_ADPT_Desc结构体
|
||||
|
||||
// CTCGreedyDecoder
|
||||
INPUT_MAP(CTCGreedyDecoder) = {{1, INPUT_DESC(inputs)}, {2, INPUT_DESC(sequence_length)}};
|
||||
ATTR_MAP(CTCGreedyDecoder) = {{"merge_repeated", ATTR_DESC(merge_repeated, AnyTraits<bool>())}};
|
||||
OUTPUT_MAP(CTCGreedyDecoder) = {{0, OUTPUT_DESC(decoded_indices)},
|
||||
{1, OUTPUT_DESC(decoded_values)},
|
||||
{2, OUTPUT_DESC(decoded_shape)},
|
||||
{3, OUTPUT_DESC(log_probability)}};
|
||||
REG_ADPT_DESC(CTCGreedyDecoder, kNameCTCGreedyDecoder, ADPT_DESC(CTCGreedyDecoder))
|
||||
} // namespace mindspore::transform
|
||||
|
|
@ -0,0 +1,32 @@
|
|||
/**
|
||||
* Copyright 2019-2021 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_CTC_OPS_DECLARE_H_
|
||||
#define MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_CTC_OPS_DECLARE_H_
|
||||
|
||||
#include <string>
|
||||
#include "utils/hash_map.h"
|
||||
#include "transform/graph_ir/op_declare/op_declare_macro.h"
|
||||
#include "ops/ctc_ops.h"
|
||||
|
||||
namespace mindspore::transform {
|
||||
DECLARE_OP_ADAPTER(CTCLoss)//将收录内容的类型与标准进行对比，后进行空间调整，并用指针input_map_为key存储相应内容
|
||||
DECLARE_OP_USE_OUTPUT(CTCLoss)//将收录内容的类型与标准进行对比，后进行空间调整，并用指针output_map_为key存储相应内容
|
||||
|
||||
DECLARE_OP_ADAPTER(CTCGreedyDecoder)
|
||||
DECLARE_OP_USE_OUTPUT(CTCGreedyDecoder)
|
||||
} // namespace mindspore::transform
|
||||
#endif // MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_CTC_OPS_DECLARE_H_
|
||||
|
|
@ -0,0 +1,65 @@
|
|||
/**
|
||||
* Copyright 2022 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#include "transform/graph_ir/op_declare/data_flow_ops_declare.h"
|
||||
#include <vector>
|
||||
|
||||
namespace mindspore::transform {
|
||||
INPUT_MAP(TensorArray) = {{1, INPUT_DESC(size)}};/*
|
||||
将value处理并存入对应ATTR_DESC结构体的相应变量中
|
||||
将value内容转为字符串变量并存储至结构体的name变量中
|
||||
引用Operator空间并将指针所指的类转为ATTR_DESC结构体
|
||||
将TensorArray的类型与标准进行对比，后进行空间调整，并用指针input_map_为key存储相应内容
|
||||
*/
|
||||
ATTR_MAP(TensorArray) = {{"dtype", ATTR_DESC(dtype, AnyTraits<GEType>())},/*
|
||||
将dtype处理并存入对应ATTR_DESC结构体的相应变量中
|
||||
将dtype内容转为字符串变量并存储至结构体的name变量中
|
||||
引用Operator空间并将指针所指的类转为ATTR_DESC结构体
|
||||
//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
//存入MaxPool对应空间并用attr_map_指针保存
|
||||
//其中AnyTraits<>的作用为将<>内类型进行构建*/
|
||||
{"element_shape", ATTR_DESC(element_shape, AnyTraits<std::vector<int64_t>>())},
|
||||
{"dynamic_size", ATTR_DESC(dynamic_size, AnyTraits<bool>())},
|
||||
{"clear_after_read", ATTR_DESC(clear_after_read, AnyTraits<bool>())},
|
||||
{"identical_element_shapes", ATTR_DESC(identical_element_shapes, AnyTraits<bool>())},
|
||||
{"tensor_array_name", ATTR_DESC(tensor_array_name, AnyTraits<std::string>())}};//将TensorArray的类型与标准进行对比，后进行空间调整，并用指针attr_map_为key存储相应内容
|
||||
OUTPUT_MAP(TensorArray) = {{0, OUTPUT_DESC(handle)}, {1, OUTPUT_DESC(flow)}};/*
|
||||
将handle和flow处理并存入对应ATTR_DESC结构体的相应变量中
|
||||
将变量内容转为字符串变量并存储至结构体的name变量中
|
||||
引用Operator空间并将指针所指的类转为OUTPUT_DESC结构体*/
|
||||
//将TensorArray的类型与标准进行对比，后进行空间调整，并用指针output_map_为key存储相应内容
|
||||
REG_ADPT_DESC(TensorArray, kNameTensorArray, ADPT_DESC(TensorArray))/*
|
||||
将TensorArray处理并存入对应ADPT_DESC结构体的相应变量中
|
||||
将TensorArray内容转为字符串变量并存储至结构体的name变量中
|
||||
引用Operator空间并将指针所指的类转为ADPT_DESC结构体
|
||||
|
||||
再将TensorArray处理并存入对应REG_ADPT_DESC结构体的相应变量中
|
||||
将TensorArray内容转为字符串变量并存储至REG_ADPT_DESC的结构体的name变量中
|
||||
引用Operator空间并将指针所指的类转为REG_ADPT_DESC结构体
|
||||
*/
|
||||
|
||||
INPUT_MAP(TensorArrayWrite) = {
|
||||
{1, INPUT_DESC(handle)}, {2, INPUT_DESC(index)}, {3, INPUT_DESC(value)}, {4, INPUT_DESC(flow_in)}};
|
||||
ATTR_MAP(TensorArrayWrite) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(TensorArrayWrite) = {{0, OUTPUT_DESC(flow_out)}};
|
||||
REG_ADPT_DESC(TensorArrayWrite, kNameTensorArrayWrite, ADPT_DESC(TensorArrayWrite))
|
||||
|
||||
INPUT_MAP(TensorArrayGather) = {{1, INPUT_DESC(handle)}, {2, INPUT_DESC(indices)}, {3, INPUT_DESC(flow_in)}};
|
||||
ATTR_MAP(TensorArrayGather) = {{"dtype", ATTR_DESC(dtype, AnyTraits<GEType>())},
|
||||
{"element_shape", ATTR_DESC(element_shape, AnyTraits<std::vector<int64_t>>())}};
|
||||
OUTPUT_MAP(TensorArrayGather) = {{0, OUTPUT_DESC(value)}};
|
||||
REG_ADPT_DESC(TensorArrayGather, kNameTensorArrayGather, ADPT_DESC(TensorArrayGather))
|
||||
} // namespace mindspore::transform
|
||||
|
|
@ -0,0 +1,35 @@
|
|||
/**
|
||||
* Copyright 2022 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_DATA_FLOW_OPS_DECLARE_H_
|
||||
#define MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_DATA_FLOW_OPS_DECLARE_H_
|
||||
|
||||
#include <string>
|
||||
#include <unordered_map>
|
||||
#include "transform/graph_ir/op_declare/op_declare_macro.h"
|
||||
#include "ops/data_flow_ops.h"
|
||||
|
||||
namespace mindspore::transform {
|
||||
DECLARE_OP_ADAPTER(TensorArray)//将TensorArray收录后与标准进行比较，进行空间调整，用input_map_为key储存
|
||||
DECLARE_OP_USE_OUTPUT(TensorArray)//将TensorArray收录后与标准进行比较，进行空间调整，用output_map_为key储存
|
||||
|
||||
DECLARE_OP_ADAPTER(TensorArrayWrite)
|
||||
DECLARE_OP_USE_OUTPUT(TensorArrayWrite)
|
||||
|
||||
DECLARE_OP_ADAPTER(TensorArrayGather)
|
||||
DECLARE_OP_USE_OUTPUT(TensorArrayGather)
|
||||
} // namespace mindspore::transform
|
||||
#endif // MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_DATA_FLOW_OPS_DECLARE_H_
|
||||
|
|
@ -0,0 +1,305 @@
|
|||
/**
|
||||
* Copyright 2019 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#include "include/transform/graph_ir/df_graph_manager.h"
|
||||
|
||||
#include <sstream>
|
||||
|
||||
#ifndef ENABLE_LITE_ACL
|
||||
#include "include/common/utils/python_adapter.h"
|
||||
#include "pipeline/jit/pipeline.h"
|
||||
#endif
|
||||
#ifndef NO_DLIB
|
||||
#include "tdt/tsd_client.h"
|
||||
#endif
|
||||
|
||||
namespace mindspore {
|
||||
namespace transform {
|
||||
|
||||
//在DfGraphWrapper区域中定义名为DfGraphWrapper的函数
|
||||
DfGraphWrapper::DfGraphWrapper(const std::string &name, const int &id, const DfGraphPtr &graph_ptr,
|
||||
const OptionMap &options)
|
||||
: name_(name), id_(id), graph_ptr_(graph_ptr), options_(options) {}
|
||||
|
||||
//在DfGraphWrapper区域中定义名为DfGraphManager的函数
|
||||
//其功能主要为令将其中的三个变量“清空”
|
||||
DfGraphManager::DfGraphManager() {
|
||||
graph_id_ = 0;
|
||||
graph_runner_ptr_ = nullptr;
|
||||
sess_ptr_ = nullptr;
|
||||
}
|
||||
|
||||
//在DfGraphManager区域中定义名为~DfGraphManager的函数
|
||||
/*
|
||||
分别执行 DeleteGraphRunner();三个函数
|
||||
DeleteGeSession();
|
||||
ClearGraph();
|
||||
其次判断ENABLE_LITE_ACL是否被定义，若已经被定义则执行set_python_env_flag函数的操作，并结束条件命令
|
||||
*/
|
||||
DfGraphManager::~DfGraphManager() {
|
||||
// in python first destroy after atexit but in c++ destroy before atexit
|
||||
//在python中，destory在atexit之后，而在c++中，destory在atexit之前
|
||||
DeleteGraphRunner();
|
||||
DeleteGeSession();
|
||||
ClearGraph();
|
||||
#ifndef ENABLE_LITE_ACL
|
||||
python_adapter::set_python_env_flag(false);
|
||||
#endif
|
||||
}
|
||||
|
||||
//在DfGraphManager区域中定义名为GetInstance的函数
|
||||
//返回instance
|
||||
DfGraphManager &DfGraphManager::GetInstance() {
|
||||
static DfGraphManager instance;
|
||||
return instance;
|
||||
}
|
||||
|
||||
//在DfGraphWrapper区域中定义名为GenerateId的函数，返回值为int型
|
||||
/*
|
||||
令graph_id_自增1，其次判断其大小是否小于等于0，若是，则令其等于1（检验DfGraphManager中的语句是否成功执行）
|
||||
输出消息日志"Generate graph Id :[*此处为graph_id_]"
|
||||
并返回graph_id_
|
||||
*/
|
||||
int DfGraphManager::GenerateId() {
|
||||
graph_id_++;
|
||||
if (graph_id_ <= 0) {
|
||||
graph_id_ = 1;
|
||||
}
|
||||
MS_LOG(INFO) << "Generate graph Id : " << graph_id_;
|
||||
return graph_id_;
|
||||
}
|
||||
|
||||
//在DfGraphManager区域中定义名为AddGraph的函数，返回值为自定义类型Status
|
||||
/*
|
||||
判断name是否为空，若为空则输出错误日志"The graph name is null, add graph failed"并返回INVALID_ARGUMENT
|
||||
判断graph_ptr是否为空，若为空则输出消息日志"The new graph [*此处为name]'s pointer is null, add graph failed"并返回INVALID_ARGUMENT
|
||||
定义id，wrap_ptr，ret，运用函数进行name, id, graph_ptr, options等数据的组合存储，建立新的shared_ptr
|
||||
判断ret的第二位是否为false，若是，则代表其已经存在并将被覆写，输出相应警告日志并将其错位储存
|
||||
最后输出消息日志"Add graph "[*此处为name]" to GraphManager success!"并返回return Status::SUCCESS
|
||||
*/
|
||||
//本函数主要执行已有数据的组合重建成新的graph操作，并可检验操作是否成功
|
||||
Status DfGraphManager::AddGraph(const std::string &name, const DfGraphPtr &graph_ptr, const OptionMap &options) {
|
||||
std::lock_guard<std::mutex> lg(lock_);
|
||||
if (name.empty()) {
|
||||
MS_LOG(ERROR) << "The graph name is null, add graph failed";
|
||||
return Status::INVALID_ARGUMENT;
|
||||
}
|
||||
|
||||
if (graph_ptr == nullptr) {
|
||||
MS_LOG(INFO) << "The new graph {" << name << "}'s pointer is null, add graph failed";
|
||||
return Status::INVALID_ARGUMENT;
|
||||
}
|
||||
|
||||
int id = GenerateId();
|
||||
DfGraphWrapperPtr wrap_ptr = std::make_shared<DfGraphWrapper>(name, id, graph_ptr, options);
|
||||
auto ret = graphs_.emplace(name, wrap_ptr);
|
||||
if (ret.second == false) {
|
||||
MS_LOG(WARNING) << "The graph name:{ " << name << " }is already exists! The old graph will be overwritten!!";
|
||||
ret.first->second = wrap_ptr;
|
||||
}
|
||||
MS_LOG(INFO) << "Add graph " << name << " to GraphManager success!";
|
||||
return Status::SUCCESS;
|
||||
}
|
||||
|
||||
//在DfGraphManager区域中定义名为GetAllGraphs的函数，返回值为自定义类型std::vector<DfGraphWrapperPtr>
|
||||
/*
|
||||
令it在从第一位开始在图表范围内循环，用ss和ret储存it储存的图表内容的不同位置的数据，并用"{}"作为ss的开头和结尾
|
||||
输出消息日志"Return graphs:[*此处为ss构成的图标]"
|
||||
返回ret
|
||||
*/
|
||||
std::vector<DfGraphWrapperPtr> DfGraphManager::GetAllGraphs() {
|
||||
std::lock_guard<std::mutex> lg(lock_);
|
||||
std::vector<DfGraphWrapperPtr> ret;
|
||||
std::stringstream ss;
|
||||
ss << "{ ";
|
||||
for (auto it = graphs_.begin(); it != graphs_.end(); ++it) {
|
||||
ss << it->first << ", ";
|
||||
ret.emplace_back(it->second);
|
||||
}
|
||||
ss << "}";
|
||||
MS_LOG(INFO) << "Return graphs: " << ss.str();
|
||||
return ret;
|
||||
}
|
||||
|
||||
//在DfGraphManager区域中定义名为GetSavedGraphs的函数，返回值为std::set<string>
|
||||
//功能为返回saved_graphs_
|
||||
std::set<string> DfGraphManager::GetSavedGraphs() { return saved_graphs_; }
|
||||
|
||||
//在DfGraphManager区域中定义名为AddSavedGraphs的函数
|
||||
//功能为将id插入到saved_graphs_中
|
||||
void DfGraphManager::AddSavedGraphs(const std::string &id) { saved_graphs_.insert(id); }
|
||||
|
||||
//在DfGraphManager区域中定义名为GetGraphByName的函数，返回值为自定义类型DfGraphWrapperPtr
|
||||
/*
|
||||
判断name是否为空，若是则输出错误日志 "The graph name is null"并返回nullptr
|
||||
定义it并用其寻找储存graph的name，若其在graph中找不到name，则输出消息日志"Can't found graph name:[*此处为name]"并返回nullptr
|
||||
输出消息日志"Return graph:[*此处为name]"
|
||||
返回it->second
|
||||
*/
|
||||
DfGraphWrapperPtr DfGraphManager::GetGraphByName(const std::string &name) {
|
||||
std::lock_guard<std::mutex> lg(lock_);
|
||||
if (name.empty()) {
|
||||
MS_LOG(ERROR) << "The graph name is null";
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
auto it = graphs_.find(name);
|
||||
if (it == graphs_.end()) {
|
||||
MS_LOG(INFO) << "Can't found graph name: " << name;
|
||||
return nullptr;
|
||||
}
|
||||
MS_LOG(INFO) << "Return graph: " << name;
|
||||
return it->second;
|
||||
}
|
||||
|
||||
//在DfGraphManager区域中定义名为ClearGraph的函数，且不抛出异常
|
||||
/*
|
||||
清空graphs_和anf_graphs_并输出消息日志 "Remove all graphs in GraphManager"
|
||||
*/
|
||||
void DfGraphManager::ClearGraph() noexcept {
|
||||
std::lock_guard<std::mutex> lg(lock_);
|
||||
graphs_.clear();
|
||||
anf_graphs_.clear();
|
||||
MS_LOG(INFO) << "Remove all graphs in GraphManager";
|
||||
}
|
||||
|
||||
//在DfGraphManager区域中定义名为SetAnfGraph的函数
|
||||
/*
|
||||
获取name属性并储存在df_graph变量中
|
||||
判断其是否为空，若是则输出错误日志"Can't found graph name:[*此处为name]"
|
||||
将anf_graph_ptr存入anf_graphs_的[df_graph->id_]位置
|
||||
*/
|
||||
void DfGraphManager::SetAnfGraph(const std::string &name, const AnfGraphPtr &anf_graph_ptr) {
|
||||
DfGraphWrapperPtr df_graph = GetGraphByName(name);
|
||||
if (df_graph == nullptr) {
|
||||
MS_LOG(ERROR) << "Can't found graph name: " << name;
|
||||
return;
|
||||
}
|
||||
std::lock_guard<std::mutex> lg(lock_);
|
||||
anf_graphs_[df_graph->id_] = anf_graph_ptr;
|
||||
}
|
||||
|
||||
//在DfGraphManager区域中定义名为GetAnfGraph的函数，返回值为自定义类型AnfGraphPtr
|
||||
/*
|
||||
定义iter寻找graph_id，若找不到则输出错误日志"Can't found anf graph, graph_id =[*此处为graph_id]"并返回nullptr
|
||||
返回iter->second
|
||||
*/
|
||||
AnfGraphPtr DfGraphManager::GetAnfGraph(uint32_t graph_id) {
|
||||
std::lock_guard<std::mutex> lg(lock_);
|
||||
auto iter = anf_graphs_.find(graph_id);
|
||||
if (iter == anf_graphs_.end()) {
|
||||
MS_LOG(ERROR) << "Can't found anf graph, graph_id = " << graph_id;
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
return iter->second;
|
||||
}
|
||||
|
||||
//在DfGraphManager区域内定义名为EraseAnfGraph的函数
|
||||
//功能为清空anf_gtaphs_
|
||||
void DfGraphManager::EraseAnfGraph() {
|
||||
std::lock_guard<std::mutex> lg(lock_);
|
||||
anf_graphs_.clear();
|
||||
}
|
||||
|
||||
//在DfGraphManager区域内定义名为SetGeSession的函数
|
||||
/*
|
||||
检测sess_ptr是否为空，若是则输出错误日志"You are adding a empty Ge Session"
|
||||
检测sess_ptr_是否为空，若是则输出消息日志"Add a new Ge Session success"
|
||||
否则输出消息日志"Add a new Ge Session success, the old Ge Session will be overwritten!!"
|
||||
用sess_ptr_指针储存sess_ptr的地址
|
||||
*/
|
||||
void DfGraphManager::SetGeSession(const std::shared_ptr<ge::Session> &sess_ptr) {
|
||||
std::lock_guard<std::mutex> lg(lock_);
|
||||
if (sess_ptr == nullptr) {
|
||||
MS_LOG(WARNING) << "You are adding a empty Ge Session";
|
||||
}
|
||||
|
||||
if (sess_ptr_ == nullptr) {
|
||||
MS_LOG(INFO) << "Add a new Ge Session success";
|
||||
} else {
|
||||
MS_LOG(INFO) << "Add a new Ge Session success, the old Ge Session will be overwritten!!";
|
||||
}
|
||||
sess_ptr_ = sess_ptr;
|
||||
}
|
||||
|
||||
//在DfGraphManager区域内定义名为GetGeSession的函数，返回值为自定义类型std::shared_ptr<ge::Session>
|
||||
//函数功能为返回sess_ptr_
|
||||
std::shared_ptr<ge::Session> DfGraphManager::GetGeSession() {
|
||||
std::lock_guard<std::mutex> lg(lock_);
|
||||
return sess_ptr_;
|
||||
}
|
||||
|
||||
//在DfGraphManager区域内定义名为DeleteGeSession的函数
|
||||
/*
|
||||
检验sess_ptr_是否为空，若是则输出消息日志"Ge Session is not exist"
|
||||
否则令sess_ptr_为空并清空saved_graphs_，输出消息日志"Delete Ge Session success"
|
||||
*/
|
||||
void DfGraphManager::DeleteGeSession() noexcept {
|
||||
std::lock_guard<std::mutex> lg(lock_);
|
||||
if (sess_ptr_ == nullptr) {
|
||||
MS_LOG(INFO) << "Ge Session is not exist";
|
||||
} else {
|
||||
sess_ptr_ = nullptr;
|
||||
saved_graphs_.clear();
|
||||
MS_LOG(INFO) << "Delete Ge Session success";
|
||||
}
|
||||
}
|
||||
|
||||
//在DfGraphManager区域内定义名为SetGraphRunner的函数
|
||||
/*
|
||||
检验graph_runner_ptr是否为空，若为空则输出警告日志"You are adding a empty GraphRunner"
|
||||
检验graph_runner_ptr_是否为空，若是则输出消息日志"Add a new GraphRunner success"
|
||||
否则输出消息日志"Add a new GraphRunner success, the old GraphRunner will be overwritten!!"
|
||||
用graph_runner_ptr_指针储存graph_runner_ptr的地址
|
||||
*/
|
||||
void DfGraphManager::SetGraphRunner(const std::shared_ptr<transform::GraphRunner> &graph_runner_ptr) noexcept {
|
||||
std::lock_guard<std::mutex> lg(lock_);
|
||||
if (graph_runner_ptr == nullptr) {
|
||||
MS_LOG(WARNING) << "You are adding a empty GraphRunner";
|
||||
}
|
||||
|
||||
if (graph_runner_ptr_ == nullptr) {
|
||||
MS_LOG(INFO) << "Add a new GraphRunner success";
|
||||
} else {
|
||||
MS_LOG(INFO) << "Add a new GraphRunner success, the old GraphRunner will be overwritten!!";
|
||||
}
|
||||
graph_runner_ptr_ = graph_runner_ptr;
|
||||
}
|
||||
|
||||
//在DfGraphManager区域内定义名为GetGraphRunner的函数
|
||||
//功能为返回graph_runner_ptr_
|
||||
std::shared_ptr<transform::GraphRunner> DfGraphManager::GetGraphRunner() {
|
||||
std::lock_guard<std::mutex> lg(lock_);
|
||||
return graph_runner_ptr_;
|
||||
}
|
||||
|
||||
//在DfGraphManager区域内定义名为DeleteGraphRunner的函数
|
||||
/*
|
||||
检测graph_runner_ptr_是否为空，若是则输出消息日志"GraphRunner is not exist"
|
||||
否则令graph_runner_ptr_为空并输出消息日志"Delete GraphRunner success"
|
||||
*/
|
||||
void DfGraphManager::DeleteGraphRunner() noexcept {
|
||||
std::lock_guard<std::mutex> lg(lock_);
|
||||
if (graph_runner_ptr_ == nullptr) {
|
||||
MS_LOG(INFO) << "GraphRunner is not exist";
|
||||
} else {
|
||||
graph_runner_ptr_ = nullptr;
|
||||
MS_LOG(INFO) << "Delete GraphRunner success";
|
||||
}
|
||||
}
|
||||
} // namespace transform
|
||||
} // namespace mindspore
|
||||
|
|
@ -0,0 +1,674 @@
|
|||
/**
|
||||
* Copyright 2019-2021 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#include "transform/graph_ir/op_declare/elewise_calculation_ops_declare.h"
|
||||
#include <memory>
|
||||
#include <vector>
|
||||
|
||||
namespace mindspore::transform {
|
||||
// Assign
|
||||
INPUT_MAP(Assign) = {{1, INPUT_DESC(ref)}, {2, INPUT_DESC(value)}};/*
|
||||
将value和ref处理并存入对应INPUT_DESC结构体的相应变量中
|
||||
将value和ref内容转为字符串变量并存储至结构体的name变量中
|
||||
引用Operator空间并将指针所指的类转为INPUT_DESC结构体
|
||||
将Assign的类型与标准进行对比，后进行空间调整，并用指针input_map_为key存储相应内容
|
||||
*/
|
||||
ATTR_MAP(Assign) = EMPTY_ATTR_MAP;//将Assign与标准进行比较，使原以attr_map_为储存信息的key变为空
|
||||
OUTPUT_MAP(Assign) = {{0, OUTPUT_DESC(ref)}};/*
|
||||
将ref处理并存入对应OUTPUT_DESC结构体的相应变量中
|
||||
将ref内容转为字符串变量并存储至结构体的name变量中
|
||||
引用Operator空间并将指针所指的类转为OUTPUT_DESC结构体
|
||||
将Assign的类型与标准进行对比，后进行空间调整，并用指针output_map_为key存储相应内容
|
||||
*/
|
||||
REG_ADPT_DESC(Assign, prim::kPrimAssign->name(), ADPT_DESC(Assign))/*
|
||||
将Assign处理并存入对应ADPT_DESC结构体的相应变量中
|
||||
将Assign内容转为字符串变量并存储至结构体的name变量中
|
||||
引用Operator空间并将指针所指的类转为ADPT_DESC结构体
|
||||
|
||||
再将Assign处理并存入对应REG_ADPT_DESC结构体的相应变量中
|
||||
将Assign内容转为字符串变量并存储至REG_ADPT_DESC的结构体的name变量中
|
||||
引用Operator空间并将指针所指的类转为REG_ADPT_DESC结构体
|
||||
*/
|
||||
|
||||
REG_ADPT_DESC(StateSetItem, prim::kPrimStateSetItem->name(), ADPT_DESC(Assign))
|
||||
|
||||
// add
|
||||
INPUT_MAP(Add) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}};
|
||||
ATTR_MAP(Add) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(Add) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(Add, prim::kPrimAdd->name(),
|
||||
std::make_shared<OpAdapterDesc>(
|
||||
std::make_shared<OpAdapter<Add>>(ExtraAttr({{"mode", MakeValue(static_cast<int64_t>(1))}})),
|
||||
std::make_shared<OpAdapter<Add>>(ExtraAttr({{"mode", MakeValue(static_cast<int64_t>(1))}}))))
|
||||
|
||||
// AccumulateNV2
|
||||
INPUT_MAP(AccumulateNV2) = EMPTY_INPUT_MAP;
|
||||
DYN_INPUT_MAP(AccumulateNV2) = {{1, DYN_INPUT_DESC(x)}};
|
||||
ATTR_MAP(AccumulateNV2) = {{"n", ATTR_DESC(N, AnyTraits<int64_t>())}};
|
||||
OUTPUT_MAP(AccumulateNV2) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(AccumulateNV2, kNameAccumulateNV2, ADPT_DESC(AccumulateNV2))
|
||||
|
||||
// ConfusionMulGrad
|
||||
INPUT_MAP(ConfusionMulGrad) = {{1, INPUT_DESC(input0)}, {2, INPUT_DESC(input1)}, {3, INPUT_DESC(input2)}};
|
||||
ATTR_MAP(ConfusionMulGrad) = {{"axes", ATTR_DESC(axes, AnyTraits<std::vector<int64_t>>())},
|
||||
{"keep_dims", ATTR_DESC(keep_dims, AnyTraits<bool>())}};
|
||||
OUTPUT_MAP(ConfusionMulGrad) = {{0, OUTPUT_DESC(output0)}, {1, OUTPUT_DESC(output1)}};
|
||||
REG_ADPT_DESC(ConfusionMulGrad, kNameConfusionMulGrad, ADPT_DESC(ConfusionMulGrad))
|
||||
|
||||
// FakeQuantWithMinMaxVars
|
||||
INPUT_MAP(FakeQuantWithMinMaxVars) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(min)}, {3, INPUT_DESC(max)}};
|
||||
ATTR_MAP(FakeQuantWithMinMaxVars) = {{"num_bits", ATTR_DESC(num_bits, AnyTraits<int64_t>())},
|
||||
{"narrow_range", ATTR_DESC(narrow_range, AnyTraits<bool>())}};
|
||||
OUTPUT_MAP(FakeQuantWithMinMaxVars) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(FakeQuantWithMinMaxVars, kNameFakeQuantWithMinMaxVars, ADPT_DESC(FakeQuantWithMinMaxVars))
|
||||
|
||||
// FakeQuantWithMinMaxVarsGradient
|
||||
INPUT_MAP(FakeQuantWithMinMaxVarsGradient) = {
|
||||
{1, INPUT_DESC(gradients)}, {2, INPUT_DESC(x)}, {3, INPUT_DESC(min)}, {4, INPUT_DESC(max)}};
|
||||
ATTR_MAP(FakeQuantWithMinMaxVarsGradient) = {{"num_bits", ATTR_DESC(num_bits, AnyTraits<int64_t>())},
|
||||
{"narrow_range", ATTR_DESC(narrow_range, AnyTraits<bool>())}};
|
||||
OUTPUT_MAP(FakeQuantWithMinMaxVarsGradient) = {
|
||||
{0, OUTPUT_DESC(backprops_wrt_x)}, {1, OUTPUT_DESC(backprops_wrt_min)}, {2, OUTPUT_DESC(backprops_wrt_max)}};
|
||||
REG_ADPT_DESC(FakeQuantWithMinMaxVarsGradient, kNameFakeQuantWithMinMaxVarsGradient,
|
||||
ADPT_DESC(FakeQuantWithMinMaxVarsGradient))
|
||||
|
||||
// FakeQuantWithMinMaxVarsPerChannel
|
||||
INPUT_MAP(FakeQuantWithMinMaxVarsPerChannel) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(min)}, {3, INPUT_DESC(max)}};
|
||||
ATTR_MAP(FakeQuantWithMinMaxVarsPerChannel) = {{"num_bits", ATTR_DESC(num_bits, AnyTraits<int64_t>())},
|
||||
{"narrow_range", ATTR_DESC(narrow_range, AnyTraits<bool>())}};
|
||||
OUTPUT_MAP(FakeQuantWithMinMaxVarsPerChannel) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(FakeQuantWithMinMaxVarsPerChannel, kNameFakeQuantWithMinMaxVarsPerChannel,
|
||||
ADPT_DESC(FakeQuantWithMinMaxVarsPerChannel))
|
||||
|
||||
// FakeQuantWithMinMaxVarsPerChannelGradient
|
||||
INPUT_MAP(FakeQuantWithMinMaxVarsPerChannelGradient) = {
|
||||
{1, INPUT_DESC(gradients)}, {2, INPUT_DESC(x)}, {3, INPUT_DESC(min)}, {4, INPUT_DESC(max)}};
|
||||
ATTR_MAP(FakeQuantWithMinMaxVarsPerChannelGradient) = {{"num_bits", ATTR_DESC(num_bits, AnyTraits<int64_t>())},
|
||||
{"narrow_range", ATTR_DESC(narrow_range, AnyTraits<bool>())}};
|
||||
OUTPUT_MAP(FakeQuantWithMinMaxVarsPerChannelGradient) = {
|
||||
{0, OUTPUT_DESC(backprops_wrt_x)}, {1, OUTPUT_DESC(backprops_wrt_min)}, {2, OUTPUT_DESC(backprops_wrt_max)}};
|
||||
REG_ADPT_DESC(FakeQuantWithMinMaxVarsPerChannelGradient, kNameFakeQuantWithMinMaxVarsPerChannelGradient,
|
||||
ADPT_DESC(FakeQuantWithMinMaxVarsPerChannelGradient))
|
||||
|
||||
// GreaterEqual
|
||||
INPUT_MAP(GreaterEqual) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}};
|
||||
ATTR_MAP(GreaterEqual) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(GreaterEqual) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(GreaterEqual, kNameGreaterEqual, ADPT_DESC(GreaterEqual))
|
||||
|
||||
// AssignAdd
|
||||
INPUT_MAP(AssignAdd) = {{1, INPUT_DESC(ref)}, {2, INPUT_DESC(value)}};
|
||||
ATTR_MAP(AssignAdd) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(AssignAdd) = {{0, OUTPUT_DESC(ref)}};
|
||||
REG_ADPT_DESC(AssignAdd, kNameAssignAdd, ADPT_DESC(AssignAdd))
|
||||
|
||||
// AssignSub
|
||||
INPUT_MAP(AssignSub) = {{1, INPUT_DESC(var)}, {2, INPUT_DESC(value)}};
|
||||
ATTR_MAP(AssignSub) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(AssignSub) = {{0, OUTPUT_DESC(var)}};
|
||||
REG_ADPT_DESC(AssignSub, kNameAssignSub, ADPT_DESC(AssignSub))
|
||||
|
||||
// Cos
|
||||
INPUT_MAP(Cos) = {{1, INPUT_DESC(x)}};
|
||||
ATTR_MAP(Cos) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(Cos) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(Cos, kNameCos, ADPT_DESC(Cos))
|
||||
|
||||
// Cosh
|
||||
INPUT_MAP(Cosh) = {{1, INPUT_DESC(x)}};
|
||||
ATTR_MAP(Cosh) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(Cosh) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(Cosh, kNameCosh, ADPT_DESC(Cosh))
|
||||
|
||||
// Acos
|
||||
INPUT_MAP(Acos) = {{1, INPUT_DESC(x)}};
|
||||
ATTR_MAP(Acos) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(Acos) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(Acos, kNameACos, ADPT_DESC(Acos))
|
||||
|
||||
// AcosGrad
|
||||
INPUT_MAP(AcosGrad) = {{1, INPUT_DESC(y)}, {2, INPUT_DESC(dy)}};
|
||||
ATTR_MAP(AcosGrad) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(AcosGrad) = {{0, OUTPUT_DESC(z)}};
|
||||
REG_ADPT_DESC(AcosGrad, kNameACosGrad, ADPT_DESC(AcosGrad))
|
||||
|
||||
// Acosh
|
||||
INPUT_MAP(Acosh) = {{1, INPUT_DESC(x)}};
|
||||
ATTR_MAP(Acosh) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(Acosh) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(Acosh, kNameAcosh, ADPT_DESC(Acosh))
|
||||
|
||||
// AcoshGrad
|
||||
INPUT_MAP(AcoshGrad) = {{1, INPUT_DESC(y)}, {2, INPUT_DESC(dy)}};
|
||||
ATTR_MAP(AcoshGrad) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(AcoshGrad) = {{0, OUTPUT_DESC(z)}};
|
||||
REG_ADPT_DESC(AcoshGrad, kNameAcoshGrad, ADPT_DESC(AcoshGrad))
|
||||
|
||||
// Div
|
||||
INPUT_MAP(Div) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}};
|
||||
ATTR_MAP(Div) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(Div) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(Div, kNameDiv, ADPT_DESC(Div))
|
||||
|
||||
// TruncateDiv
|
||||
INPUT_MAP(TruncateDiv) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}};
|
||||
ATTR_MAP(TruncateDiv) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(TruncateDiv) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(TruncateDiv, kNameTruncateDiv, ADPT_DESC(TruncateDiv))
|
||||
|
||||
// TruncateMod
|
||||
INPUT_MAP(TruncateMod) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}};
|
||||
ATTR_MAP(TruncateMod) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(TruncateMod) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(TruncateMod, kNameTruncateMod, ADPT_DESC(TruncateMod))
|
||||
|
||||
// Xlogy
|
||||
INPUT_MAP(Xlogy) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}};
|
||||
ATTR_MAP(Xlogy) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(Xlogy) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(Xlogy, kNameXlogy, ADPT_DESC(Xlogy))
|
||||
|
||||
// DivNoNan
|
||||
INPUT_MAP(DivNoNan) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}};
|
||||
ATTR_MAP(DivNoNan) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(DivNoNan) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(DivNoNan, kNameDivNoNan, ADPT_DESC(DivNoNan))
|
||||
|
||||
// Floor
|
||||
INPUT_MAP(Floor) = {{1, INPUT_DESC(x)}};
|
||||
ATTR_MAP(Floor) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(Floor) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(Floor, kNameFloor, ADPT_DESC(Floor))
|
||||
|
||||
// FloorDiv
|
||||
INPUT_MAP(FloorDiv) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}};
|
||||
ATTR_MAP(FloorDiv) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(FloorDiv) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(FloorDiv, kNameFloorDiv, ADPT_DESC(FloorDiv))
|
||||
|
||||
// FloorMod
|
||||
INPUT_MAP(FloorMod) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}};
|
||||
ATTR_MAP(FloorMod) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(FloorMod) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(FloorMod, kNameFloorMod, ADPT_DESC(FloorMod))
|
||||
|
||||
// Sin
|
||||
INPUT_MAP(Sin) = {{1, INPUT_DESC(x)}};
|
||||
ATTR_MAP(Sin) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(Sin) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(Sin, kNameSin, ADPT_DESC(Sin))
|
||||
|
||||
// Sinh
|
||||
INPUT_MAP(Sinh) = {{1, INPUT_DESC(x)}};
|
||||
ATTR_MAP(Sinh) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(Sinh) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(Sinh, kNameSinh, ADPT_DESC(Sinh))
|
||||
|
||||
// Asin
|
||||
INPUT_MAP(Asin) = {{1, INPUT_DESC(x)}};
|
||||
ATTR_MAP(Asin) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(Asin) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(Asin, kNameAsin, ADPT_DESC(Asin))
|
||||
|
||||
// AsinGrad
|
||||
INPUT_MAP(AsinGrad) = {{1, INPUT_DESC(y)}, {2, INPUT_DESC(dy)}};
|
||||
ATTR_MAP(AsinGrad) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(AsinGrad) = {{0, OUTPUT_DESC(z)}};
|
||||
REG_ADPT_DESC(AsinGrad, kNameAsinGrad, ADPT_DESC(AsinGrad))
|
||||
|
||||
// Asinh
|
||||
INPUT_MAP(Asinh) = {{1, INPUT_DESC(x)}};
|
||||
ATTR_MAP(Asinh) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(Asinh) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(Asinh, kNameAsinh, ADPT_DESC(Asinh))
|
||||
|
||||
// AsinhGrad
|
||||
INPUT_MAP(AsinhGrad) = {{1, INPUT_DESC(y)}, {2, INPUT_DESC(dy)}};
|
||||
ATTR_MAP(AsinhGrad) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(AsinhGrad) = {{0, OUTPUT_DESC(z)}};
|
||||
REG_ADPT_DESC(AsinhGrad, kNameAsinhGrad, ADPT_DESC(AsinhGrad))
|
||||
|
||||
// BitwiseAnd
|
||||
INPUT_MAP(BitwiseAnd) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}};
|
||||
ATTR_MAP(BitwiseAnd) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(BitwiseAnd) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(BitwiseAnd, kNameBitwiseAnd, ADPT_DESC(BitwiseAnd))
|
||||
|
||||
// BitwiseOr
|
||||
INPUT_MAP(BitwiseOr) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}};
|
||||
ATTR_MAP(BitwiseOr) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(BitwiseOr) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(BitwiseOr, kNameBitwiseOr, ADPT_DESC(BitwiseOr))
|
||||
|
||||
// BitwiseXor
|
||||
INPUT_MAP(BitwiseXor) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}};
|
||||
ATTR_MAP(BitwiseXor) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(BitwiseXor) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(BitwiseXor, kNameBitwiseXor, ADPT_DESC(BitwiseXor))
|
||||
|
||||
// Ceil
|
||||
INPUT_MAP(Ceil) = {{1, INPUT_DESC(x)}};
|
||||
ATTR_MAP(Ceil) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(Ceil) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(Ceil, kNameCeil, ADPT_DESC(Ceil))
|
||||
|
||||
// CosineEmbeddingLoss
|
||||
INPUT_MAP(CosineEmbeddingLoss) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}, {3, INPUT_DESC(target)}};
|
||||
ATTR_MAP(CosineEmbeddingLoss) = {{"margin", ATTR_DESC(margin, AnyTraits<float>())},
|
||||
{"reduction", ATTR_DESC(reduction, AnyTraits<std::string>())}};
|
||||
OUTPUT_MAP(CosineEmbeddingLoss) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(CosineEmbeddingLoss, kNameCosineEmbeddingLoss, ADPT_DESC(CosineEmbeddingLoss))
|
||||
|
||||
// Xdivy
|
||||
INPUT_MAP(Xdivy) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}};
|
||||
ATTR_MAP(Xdivy) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(Xdivy) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(Xdivy, kNameXdivy, ADPT_DESC(Xdivy))
|
||||
|
||||
// Mod
|
||||
INPUT_MAP(Mod) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}};
|
||||
ATTR_MAP(Mod) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(Mod) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(Mod, kNameMod, ADPT_DESC(Mod))
|
||||
|
||||
// Exp
|
||||
INPUT_MAP(Exp) = {{1, INPUT_DESC(x)}};
|
||||
ATTR_MAP(Exp) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(Exp) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(Exp, kNameExp, ADPT_DESC(Exp))
|
||||
|
||||
// Expm1
|
||||
INPUT_MAP(Expm1) = {{1, INPUT_DESC(x)}};
|
||||
ATTR_MAP(Expm1) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(Expm1) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(Expm1, kNameExpm1, ADPT_DESC(Expm1))
|
||||
|
||||
// BiasAdd
|
||||
INPUT_MAP(BiasAdd) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(bias)}};
|
||||
ATTR_MAP(BiasAdd) = {{"format", ATTR_DESC(data_format, AnyTraits<std::string>())}};
|
||||
OUTPUT_MAP(BiasAdd) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(BiasAdd, kNameBiasAdd, ADPT_DESC(BiasAdd))
|
||||
|
||||
// ZerosLike
|
||||
INPUT_MAP(ZerosLike) = {{1, INPUT_DESC(x)}};
|
||||
ATTR_MAP(ZerosLike) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(ZerosLike) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(ZerosLike, kNameZerosLike, ADPT_DESC(ZerosLike))
|
||||
|
||||
// OnesLike
|
||||
INPUT_MAP(OnesLike) = {{1, INPUT_DESC(x)}};
|
||||
ATTR_MAP(OnesLike) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(OnesLike) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(OnesLike, kNameOnesLike, ADPT_DESC(OnesLike))
|
||||
|
||||
// ArgMaxD
|
||||
INPUT_MAP(ArgMaxD) = {{1, INPUT_DESC(x)}};
|
||||
ATTR_MAP(ArgMaxD) = {{"axis", ATTR_DESC(dimension, AnyTraits<int64_t>())},
|
||||
{"output_type", ATTR_DESC(dtype, AnyTraits<GEType>())}};
|
||||
OUTPUT_MAP(ArgMaxD) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(ArgMaxD, kNameArgmax, ADPT_DESC(ArgMaxD))
|
||||
|
||||
// ArgMaxV2
|
||||
INPUT_MAP(ArgMaxV2) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(dimension)}};
|
||||
ATTR_MAP(ArgMaxV2) = {{"output_type", ATTR_DESC(dtype, AnyTraits<GEType>())}};
|
||||
OUTPUT_MAP(ArgMaxV2) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(ArgMaxV2, kNameArgMaxV2, ADPT_DESC(ArgMaxV2))
|
||||
|
||||
// ArgMinD
|
||||
INPUT_MAP(ArgMinD) = {{1, INPUT_DESC(x)}};
|
||||
ATTR_MAP(ArgMinD) = {{"axis", ATTR_DESC(dimension, AnyTraits<int64_t>())},
|
||||
{"output_type", ATTR_DESC(dtype, AnyTraits<GEType>())}};
|
||||
OUTPUT_MAP(ArgMinD) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(ArgMinD, kNameArgmin, ADPT_DESC(ArgMinD))
|
||||
|
||||
// ArgMaxWithValue
|
||||
INPUT_MAP(ArgMaxWithValue) = {{1, INPUT_DESC(x)}};
|
||||
ATTR_MAP(ArgMaxWithValue) = {{"axis", ATTR_DESC(dimension, AnyTraits<int64_t>())},
|
||||
{"keep_dims", ATTR_DESC(keep_dims, AnyTraits<bool>())}};
|
||||
OUTPUT_MAP(ArgMaxWithValue) = {{0, OUTPUT_DESC(indice)}, {1, OUTPUT_DESC(values)}};
|
||||
REG_ADPT_DESC(ArgMaxWithValue, kNameArgMaxWithValue, ADPT_DESC(ArgMaxWithValue))
|
||||
|
||||
// ArgMinWithValue
|
||||
INPUT_MAP(ArgMinWithValue) = {{1, INPUT_DESC(x)}};
|
||||
ATTR_MAP(ArgMinWithValue) = {{"axis", ATTR_DESC(dimension, AnyTraits<int64_t>())},
|
||||
{"keep_dims", ATTR_DESC(keep_dims, AnyTraits<bool>())}};
|
||||
OUTPUT_MAP(ArgMinWithValue) = {{0, OUTPUT_DESC(indice)}, {1, OUTPUT_DESC(values)}};
|
||||
REG_ADPT_DESC(ArgMinWithValue, kNameArgMinWithValue, ADPT_DESC(ArgMinWithValue))
|
||||
|
||||
// Rint
|
||||
INPUT_MAP(Rint) = {{1, INPUT_DESC(x)}};
|
||||
ATTR_MAP(Rint) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(Rint) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(Rint, kNameRint, ADPT_DESC(Rint))
|
||||
|
||||
// BesselI0e
|
||||
INPUT_MAP(BesselI0e) = {{1, INPUT_DESC(x)}};
|
||||
ATTR_MAP(BesselI0e) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(BesselI0e) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(BesselI0e, kNameBesselI0e, ADPT_DESC(BesselI0e))
|
||||
|
||||
// BesselI1e
|
||||
INPUT_MAP(BesselI1e) = {{1, INPUT_DESC(x)}};
|
||||
ATTR_MAP(BesselI1e) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(BesselI1e) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(BesselI1e, kNameBesselI1e, ADPT_DESC(BesselI1e))
|
||||
|
||||
// Inv
|
||||
INPUT_MAP(Inv) = {{1, INPUT_DESC(x)}};
|
||||
ATTR_MAP(Inv) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(Inv) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(Inv, kNameInv, ADPT_DESC(Inv))
|
||||
|
||||
// InvGrad
|
||||
INPUT_MAP(InvGrad) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(grad)}};
|
||||
ATTR_MAP(InvGrad) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(InvGrad) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(InvGrad, kNameInvGrad, ADPT_DESC(InvGrad))
|
||||
|
||||
// Invert
|
||||
INPUT_MAP(Invert) = {{1, INPUT_DESC(x)}};
|
||||
ATTR_MAP(Invert) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(Invert) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(Invert, kNameInvert, ADPT_DESC(Invert))
|
||||
|
||||
// Log1p
|
||||
INPUT_MAP(Log1p) = {{1, INPUT_DESC(x)}};
|
||||
ATTR_MAP(Log1p) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(Log1p) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(Log1p, kNameLog1p, ADPT_DESC(Log1p))
|
||||
|
||||
// RsqrtGrad
|
||||
INPUT_MAP(RsqrtGrad) = {{1, INPUT_DESC(y)}, {2, INPUT_DESC(dy)}};
|
||||
ATTR_MAP(RsqrtGrad) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(RsqrtGrad) = {{0, OUTPUT_DESC(z)}};
|
||||
REG_ADPT_DESC(RsqrtGrad, kNameRsqrtGrad, ADPT_DESC(RsqrtGrad))
|
||||
|
||||
// SqrtGrad
|
||||
INPUT_MAP(SqrtGrad) = {{1, INPUT_DESC(y)}, {2, INPUT_DESC(dy)}};
|
||||
ATTR_MAP(SqrtGrad) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(SqrtGrad) = {{0, OUTPUT_DESC(z)}};
|
||||
REG_ADPT_DESC(SqrtGrad, kNameSqrtGrad, ADPT_DESC(SqrtGrad))
|
||||
|
||||
// ReciprocalGrad
|
||||
INPUT_MAP(ReciprocalGrad) = {{1, INPUT_DESC(y)}, {2, INPUT_DESC(dy)}};
|
||||
ATTR_MAP(ReciprocalGrad) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(ReciprocalGrad) = {{0, OUTPUT_DESC(z)}};
|
||||
REG_ADPT_DESC(ReciprocalGrad, kNameReciprocalGrad, ADPT_DESC(ReciprocalGrad))
|
||||
|
||||
// AddN
|
||||
INPUT_MAP(AddN) = EMPTY_INPUT_MAP;
|
||||
DYN_INPUT_MAP(AddN) = {{1, DYN_INPUT_DESC(x)}};
|
||||
ATTR_MAP(AddN) = {{"n", ATTR_DESC(N, AnyTraits<int64_t>())}};
|
||||
OUTPUT_MAP(AddN) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(AddN, kNameAddN, ADPT_DESC(AddN))
|
||||
|
||||
// Mul
|
||||
INPUT_MAP(Mul) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}};
|
||||
ATTR_MAP(Mul) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(Mul) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(Mul, prim::kPrimMul->name(), ADPT_DESC(Mul))
|
||||
|
||||
// MulNoNan
|
||||
INPUT_MAP(MulNoNan) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}};
|
||||
ATTR_MAP(MulNoNan) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(MulNoNan) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(MulNoNan, kNameMulNoNan, ADPT_DESC(MulNoNan))
|
||||
|
||||
// RealDiv
|
||||
INPUT_MAP(RealDiv) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}};
|
||||
ATTR_MAP(RealDiv) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(RealDiv) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(RealDiv, kNameRealDiv, ADPT_DESC(RealDiv))
|
||||
|
||||
// Cast
|
||||
INPUT_MAP(Cast) = {{1, INPUT_DESC(x)}};
|
||||
INPUT_ATTR_MAP(Cast) = {{2, ATTR_DESC(dst_type, AnyTraits<GEType>())}};
|
||||
ATTR_MAP(Cast) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(Cast) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(Cast, prim::kPrimCast->name(), ADPT_DESC(Cast))
|
||||
|
||||
// Reciprocal
|
||||
INPUT_MAP(Reciprocal) = {{1, INPUT_DESC(x)}};
|
||||
ATTR_MAP(Reciprocal) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(Reciprocal) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(Reciprocal, kNameReciprocal, ADPT_DESC(Reciprocal))
|
||||
|
||||
// Sub
|
||||
INPUT_MAP(Sub) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}};
|
||||
ATTR_MAP(Sub) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(Sub) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(Sub, prim::kPrimSub->name(), ADPT_DESC(Sub))
|
||||
|
||||
// Neg
|
||||
INPUT_MAP(Neg) = {{1, INPUT_DESC(x)}};
|
||||
ATTR_MAP(Neg) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(Neg) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(Neg, prim::kPrimNeg->name(), ADPT_DESC(Neg))
|
||||
|
||||
// Less
|
||||
INPUT_MAP(Less) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}};
|
||||
ATTR_MAP(Less) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(Less) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(Less, kNameLess, ADPT_DESC(Less))
|
||||
|
||||
// Rsqrt
|
||||
INPUT_MAP(Rsqrt) = {{1, INPUT_DESC(x)}};
|
||||
ATTR_MAP(Rsqrt) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(Rsqrt) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(Rsqrt, kNameRsqrt, ADPT_DESC(Rsqrt))
|
||||
|
||||
// Sqrt
|
||||
INPUT_MAP(Sqrt) = {{1, INPUT_DESC(x)}};
|
||||
ATTR_MAP(Sqrt) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(Sqrt) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(Sqrt, kNameSqrt, ADPT_DESC(Sqrt))
|
||||
|
||||
// Square
|
||||
INPUT_MAP(Square) = {{1, INPUT_DESC(x)}};
|
||||
ATTR_MAP(Square) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(Square) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(Square, kNameSquare, ADPT_DESC(Square))
|
||||
|
||||
// SquaredDifference
|
||||
INPUT_MAP(SquaredDifference) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}};
|
||||
ATTR_MAP(SquaredDifference) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(SquaredDifference) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(SquaredDifference, kNameSquaredDifference, ADPT_DESC(SquaredDifference))
|
||||
|
||||
// SquareSumAll
|
||||
INPUT_MAP(SquareSumAll) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}};
|
||||
ATTR_MAP(SquareSumAll) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(SquareSumAll) = {{0, OUTPUT_DESC(y1)}, {1, OUTPUT_DESC(y2)}};
|
||||
REG_ADPT_DESC(SquareSumAll, kNameSquareSumAll, ADPT_DESC(SquareSumAll))
|
||||
|
||||
// Maximum
|
||||
INPUT_MAP(Maximum) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}};
|
||||
ATTR_MAP(Maximum) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(Maximum) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(Maximum, prim::kPrimMaximum->name(), ADPT_DESC(Maximum))
|
||||
|
||||
// Minimum
|
||||
INPUT_MAP(Minimum) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}};
|
||||
ATTR_MAP(Minimum) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(Minimum) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(Minimum, prim::kPrimMinimum->name(), ADPT_DESC(Minimum))
|
||||
|
||||
// MaximumGrad
|
||||
INPUT_MAP(MaximumGrad) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}, {3, INPUT_DESC(grads)}};
|
||||
ATTR_MAP(MaximumGrad) = {{"grad_x", ATTR_DESC(grad_x, AnyTraits<bool>())},
|
||||
{"grad_y", ATTR_DESC(grad_y, AnyTraits<bool>())}};
|
||||
OUTPUT_MAP(MaximumGrad) = {{0, OUTPUT_DESC(y1)}, {1, OUTPUT_DESC(y2)}};
|
||||
REG_ADPT_DESC(MaximumGrad, prim::kPrimMaximumGrad->name(), ADPT_DESC(MaximumGrad))
|
||||
|
||||
// MinimumGrad
|
||||
INPUT_MAP(MinimumGrad) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}, {3, INPUT_DESC(grads)}};
|
||||
ATTR_MAP(MinimumGrad) = {{"grad_x", ATTR_DESC(grad_x, AnyTraits<bool>())},
|
||||
{"grad_y", ATTR_DESC(grad_y, AnyTraits<bool>())}};
|
||||
OUTPUT_MAP(MinimumGrad) = {{0, OUTPUT_DESC(y1)}, {1, OUTPUT_DESC(y2)}};
|
||||
REG_ADPT_DESC(MinimumGrad, prim::kPrimMinimumGrad->name(), ADPT_DESC(MinimumGrad))
|
||||
|
||||
// Pow
|
||||
INPUT_MAP(Pow) = {
|
||||
{1, INPUT_DESC(x1)},
|
||||
{2, INPUT_DESC(x2)},
|
||||
};
|
||||
ATTR_MAP(Pow) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(Pow) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(Pow, kNamePow, ADPT_DESC(Pow))
|
||||
|
||||
// PopulationCount
|
||||
INPUT_MAP(PopulationCount) = {{1, INPUT_DESC(x)}};
|
||||
ATTR_MAP(PopulationCount) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(PopulationCount) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(PopulationCount, kNamePopulationCount, ADPT_DESC(PopulationCount))
|
||||
|
||||
// Equal
|
||||
INPUT_MAP(Equal) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}};
|
||||
ATTR_MAP(Equal) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(Equal) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(Equal, kNameEqual, ADPT_DESC(Equal))
|
||||
|
||||
// ApproximateEqual
|
||||
INPUT_MAP(ApproximateEqual) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}};
|
||||
ATTR_MAP(ApproximateEqual) = {{"tolerance", ATTR_DESC(tolerance, AnyTraits<float>())}};
|
||||
OUTPUT_MAP(ApproximateEqual) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(ApproximateEqual, kNameApproximateEqual, ADPT_DESC(ApproximateEqual))
|
||||
|
||||
// NotEqual
|
||||
INPUT_MAP(NotEqual) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}};
|
||||
ATTR_MAP(NotEqual) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(NotEqual) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(NotEqual, kNameNotEqual, ADPT_DESC(NotEqual))
|
||||
|
||||
// Log
|
||||
INPUT_MAP(Log) = {{1, INPUT_DESC(x)}};
|
||||
ATTR_MAP(Log) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(Log) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(Log, kNameLog, ADPT_DESC(Log))
|
||||
|
||||
// LogicalAnd
|
||||
INPUT_MAP(LogicalAnd) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}};
|
||||
ATTR_MAP(LogicalAnd) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(LogicalAnd) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(LogicalAnd, kNameLogicalAnd, ADPT_DESC(LogicalAnd))
|
||||
|
||||
// LogicalOr
|
||||
INPUT_MAP(LogicalOr) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}};
|
||||
ATTR_MAP(LogicalOr) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(LogicalOr) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(LogicalOr, kNameLogicalOr, ADPT_DESC(LogicalOr))
|
||||
|
||||
// LogicalNot
|
||||
INPUT_MAP(LogicalNot) = {{1, INPUT_DESC(x)}};
|
||||
ATTR_MAP(LogicalNot) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(LogicalNot) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(LogicalNot, kNameLogicalNot, ADPT_DESC(LogicalNot))
|
||||
|
||||
// Greater
|
||||
INPUT_MAP(Greater) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}};
|
||||
ATTR_MAP(Greater) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(Greater) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(Greater, kNameGreater, ADPT_DESC(Greater))
|
||||
|
||||
// LessEqual
|
||||
INPUT_MAP(LessEqual) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}};
|
||||
ATTR_MAP(LessEqual) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(LessEqual) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(LessEqual, kNameLessEqual, ADPT_DESC(LessEqual))
|
||||
|
||||
// Abs
|
||||
INPUT_MAP(Abs) = {{1, INPUT_DESC(x)}};
|
||||
ATTR_MAP(Abs) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(Abs) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(Abs, kNameAbs, ADPT_DESC(Abs))
|
||||
|
||||
// AbsGrad
|
||||
INPUT_MAP(AbsGrad) = {{1, INPUT_DESC(y)}, {2, INPUT_DESC(dy)}};
|
||||
ATTR_MAP(AbsGrad) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(AbsGrad) = {{0, OUTPUT_DESC(z)}};
|
||||
REG_ADPT_DESC(AbsGrad, kNameAbsGrad, ADPT_DESC(AbsGrad))
|
||||
|
||||
// Sign
|
||||
INPUT_MAP(Sign) = {{1, INPUT_DESC(x)}};
|
||||
ATTR_MAP(Sign) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(Sign) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(Sign, kNameSign, ADPT_DESC(Sign))
|
||||
|
||||
// Round
|
||||
INPUT_MAP(Round) = {{1, INPUT_DESC(x)}};
|
||||
ATTR_MAP(Round) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(Round) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(Round, kNameRound, ADPT_DESC(Round))
|
||||
|
||||
// Tan
|
||||
INPUT_MAP(Tan) = {{1, INPUT_DESC(x)}};
|
||||
ATTR_MAP(Tan) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(Tan) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(Tan, kNameTan, ADPT_DESC(Tan))
|
||||
|
||||
// Atan
|
||||
INPUT_MAP(Atan) = {{1, INPUT_DESC(x)}};
|
||||
ATTR_MAP(Atan) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(Atan) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(Atan, kNameAtan, ADPT_DESC(Atan))
|
||||
|
||||
// AtanGrad
|
||||
INPUT_MAP(AtanGrad) = {{1, INPUT_DESC(y)}, {2, INPUT_DESC(dy)}};
|
||||
ATTR_MAP(AtanGrad) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(AtanGrad) = {{0, OUTPUT_DESC(z)}};
|
||||
REG_ADPT_DESC(AtanGrad, kNameAtanGrad, ADPT_DESC(AtanGrad))
|
||||
|
||||
// Atanh
|
||||
INPUT_MAP(Atanh) = {{1, INPUT_DESC(x)}};
|
||||
ATTR_MAP(Atanh) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(Atanh) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(Atanh, kNameAtanh, ADPT_DESC(Atanh))
|
||||
|
||||
// Atan2
|
||||
INPUT_MAP(Atan2) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}};
|
||||
ATTR_MAP(Atan2) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(Atan2) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(Atan2, kNameAtan2, ADPT_DESC(Atan2))
|
||||
|
||||
// LambApplyOptimizerAssign
|
||||
INPUT_MAP(LambApplyOptimizerAssign) = {
|
||||
{1, INPUT_DESC(grad)}, {2, INPUT_DESC(inputv)}, {3, INPUT_DESC(inputm)},
|
||||
{4, INPUT_DESC(input3)}, {5, INPUT_DESC(mul0_x)}, {6, INPUT_DESC(mul1_x)},
|
||||
{7, INPUT_DESC(mul2_x)}, {8, INPUT_DESC(mul3_x)}, {9, INPUT_DESC(add2_y)},
|
||||
{10, INPUT_DESC(steps)}, {11, INPUT_DESC(do_use_weight)}, {12, INPUT_DESC(weight_decay_rate)}};
|
||||
ATTR_MAP(LambApplyOptimizerAssign) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(LambApplyOptimizerAssign) = {{0, OUTPUT_DESC(output0)}, {1, OUTPUT_DESC(inputv)}, {2, OUTPUT_DESC(inputm)}};
|
||||
REG_ADPT_DESC(LambApplyOptimizerAssign, kNameLambApplyOptimizerAssign, ADPT_DESC(LambApplyOptimizerAssign))
|
||||
|
||||
// LambApplyWeightAssign
|
||||
INPUT_MAP(LambApplyWeightAssign) = {{1, INPUT_DESC(input0)},
|
||||
{2, INPUT_DESC(input1)},
|
||||
{3, INPUT_DESC(input2)},
|
||||
{4, INPUT_DESC(input3)},
|
||||
{5, INPUT_DESC(input_param)}};
|
||||
ATTR_MAP(LambApplyWeightAssign) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(LambApplyWeightAssign) = {{0, OUTPUT_DESC(input_param)}};
|
||||
REG_ADPT_DESC(LambApplyWeightAssign, kNameLambApplyWeightAssign, ADPT_DESC(LambApplyWeightAssign))
|
||||
|
||||
// Eltwise
|
||||
INPUT_MAP(Eltwise) = EMPTY_INPUT_MAP;
|
||||
DYN_INPUT_MAP(Eltwise) = {{1, DYN_INPUT_DESC(x)}};
|
||||
ATTR_MAP(Eltwise) = {{"n", ATTR_DESC(N, AnyTraits<int64_t>())},
|
||||
{"mode", ATTR_DESC(mode, AnyTraits<int64_t>())},
|
||||
{"coeff", ATTR_DESC(coeff, AnyTraits<std::vector<float>>(), AnyTraits<float>())}};
|
||||
OUTPUT_MAP(Eltwise) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(Eltwise, kNameEltwise, ADPT_DESC(Eltwise))
|
||||
} // namespace mindspore::transform
|
||||
|
|
@ -0,0 +1,326 @@
|
|||
/**
|
||||
* Copyright 2019-2021 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_ELEWISE_CALCULATION_OPS_DECLARE_H_
|
||||
#define MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_ELEWISE_CALCULATION_OPS_DECLARE_H_
|
||||
|
||||
#include <string>
|
||||
#include "utils/hash_map.h"
|
||||
#include "transform/graph_ir/op_declare/op_declare_macro.h"
|
||||
#include "ops/elewise_calculation_ops.h"
|
||||
|
||||
namespace mindspore::transform {
|
||||
DECLARE_OP_ADAPTER(AccumulateNV2)//将AccumulateNV2的类型与标准进行对比，后进行空间调整，并用指针input_map_为key存储相应内容
|
||||
DECLARE_OP_USE_DYN_INPUT(AccumulateNV2)//将AccumulateNV2的类型与标准进行对比，后进行空间调整，并用指针dyn_input_map_为key存储相应内容
|
||||
DECLARE_OP_USE_OUTPUT(AccumulateNV2)//将AccumulateNV2的类型与标准进行对比，后进行空间调整，并用指针output_map_为key存储相应内容
|
||||
|
||||
DECLARE_OP_ADAPTER(ConfusionMulGrad)
|
||||
DECLARE_OP_USE_OUTPUT(ConfusionMulGrad)
|
||||
|
||||
DECLARE_OP_ADAPTER(FakeQuantWithMinMaxVars)
|
||||
DECLARE_OP_USE_OUTPUT(FakeQuantWithMinMaxVars)
|
||||
|
||||
DECLARE_OP_ADAPTER(FakeQuantWithMinMaxVarsGradient)
|
||||
DECLARE_OP_USE_OUTPUT(FakeQuantWithMinMaxVarsGradient)
|
||||
|
||||
DECLARE_OP_ADAPTER(FakeQuantWithMinMaxVarsPerChannel)
|
||||
DECLARE_OP_USE_OUTPUT(FakeQuantWithMinMaxVarsPerChannel)
|
||||
|
||||
DECLARE_OP_ADAPTER(FakeQuantWithMinMaxVarsPerChannelGradient)
|
||||
DECLARE_OP_USE_OUTPUT(FakeQuantWithMinMaxVarsPerChannelGradient)
|
||||
|
||||
DECLARE_OP_ADAPTER(GreaterEqual)
|
||||
DECLARE_OP_USE_OUTPUT(GreaterEqual)
|
||||
|
||||
DECLARE_OP_ADAPTER(AssignAdd)
|
||||
DECLARE_OP_USE_OUTPUT(AssignAdd)
|
||||
|
||||
DECLARE_OP_ADAPTER(AssignSub)
|
||||
DECLARE_OP_USE_OUTPUT(AssignSub)
|
||||
|
||||
DECLARE_OP_ADAPTER(ZerosLike)
|
||||
DECLARE_OP_USE_OUTPUT(ZerosLike)
|
||||
|
||||
DECLARE_OP_ADAPTER(OnesLike)
|
||||
DECLARE_OP_USE_OUTPUT(OnesLike)
|
||||
|
||||
DECLARE_OP_ADAPTER(ArgMaxD)
|
||||
DECLARE_OP_USE_OUTPUT(ArgMaxD)
|
||||
|
||||
DECLARE_OP_ADAPTER(ArgMaxV2)
|
||||
DECLARE_OP_USE_OUTPUT(ArgMaxV2)
|
||||
|
||||
DECLARE_OP_ADAPTER(ArgMinD)
|
||||
DECLARE_OP_USE_OUTPUT(ArgMinD)
|
||||
|
||||
DECLARE_OP_ADAPTER(ArgMaxWithValue)
|
||||
DECLARE_OP_USE_OUTPUT(ArgMaxWithValue)
|
||||
|
||||
DECLARE_OP_ADAPTER(ArgMinWithValue)
|
||||
DECLARE_OP_USE_OUTPUT(ArgMinWithValue)
|
||||
|
||||
DECLARE_OP_ADAPTER(Mul)
|
||||
DECLARE_OP_USE_OUTPUT(Mul)
|
||||
|
||||
DECLARE_OP_ADAPTER(MulNoNan)
|
||||
DECLARE_OP_USE_OUTPUT(MulNoNan)
|
||||
|
||||
DECLARE_OP_ADAPTER(AddN)
|
||||
DECLARE_OP_USE_DYN_INPUT(AddN)
|
||||
DECLARE_OP_USE_OUTPUT(AddN)
|
||||
|
||||
DECLARE_OP_ADAPTER(Less)
|
||||
DECLARE_OP_USE_OUTPUT(Less)
|
||||
|
||||
DECLARE_OP_ADAPTER(Rsqrt)
|
||||
DECLARE_OP_USE_OUTPUT(Rsqrt)
|
||||
|
||||
DECLARE_OP_ADAPTER(Sqrt)
|
||||
DECLARE_OP_USE_OUTPUT(Sqrt)
|
||||
|
||||
DECLARE_OP_ADAPTER(Square)
|
||||
DECLARE_OP_USE_OUTPUT(Square)
|
||||
|
||||
DECLARE_OP_ADAPTER(SquaredDifference)
|
||||
DECLARE_OP_USE_OUTPUT(SquaredDifference)
|
||||
|
||||
DECLARE_OP_ADAPTER(SquareSumAll)
|
||||
DECLARE_OP_USE_OUTPUT(SquareSumAll)
|
||||
|
||||
DECLARE_OP_ADAPTER(Maximum)
|
||||
DECLARE_OP_USE_OUTPUT(Maximum)
|
||||
|
||||
DECLARE_OP_ADAPTER(Minimum)
|
||||
DECLARE_OP_USE_OUTPUT(Minimum)
|
||||
|
||||
DECLARE_OP_ADAPTER(MaximumGrad)
|
||||
DECLARE_OP_USE_OUTPUT(MaximumGrad)
|
||||
|
||||
DECLARE_OP_ADAPTER(MinimumGrad)
|
||||
DECLARE_OP_USE_OUTPUT(MinimumGrad)
|
||||
|
||||
DECLARE_OP_ADAPTER(RealDiv)
|
||||
DECLARE_OP_USE_OUTPUT(RealDiv)
|
||||
|
||||
DECLARE_OP_ADAPTER(BitwiseAnd)
|
||||
DECLARE_OP_USE_OUTPUT(BitwiseAnd)
|
||||
|
||||
DECLARE_OP_ADAPTER(BitwiseOr)
|
||||
DECLARE_OP_USE_OUTPUT(BitwiseOr)
|
||||
|
||||
DECLARE_OP_ADAPTER(BitwiseXor)
|
||||
DECLARE_OP_USE_OUTPUT(BitwiseXor)
|
||||
|
||||
DECLARE_OP_ADAPTER(Rint)
|
||||
DECLARE_OP_USE_OUTPUT(Rint)
|
||||
|
||||
DECLARE_OP_ADAPTER(BesselI0e)
|
||||
DECLARE_OP_USE_OUTPUT(BesselI0e)
|
||||
|
||||
DECLARE_OP_ADAPTER(BesselI1e)
|
||||
DECLARE_OP_USE_OUTPUT(BesselI1e)
|
||||
|
||||
DECLARE_OP_ADAPTER(Inv)
|
||||
DECLARE_OP_USE_OUTPUT(Inv)
|
||||
|
||||
DECLARE_OP_ADAPTER(InvGrad)
|
||||
DECLARE_OP_USE_OUTPUT(InvGrad)
|
||||
|
||||
DECLARE_OP_ADAPTER(Invert)
|
||||
DECLARE_OP_USE_OUTPUT(Invert)
|
||||
|
||||
DECLARE_OP_ADAPTER(Log1p)
|
||||
DECLARE_OP_USE_OUTPUT(Log1p)
|
||||
|
||||
DECLARE_OP_ADAPTER(Ceil)
|
||||
DECLARE_OP_USE_OUTPUT(Ceil)
|
||||
|
||||
DECLARE_OP_ADAPTER(CosineEmbeddingLoss)
|
||||
DECLARE_OP_USE_OUTPUT(CosineEmbeddingLoss)
|
||||
|
||||
DECLARE_OP_ADAPTER(Xdivy)
|
||||
DECLARE_OP_USE_OUTPUT(Xdivy)
|
||||
|
||||
DECLARE_OP_ADAPTER(Mod)
|
||||
DECLARE_OP_USE_OUTPUT(Mod)
|
||||
|
||||
DECLARE_OP_ADAPTER(Cast)
|
||||
DECLARE_OP_USE_INPUT_ATTR(Cast)
|
||||
DECLARE_OP_USE_OUTPUT(Cast)
|
||||
|
||||
DECLARE_OP_ADAPTER(Reciprocal)
|
||||
DECLARE_OP_USE_OUTPUT(Reciprocal)
|
||||
|
||||
DECLARE_OP_ADAPTER(Neg)
|
||||
DECLARE_OP_USE_OUTPUT(Neg)
|
||||
|
||||
DECLARE_OP_ADAPTER(Sub)
|
||||
DECLARE_OP_USE_OUTPUT(Sub)
|
||||
|
||||
DECLARE_OP_ADAPTER(Pow)
|
||||
DECLARE_OP_USE_OUTPUT(Pow)
|
||||
|
||||
DECLARE_OP_ADAPTER(PopulationCount)
|
||||
DECLARE_OP_USE_OUTPUT(PopulationCount)
|
||||
|
||||
DECLARE_OP_ADAPTER(Equal)
|
||||
DECLARE_OP_USE_OUTPUT(Equal)
|
||||
|
||||
DECLARE_OP_ADAPTER(ApproximateEqual)
|
||||
DECLARE_OP_USE_OUTPUT(ApproximateEqual)
|
||||
|
||||
DECLARE_OP_ADAPTER(NotEqual)
|
||||
DECLARE_OP_USE_OUTPUT(NotEqual)
|
||||
|
||||
DECLARE_OP_ADAPTER(Log)
|
||||
DECLARE_OP_USE_OUTPUT(Log)
|
||||
|
||||
DECLARE_OP_ADAPTER(LogicalAnd)
|
||||
DECLARE_OP_USE_OUTPUT(LogicalAnd)
|
||||
|
||||
DECLARE_OP_ADAPTER(LogicalOr)
|
||||
DECLARE_OP_USE_OUTPUT(LogicalOr)
|
||||
|
||||
DECLARE_OP_ADAPTER(LogicalNot)
|
||||
DECLARE_OP_USE_OUTPUT(LogicalNot)
|
||||
|
||||
DECLARE_OP_ADAPTER(LessEqual)
|
||||
DECLARE_OP_USE_OUTPUT(LessEqual)
|
||||
|
||||
DECLARE_OP_ADAPTER(Assign)
|
||||
DECLARE_OP_USE_OUTPUT(Assign)
|
||||
|
||||
DECLARE_OP_ADAPTER(Add)
|
||||
DECLARE_OP_USE_OUTPUT(Add)
|
||||
|
||||
DECLARE_OP_ADAPTER(Cos)
|
||||
DECLARE_OP_USE_OUTPUT(Cos)
|
||||
|
||||
DECLARE_OP_ADAPTER(Cosh)
|
||||
DECLARE_OP_USE_OUTPUT(Cosh)
|
||||
|
||||
DECLARE_OP_ADAPTER(Acos)
|
||||
DECLARE_OP_USE_OUTPUT(Acos)
|
||||
|
||||
DECLARE_OP_ADAPTER(AcosGrad)
|
||||
DECLARE_OP_USE_OUTPUT(AcosGrad)
|
||||
|
||||
DECLARE_OP_ADAPTER(Acosh)
|
||||
DECLARE_OP_USE_OUTPUT(Acosh)
|
||||
|
||||
DECLARE_OP_ADAPTER(AcoshGrad)
|
||||
DECLARE_OP_USE_OUTPUT(AcoshGrad)
|
||||
|
||||
DECLARE_OP_ADAPTER(Div)
|
||||
DECLARE_OP_USE_OUTPUT(Div)
|
||||
|
||||
DECLARE_OP_ADAPTER(TruncateDiv)
|
||||
DECLARE_OP_USE_OUTPUT(TruncateDiv)
|
||||
|
||||
DECLARE_OP_ADAPTER(TruncateMod)
|
||||
DECLARE_OP_USE_OUTPUT(TruncateMod)
|
||||
|
||||
DECLARE_OP_ADAPTER(Xlogy)
|
||||
DECLARE_OP_USE_OUTPUT(Xlogy)
|
||||
|
||||
DECLARE_OP_ADAPTER(DivNoNan)
|
||||
DECLARE_OP_USE_OUTPUT(DivNoNan)
|
||||
|
||||
DECLARE_OP_ADAPTER(Floor)
|
||||
DECLARE_OP_USE_OUTPUT(Floor)
|
||||
|
||||
DECLARE_OP_ADAPTER(FloorDiv)
|
||||
DECLARE_OP_USE_OUTPUT(FloorDiv)
|
||||
|
||||
DECLARE_OP_ADAPTER(FloorMod)
|
||||
DECLARE_OP_USE_OUTPUT(FloorMod)
|
||||
|
||||
DECLARE_OP_ADAPTER(Sin)
|
||||
DECLARE_OP_USE_OUTPUT(Sin)
|
||||
|
||||
DECLARE_OP_ADAPTER(Sinh)
|
||||
DECLARE_OP_USE_OUTPUT(Sinh)
|
||||
|
||||
DECLARE_OP_ADAPTER(Asin)
|
||||
DECLARE_OP_USE_OUTPUT(Asin)
|
||||
|
||||
DECLARE_OP_ADAPTER(AsinGrad)
|
||||
DECLARE_OP_USE_OUTPUT(AsinGrad)
|
||||
|
||||
DECLARE_OP_ADAPTER(Asinh)
|
||||
DECLARE_OP_USE_OUTPUT(Asinh)
|
||||
|
||||
DECLARE_OP_ADAPTER(AsinhGrad)
|
||||
DECLARE_OP_USE_OUTPUT(AsinhGrad)
|
||||
|
||||
DECLARE_OP_ADAPTER(Exp)
|
||||
DECLARE_OP_USE_OUTPUT(Exp)
|
||||
|
||||
DECLARE_OP_ADAPTER(Expm1)
|
||||
DECLARE_OP_USE_OUTPUT(Expm1)
|
||||
|
||||
DECLARE_OP_ADAPTER(BiasAdd)
|
||||
DECLARE_OP_USE_OUTPUT(BiasAdd)
|
||||
|
||||
DECLARE_OP_ADAPTER(Greater)
|
||||
DECLARE_OP_USE_OUTPUT(Greater)
|
||||
|
||||
DECLARE_OP_ADAPTER(SqrtGrad)
|
||||
DECLARE_OP_USE_OUTPUT(SqrtGrad)
|
||||
|
||||
DECLARE_OP_ADAPTER(ReciprocalGrad)
|
||||
DECLARE_OP_USE_OUTPUT(ReciprocalGrad)
|
||||
|
||||
DECLARE_OP_ADAPTER(RsqrtGrad)
|
||||
DECLARE_OP_USE_OUTPUT(RsqrtGrad)
|
||||
|
||||
DECLARE_OP_ADAPTER(Abs)
|
||||
DECLARE_OP_USE_OUTPUT(Abs)
|
||||
|
||||
DECLARE_OP_ADAPTER(AbsGrad)
|
||||
DECLARE_OP_USE_OUTPUT(AbsGrad)
|
||||
|
||||
DECLARE_OP_ADAPTER(Sign)
|
||||
DECLARE_OP_USE_OUTPUT(Sign)
|
||||
|
||||
DECLARE_OP_ADAPTER(Round)
|
||||
DECLARE_OP_USE_OUTPUT(Round)
|
||||
|
||||
DECLARE_OP_ADAPTER(Tan)
|
||||
DECLARE_OP_USE_OUTPUT(Tan)
|
||||
|
||||
DECLARE_OP_ADAPTER(Atan)
|
||||
DECLARE_OP_USE_OUTPUT(Atan)
|
||||
|
||||
DECLARE_OP_ADAPTER(AtanGrad)
|
||||
DECLARE_OP_USE_OUTPUT(AtanGrad)
|
||||
|
||||
DECLARE_OP_ADAPTER(Atanh)
|
||||
DECLARE_OP_USE_OUTPUT(Atanh)
|
||||
|
||||
DECLARE_OP_ADAPTER(Atan2)
|
||||
DECLARE_OP_USE_OUTPUT(Atan2)
|
||||
|
||||
DECLARE_OP_ADAPTER(LambApplyOptimizerAssign)
|
||||
DECLARE_OP_USE_OUTPUT(LambApplyOptimizerAssign)
|
||||
|
||||
DECLARE_OP_ADAPTER(LambApplyWeightAssign)
|
||||
DECLARE_OP_USE_OUTPUT(LambApplyWeightAssign)
|
||||
|
||||
DECLARE_OP_ADAPTER(Eltwise)
|
||||
DECLARE_OP_USE_OUTPUT(Eltwise)
|
||||
} // namespace mindspore::transform
|
||||
#endif // MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_ELEWISE_CALCULATION_OPS_DECLARE_H_
|
||||
|
|
@ -0,0 +1,55 @@
|
|||
/**
|
||||
* Copyright 2019 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#include "transform/graph_ir/op_declare/functional_ops_declare.h"
|
||||
|
||||
namespace mindspore::transform {
|
||||
// Case
|
||||
INPUT_MAP(Case) = {{1, INPUT_DESC(branch_index)}};/*
|
||||
将branch_index处理并存入对应INPUT_DESC结构体的相应变量中
|
||||
将branch_index内容转为字符串变量并存储至结构体的name变量中
|
||||
引用Operator空间并将指针所指的类转为INPUT_DESC结构体
|
||||
将Case的类型与标准进行对比，后进行空间调整，并用指针input_map_为key存储相应内容
|
||||
*/
|
||||
DYN_INPUT_MAP(Case) = {{2, DYN_INPUT_DESC(input)}};/*
|
||||
将branch_index处理并存入对应DYN_INPUT_DESC结构体的相应变量中
|
||||
将branch_index内容转为字符串变量并存储至结构体的name变量中
|
||||
引用Operator空间并将指针所指的类转为DYN_INPUT_DESC结构体
|
||||
将Case的类型与标准进行对比，后进行空间调整，并用指针dyn_input_map_为key存储相应内容
|
||||
*/
|
||||
ATTR_MAP(Case) = EMPTY_ATTR_MAP;//将Case与标准进行比较，使原以attr_map_为储存信息的key变为空
|
||||
DYN_OUTPUT_MAP(Case) = {{0, DYN_OUTPUT_DESC(output)}};/*
|
||||
将(output处理并存入对应DYN_OUTPUT_DESC结构体的相应变量中
|
||||
将(output内容转为字符串变量并存储至结构体的name变量中
|
||||
引用Operator空间并将指针所指的类转为DYN_OUTPUT_DESC结构体
|
||||
将Case的类型与标准进行对比，后进行空间调整，并用指针dyn_output_map_为key存储相应内容
|
||||
*/
|
||||
DYN_SUBGRAPH_MAP(Case) = {{0, DYN_SUBGRAPH_DESC(branches)}};/*
|
||||
将branches处理并存入对应DYN_SUBGRAPH_DESC结构体的相应变量中
|
||||
将branches内容转为字符串变量并存储至结构体的name变量中
|
||||
引用Operator空间并将指针所指的类转为DYN_SUBGRAPH_DESC结构体
|
||||
将Case的类型与标准进行对比，后进行空间调整，并用指针dyn_subgrraph_map_为key存储相应内容
|
||||
*/
|
||||
REG_ADPT_DESC(Case, kNameCase, ADPT_DESC(Case))/*
|
||||
将Case处理并存入对应ADPT_DESC结构体的相应变量中
|
||||
将Case内容转为字符串变量并存储至结构体的name变量中
|
||||
引用Operator空间并将指针所指的类转为ADPT_DESC结构体
|
||||
|
||||
再将Case处理并存入对应REG_ADPT_DESC结构体的相应变量中
|
||||
将Case内容转为字符串变量并存储至REG_ADPT_DESC的结构体的name变量中
|
||||
引用Operator空间并将指针所指的类转为REG_ADPT_DESC结构体
|
||||
*/
|
||||
} // namespace mindspore::transform
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
/**
|
||||
* Copyright 2019-2021 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_FUNCTIONAL_OPS_DECLARE_H_
|
||||
#define MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_FUNCTIONAL_OPS_DECLARE_H_
|
||||
|
||||
#include <string>
|
||||
#include "utils/hash_map.h"
|
||||
#include "transform/graph_ir/op_declare/op_declare_macro.h"
|
||||
#include "ops/functional_ops.h"
|
||||
|
||||
namespace mindspore::transform {
|
||||
DECLARE_OP_ADAPTER(Case)//将Case的类型与标准进行对比，后进行空间调整，并用指针input_map_为key存储相应内容
|
||||
DECLARE_OP_USE_DYN_INPUT(Case)//将Case的类型与标准进行对比，后进行空间调整，并用指针dyn_input_map_为key存储相应内容
|
||||
DECLARE_OP_USE_DYN_SUBGRAPH(Case)//将ACase的类型与标准进行对比，后进行空间调整，并用指针dyn_subgraph__map_为key存储相应内容
|
||||
DECLARE_OP_USE_DYN_OUTPUT(Case)//将Case的类型与标准进行对比，后进行空间调整，并用指针dyn_output_map_为key存储相应内容
|
||||
} // namespace mindspore::transform
|
||||
#endif // MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_FUNCTIONAL_OPS_DECLARE_H_
|
||||
|
|
@ -0,0 +1,73 @@
|
|||
/**
|
||||
* Copyright 2019 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#include "include/transform/graph_ir/graph_builder.h"
|
||||
|
||||
#include <sstream>
|
||||
|
||||
#include "ops/math_ops.h"
|
||||
|
||||
namespace mindspore {
|
||||
namespace transform {
|
||||
//定义名为BuildMDDatasetGraph的函数，返回值为自定义类型DfGraphPtr
|
||||
//此函数为建立MD数据设置表格，返回为用指针引出的一系列数据
|
||||
/*
|
||||
首先输出消息日志 "BuildMDDatasetGraph."，其次初始化数据并获取变量param的name属性
|
||||
再其次借助SetInputs函数和SetOutputs函数分别设置图表的输入和输出
|
||||
最后返回dataset_graph
|
||||
*/
|
||||
DfGraphPtr BuildMDDatasetGraph(const DatasetGraphParam ¶m) {
|
||||
MS_LOG(INFO) << "BuildMDDatasetGraph.";
|
||||
|
||||
// InitData
|
||||
auto d = ge::op::InitData("init_data_tmp").set_attr_channel_name(param.queue_name());
|
||||
|
||||
// set graph inputs & outputs
|
||||
std::vector<ge::Operator> inputs{d};
|
||||
std::vector<ge::Operator> outputs{d};
|
||||
DfGraphPtr dataset_graph = std::make_shared<DfGraph>("dataset");
|
||||
(void)dataset_graph->SetInputs(inputs);
|
||||
(void)dataset_graph->SetOutputs(outputs);
|
||||
|
||||
return dataset_graph;
|
||||
}
|
||||
//定义名为BuildDatasetGraph的函数，返回值为自定义类型Status
|
||||
/*
|
||||
首先定义Status类型变量ret，并将phase存入graph_name
|
||||
其次输出消息日志"BuildDatasetGraph begin. phase is [*此处为phase]"
|
||||
"param is[*此处打印param]."
|
||||
再其次，先建立MD数据设置表格，再在此基础上执行对graph_name和dataset_graph的进一步操作并存入ret
|
||||
判断上述操作是否成功，若不成功则输出错误日志"BuildDatasetGraph failed."，若成功则输出消息日志"BuildDatasetGraph end."
|
||||
最后返回ret
|
||||
*/
|
||||
Status BuildDatasetGraph(const DatasetGraphParam ¶m, const std::string &phase) {
|
||||
Status ret;
|
||||
std::string graph_name = phase;
|
||||
|
||||
MS_LOG(INFO) << "BuildDatasetGraph begin. phase is " << phase;
|
||||
MS_LOG(INFO) << "param is " << param.ToString() << ".";
|
||||
|
||||
DfGraphPtr dataset_graph = BuildMDDatasetGraph(param);
|
||||
ret = DfGraphManager::GetInstance().AddGraph(graph_name, dataset_graph);
|
||||
if (ret != Status::SUCCESS) {
|
||||
MS_LOG(ERROR) << "BuildDatasetGraph failed.";
|
||||
} else {
|
||||
MS_LOG(INFO) << "BuildDatasetGraph end.";
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
} // namespace transform
|
||||
} // namespace mindspore
|
||||
|
|
@ -0,0 +1,283 @@
|
|||
/**
|
||||
* Copyright 2019 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* Limitations under the License.
|
||||
*/
|
||||
|
||||
#include "include/transform/graph_ir/graph_runner.h"
|
||||
#include <algorithm>
|
||||
#include <string>
|
||||
#include <memory>
|
||||
|
||||
#ifndef ENABLE_LITE_ACL
|
||||
#include "pybind11/pybind11.h"
|
||||
#endif
|
||||
#include "utils/log_adapter.h"
|
||||
#include "include/common/utils/config_manager.h"
|
||||
#include "sys/time.h"
|
||||
#include "include/common/utils/utils.h"
|
||||
#include "include/common/utils/callbacks.h"
|
||||
#ifdef ENABLE_D
|
||||
#include "include/common/utils/callbacks_ge.h"
|
||||
#endif
|
||||
#include "utils/ms_context.h"
|
||||
|
||||
#ifndef ENABLE_LITE_ACL
|
||||
namespace py = pybind11;
|
||||
#endif
|
||||
namespace mindspore {
|
||||
namespace transform {
|
||||
//在GraphRunner区域定义名为NewSession的函数，返回值类型为std::shared_ptr<ge::Session>
|
||||
/*
|
||||
首先判断ENABLE_D是否被定义，若是，则执行条件命令：
|
||||
定义变量ret和ms_context并令后者储存MsContext::GetInstance()数据
|
||||
判断ms_context是否为空
|
||||
判断ms_context的backend_policy()获取的属性是否为"ge"，若是，则对sess_options进行make_shared并储存到ret中
|
||||
在此基础上在判断ret是否为空，若是则输出EXCEPTION日志"Create GE session failed!"
|
||||
输出消息日志"Create new GE session success!"并返回ret，结束条件命令
|
||||
若ENABLE_D没有被定义，则输出警告日志"no GE client, return nullptr!"并返回nullptr
|
||||
*/
|
||||
std::shared_ptr<ge::Session> GraphRunner::NewSession(const SessionOptions &sess_options) {
|
||||
#ifdef ENABLE_D
|
||||
std::shared_ptr<ge::Session> ret;
|
||||
auto ms_context = MsContext::GetInstance();
|
||||
MS_EXCEPTION_IF_NULL(ms_context);
|
||||
if (ms_context->backend_policy() == "ge") {
|
||||
ret = std::make_shared<ge::Session>(sess_options);
|
||||
if (ret == nullptr) {
|
||||
MS_LOG(EXCEPTION) << "Create GE session failed!";
|
||||
}
|
||||
MS_LOG(INFO) << "Create new GE session success!";
|
||||
return ret;
|
||||
}
|
||||
#endif
|
||||
|
||||
MS_LOG(WARNING) << "no GE client, return nullptr!";
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
//GraphRunner区域的GraphRunner函数
|
||||
/*
|
||||
获取实例的并行策略是否为ONE_DEVICE，若是，则输出消息日志"ME run in ONE_DEVICE strategy mode"
|
||||
判断option的sess_ptr属性是否为空，若不为空则令sess_ = options.sess_ptr
|
||||
若为空则用变量属性options.options创建新会话并储存在sess_中
|
||||
在此基础上再判断sess_是否为空，若是则输出警告日志"graph runner sess_ is nullptr!"
|
||||
*/
|
||||
GraphRunner::GraphRunner(const GraphRunnerOptions &options)
|
||||
: options_(options), graph_manager_(DfGraphManager::GetInstance()) {
|
||||
if (ConfigManager::GetInstance().parallel_strategy() == ParallelStrategy::ONE_DEVICE) {
|
||||
MS_LOG(INFO) << "ME run in ONE_DEVICE strategy mode";
|
||||
}
|
||||
|
||||
if (options.sess_ptr != nullptr) {
|
||||
sess_ = options.sess_ptr;
|
||||
} else {
|
||||
sess_ = NewSession(options.options);
|
||||
if (sess_ == nullptr) {
|
||||
MS_LOG(WARNING) << "graph runner sess_ is nullptr!";
|
||||
}
|
||||
}
|
||||
/*
|
||||
判断ENABLE_D是否被定义，若是则执行条件命令：
|
||||
定义变量ms_context并将获取的实例储存在其中
|
||||
判断ms_context的backend_policy()属性是否为ge，若是则：
|
||||
判断会话对于kCheckPoint和CheckpointSaveCallback回叫功能是否实现，若没有实现则输出警告日志"register callback failed!"
|
||||
判断会话对于kSummary和SummarySaveCallback回叫功能是否实现，若没有实现则输出警告日志"register summary callback failed!"
|
||||
结束条件命令
|
||||
*/
|
||||
#ifdef ENABLE_D
|
||||
auto ms_context = MsContext::GetInstance();
|
||||
MS_EXCEPTION_IF_NULL(ms_context);
|
||||
if (ms_context->backend_policy() == "ge") {
|
||||
// register the callback function注册表的回叫信号功能
|
||||
if (sess_->RegisterCallBackFunc(callbacks::kCheckPoint, callbacks::CheckpointSaveCallback) != ge::GRAPH_SUCCESS) {
|
||||
MS_LOG(EXCEPTION) << "register callback failed!";
|
||||
}
|
||||
|
||||
if (sess_->RegisterCallBackFunc(callbacks::kSummary, callbacks::SummarySaveCallback) != ge::GRAPH_SUCCESS) {
|
||||
MS_LOG(EXCEPTION) << "register summary callback failed!";
|
||||
}
|
||||
}
|
||||
#endif
|
||||
//定义变量wrappers储存graph_manager_.GetAllGraphs()并判断其是否为空，若是，则输出消息日志"The GraphManager is empty!!"并返回
|
||||
std::vector<DfGraphWrapperPtr> wrappers = graph_manager_.GetAllGraphs();
|
||||
if (wrappers.empty()) {
|
||||
MS_LOG(INFO) << "The GraphManager is empty!!";
|
||||
return;
|
||||
}
|
||||
/*
|
||||
判断ENABLE_D是否被定义，若是则执行条件命令：
|
||||
判断ms_context的backend_policy()属性是否为ge，若不是，则返回
|
||||
利用for循环获取并历遍已储存的图表
|
||||
输出消息日志"Add the graph[*此处为name属性]to GE, it's id is:[*此处为id属性]"
|
||||
将id对应的it与graph_manager_进行AddSavedGraphs操作
|
||||
将static_cast<uint32_t>(it->id_), *(it->graph_ptr_), it->options_执行AddGraph操作储存在sess_指针中
|
||||
结束条件命令
|
||||
*/
|
||||
#ifdef ENABLE_D
|
||||
if (ms_context->backend_policy() != "ge") {
|
||||
return;
|
||||
}
|
||||
|
||||
for (auto &it : wrappers) {
|
||||
std::set<string> saved_graph = graph_manager_.GetSavedGraphs();
|
||||
auto iter_find = saved_graph.find(std::to_string(it->id_));
|
||||
if (iter_find != saved_graph.end()) {
|
||||
continue;
|
||||
}
|
||||
MS_LOG(INFO) << "Add the graph " << (*it).name_ << " to GE, it's id is: " << (*it).id_;
|
||||
graph_manager_.AddSavedGraphs(std::to_string(it->id_));
|
||||
(void)sess_->AddGraph(static_cast<uint32_t>(it->id_), *(it->graph_ptr_), it->options_);
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
//在GraphRunner区域定义名为RunGraph的函数，返回值为自定义类型Status
|
||||
/*
|
||||
获取options的name属性储存在name变量中并判断其是否为空，若是则输出错误日志"The graph name is null"并返回return Status::INVALID_ARGUMENT
|
||||
定义wrap_ptr获取graph的name属性并检验其是否为空，若为空则输出错误日志"Get graph form DfGraphManager failed!"并返回Status::NOT_FOUND
|
||||
检测wrap_ptr->graph_ptr_是否为空，若为空则输出警告日志"The graph is null"并返回Status::NOT_FOUND
|
||||
*/
|
||||
Status GraphRunner::RunGraph(const RunOptions &options, const std::vector<GeTensorPtr> &inputs,
|
||||
std::vector<GeTensorPtr> *outputs) {
|
||||
std::string name = options.name;
|
||||
if (name.empty()) {
|
||||
MS_LOG(ERROR) << "The graph name is null";
|
||||
return Status::INVALID_ARGUMENT;
|
||||
}
|
||||
|
||||
DfGraphWrapperPtr wrap_ptr = graph_manager_.GetGraphByName(name);
|
||||
if (wrap_ptr == nullptr) {
|
||||
MS_LOG(ERROR) << "Get graph form DfGraphManager failed!";
|
||||
return Status::NOT_FOUND;
|
||||
}
|
||||
|
||||
if (wrap_ptr->graph_ptr_ == nullptr) {
|
||||
MS_LOG(WARNING) << "The graph is null";
|
||||
return Status::NOT_FOUND;
|
||||
}
|
||||
|
||||
// call ge::RunGraph() to exec a graph;//调试ge::RunGraph()执行图表
|
||||
//获取输出的size属性并输出相应消息日志
|
||||
//获取开始时间
|
||||
std::vector<GeTensor> ge_inputs;
|
||||
std::vector<GeTensor> ge_outputs;
|
||||
|
||||
(void)std::transform(inputs.begin(), inputs.end(), std::back_inserter(ge_inputs),
|
||||
[](const GeTensorPtr &i) { return *i; });
|
||||
|
||||
MS_LOG(INFO) << "Run the graph in GE with " << ge_inputs.size() << " inputs";
|
||||
|
||||
struct timeval start_time, end_time;
|
||||
(void)gettimeofday(&start_time, nullptr);
|
||||
/*
|
||||
检测ENABLE_D是否被定义，若是，则执行条件命令：
|
||||
获取实例并储存在ms_context中并检测其是否为空
|
||||
检测ms_context的backend_policy是否为ge，若是则
|
||||
检测sess_是否为空，若是则输出错误日志"The GE session is null, can't run the graph!"并返回Status::FAILED
|
||||
执行RunGraph功能并将结果数据储存在ret中
|
||||
检测上一步是否成功，若不成功则输出错误日志"Call GE RunGraph Failed, ret is:[*此处为ret]"并返回Status::FAILED
|
||||
若ENABLE_D没有被定义，则执行 ge_outputs.swap(ge_inputs)
|
||||
结束条件命令
|
||||
*/
|
||||
#ifdef ENABLE_D
|
||||
auto ms_context = MsContext::GetInstance();
|
||||
MS_EXCEPTION_IF_NULL(ms_context);
|
||||
if (ms_context->backend_policy() == "ge") {
|
||||
if (sess_ == nullptr) {
|
||||
MS_LOG(ERROR) << "The GE session is null, can't run the graph!";
|
||||
return Status::FAILED;
|
||||
}
|
||||
ge::Status ret = sess_->RunGraph(static_cast<uint32_t>(wrap_ptr->id_), ge_inputs, ge_outputs);
|
||||
if (ret != ge::GRAPH_SUCCESS) {
|
||||
MS_LOG(ERROR) << "Call GE RunGraph Failed, ret is: " << ret;
|
||||
return Status::FAILED;
|
||||
}
|
||||
}
|
||||
#else
|
||||
ge_outputs.swap(ge_inputs);
|
||||
#endif
|
||||
//获取结束时间并计算出调试所用时长
|
||||
//输出消息日志"Call GE RunGraph Success in "[*此处为消耗时长]" us, the GE outputs num is:[*此处为e_outputs.size]"
|
||||
(void)gettimeofday(&end_time, nullptr);
|
||||
const uint64_t kUSecondInSecond = 1000000;
|
||||
uint64_t cost = kUSecondInSecond * static_cast<uint64_t>(end_time.tv_sec - start_time.tv_sec);
|
||||
cost += static_cast<uint64_t>(end_time.tv_usec - start_time.tv_usec);
|
||||
MS_LOG(INFO) << "Call GE RunGraph Success in " << cost << " us, the GE outputs num is: " << ge_outputs.size();
|
||||
|
||||
(void)std::transform(ge_outputs.begin(), ge_outputs.end(), std::back_inserter(*outputs),
|
||||
[](const GeTensor &ge_tensor) { return std::make_shared<GeTensor>(ge_tensor); });
|
||||
|
||||
return Status::SUCCESS;
|
||||
}
|
||||
//在GraphRunner区域定义名为RunGraph的函数，返回值为自定义类型Status
|
||||
/*
|
||||
利用for循环分别对inputs的size属性和shape属性进行转存，并分别输出消息日志："inputs tensor's data size is:[*此处为size]"
|
||||
"inputs tensor's shape is: {[*此处为shape]}"
|
||||
利用ge_tensor_ptr转存对以it和kOpFormat_NCHW为变量进行ConvertTensor操作之后的结果并检验其是否为空
|
||||
若不为空则进行ge_inputs.emplace_back(ge_tensor_ptr)操作
|
||||
若为空则输出消息日志"Convert input Me tensor to Ge tensor failed. Abort this graph"并返回Status::FAILED
|
||||
*/
|
||||
Status GraphRunner::RunGraph(const RunOptions &options, const std::vector<MeTensorPtr> &inputs,
|
||||
std::vector<MeTensorPtr> *const outputs) {
|
||||
std::vector<GeTensorPtr> ge_inputs;
|
||||
for (auto it : inputs) {
|
||||
MS_EXCEPTION_IF_NULL(it);
|
||||
MS_LOG(INFO) << "inputs tensor's data size is: " << (*it).DataSize();
|
||||
auto shape = (*it).shape();
|
||||
std::string shape_str;
|
||||
for (const auto &elem : shape) {
|
||||
shape_str += std::to_string(elem);
|
||||
shape_str += " ";
|
||||
}
|
||||
MS_LOG(INFO) << "inputs tensor's shape is: { " << shape_str << "}";
|
||||
|
||||
auto ge_tensor_ptr = TransformUtil::ConvertTensor(it, kOpFormat_NCHW);
|
||||
if (ge_tensor_ptr != nullptr) {
|
||||
ge_inputs.emplace_back(ge_tensor_ptr);
|
||||
} else {
|
||||
MS_LOG(INFO) << "Convert input Me tensor to Ge tensor failed. Abort this graph";
|
||||
return Status::FAILED;
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<GeTensorPtr> ge_outputs;
|
||||
Status ret;
|
||||
{
|
||||
// Release GIL before calling into (potentially long-running) C++ code//在调试为c++代码前释放GIL
|
||||
#ifndef ENABLE_LITE_ACL
|
||||
py::gil_scoped_release release;
|
||||
#endif
|
||||
/*使用ret转存对变量options, ge_inputs, &ge_outputs进行RunGraph之后的数据结果并检测其是否成功
|
||||
若不成功则返回ret
|
||||
若成功则将it转化为GeTensors并存入tensor并检测tensor是否为空，若不为空则执行(void)outputs->emplace_back(tensor)
|
||||
输出消息日志"Return Me tensor outputs num is:[*此处为outputs->size]"并返回Status::SUCCESS
|
||||
*/
|
||||
ret = RunGraph(options, ge_inputs, &ge_outputs);
|
||||
}
|
||||
if (ret != Status::SUCCESS) {
|
||||
return ret;
|
||||
} else {
|
||||
// convert GeTensor to MeTensor//将GeTensor转化为MeTensor
|
||||
for (auto &it : ge_outputs) {
|
||||
auto tensor = TransformUtil::ConvertGeTensor(it);
|
||||
if (tensor != nullptr) {
|
||||
(void)outputs->emplace_back(tensor);
|
||||
}
|
||||
}
|
||||
MS_LOG(INFO) << "Return Me tensor outputs num is: " << outputs->size();
|
||||
return Status::SUCCESS;
|
||||
}
|
||||
}
|
||||
} // namespace transform
|
||||
} // namespace mindspore
|
||||
|
|
@ -0,0 +1,76 @@
|
|||
/**
|
||||
* Copyright 2019 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#include "transform/graph_ir/op_declare/hcom_ops_declare.h"
|
||||
|
||||
namespace mindspore::transform {
|
||||
// HCOMAllreduce
|
||||
INPUT_MAP(HcomAllReduce) = {{1, INPUT_DESC(x)}};/*
|
||||
将x处理并存入对应INPUT_DESC结构体的相应变量中
|
||||
将x内容转为字符串变量并存储至结构体的name变量中
|
||||
引用Operator空间并将指针所指的类转为INPUT_DESC结构体
|
||||
将HcomAllReduce的类型与标准进行对比，后进行空间调整，并用指针input_map_为key存储相应内容
|
||||
*/
|
||||
OUTPUT_MAP(HcomAllReduce) = {{0, OUTPUT_DESC(y)}};/*
|
||||
将y处理并存入对应OUTPUT_DESC结构体的相应变量中
|
||||
将x内容转为字符串变量并存储至结构体的name变量中
|
||||
引用Operator空间并将指针所指的类转为OUTPUT_DESC结构体
|
||||
将(HcomAllReduce的类型与标准进行对比，后进行空间调整，并用指针output_map_为key存储相应内容
|
||||
*/
|
||||
ATTR_MAP(HcomAllReduce) = {{"op", ATTR_DESC(reduction, AnyTraits<std::string>())},
|
||||
{"group", ATTR_DESC(group, AnyTraits<std::string>())},
|
||||
{"fusion", ATTR_DESC(fusion, AnyTraits<int64_t>())}};/*
|
||||
将reduction处理并存入对应ATTR_DESC结构体的相应变量中
|
||||
将reduction内容转为字符串变量并存储至结构体的name变量中
|
||||
引用Operator空间并将指针所指的类转为ATTR_DESC结构体
|
||||
//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
//存入MaxPool对应空间并用attr_map_指针保存
|
||||
//其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
将HcomAllReduce的类型与标准进行对比，后进行空间调整，并用指针attr_map_为key存储相应内容
|
||||
*/
|
||||
REG_ADPT_DESC(HcomAllReduce, kNameAllReduce, ADPT_DESC(HcomAllReduce))/*
|
||||
将HcomAllReduce处理并存入对应ADPT_DESC结构体的相应变量中
|
||||
将HcomAllReduce内容转为字符串变量并存储至结构体的name变量中
|
||||
引用Operator空间并将指针所指的类转为ADPT_DESC结构体
|
||||
|
||||
再将HcomAllReduce处理并存入对应REG_ADPT_DESC结构体的相应变量中
|
||||
将HcomAllReduce内容转为字符串变量并存储至REG_ADPT_DESC的结构体的name变量中
|
||||
引用Operator空间并将指针所指的类转为REG_ADPT_DESC结构体
|
||||
*/
|
||||
|
||||
// HCOMBraodcast
|
||||
INPUT_MAP(HcomBroadcast) = EMPTY_INPUT_MAP;
|
||||
DYN_INPUT_MAP(HcomBroadcast) = {{1, DYN_INPUT_DESC(x)}};
|
||||
DYN_OUTPUT_MAP(HcomBroadcast) = {{0, DYN_OUTPUT_DESC(y)}};
|
||||
ATTR_MAP(HcomBroadcast) = {{"root_rank", ATTR_DESC(root_rank, AnyTraits<int64_t>())},
|
||||
{"group", ATTR_DESC(group, AnyTraits<std::string>())}};
|
||||
REG_ADPT_DESC(HcomBroadcast, kNameBroadcast, ADPT_DESC(HcomBroadcast))
|
||||
|
||||
// HcomAllGather
|
||||
INPUT_MAP(HcomAllGather) = {{1, INPUT_DESC(x)}};
|
||||
OUTPUT_MAP(HcomAllGather) = {{0, OUTPUT_DESC(y)}};
|
||||
ATTR_MAP(HcomAllGather) = {{"group", ATTR_DESC(group, AnyTraits<std::string>())},
|
||||
{"rank_size", ATTR_DESC(rank_size, AnyTraits<int64_t>())}};
|
||||
REG_ADPT_DESC(HcomAllGather, kNameAllgather, ADPT_DESC(HcomAllGather))
|
||||
|
||||
// HCOMReduceScatter
|
||||
INPUT_MAP(HcomReduceScatter) = {{1, INPUT_DESC(x)}};
|
||||
OUTPUT_MAP(HcomReduceScatter) = {{0, OUTPUT_DESC(y)}};
|
||||
ATTR_MAP(HcomReduceScatter) = {{"group", ATTR_DESC(group, AnyTraits<std::string>())},
|
||||
{"op", ATTR_DESC(reduction, AnyTraits<std::string>())},
|
||||
{"rank_size", ATTR_DESC(rank_size, AnyTraits<int64_t>())}};
|
||||
REG_ADPT_DESC(HcomReduceScatter, kNameReduceScatter, ADPT_DESC(HcomReduceScatter))
|
||||
} // namespace mindspore::transform
|
||||
|
|
@ -0,0 +1,39 @@
|
|||
/**
|
||||
* Copyright 2019-2021 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_HCOM_OPS_DECLARE_H_
|
||||
#define MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_HCOM_OPS_DECLARE_H_
|
||||
|
||||
#include <string>
|
||||
#include "utils/hash_map.h"
|
||||
#include "transform/graph_ir/op_declare/op_declare_macro.h"
|
||||
#include "ops/hcom_ops.h"
|
||||
|
||||
namespace mindspore::transform {
|
||||
DECLARE_OP_ADAPTER(HcomReduceScatter)//将HcomReduceScatter的类型与标准进行对比，后进行空间调整，并用指针input_map_为key存储相应内容
|
||||
DECLARE_OP_USE_OUTPUT(HcomReduceScatter)//将HcomReduceScatter的类型与标准进行对比，后进行空间调整，并用指针output_map_为key存储相应内容
|
||||
|
||||
DECLARE_OP_ADAPTER(HcomBroadcast)
|
||||
DECLARE_OP_USE_DYN_INPUT(HcomBroadcast)//将HcomReduceScatter的类型与标准进行对比，后进行空间调整，并用指针dyn_input_map_为key存储相应内容
|
||||
DECLARE_OP_USE_DYN_OUTPUT(HcomBroadcast)//将HcomReduceScatter的类型与标准进行对比，后进行空间调整，并用指针dyn_output_map_为key存储相应内容
|
||||
|
||||
DECLARE_OP_ADAPTER(HcomAllReduce)
|
||||
DECLARE_OP_USE_OUTPUT(HcomAllReduce)
|
||||
|
||||
DECLARE_OP_ADAPTER(HcomAllGather)
|
||||
DECLARE_OP_USE_OUTPUT(HcomAllGather)
|
||||
} // namespace mindspore::transform
|
||||
#endif // MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_HCOM_OPS_DECLARE_H_
|
||||
|
|
@ -0,0 +1,90 @@
|
|||
/**
|
||||
* Copyright 2019 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#include "transform/graph_ir/op_declare/image_ops_declare.h"
|
||||
#include <vector>
|
||||
|
||||
namespace mindspore::transform {
|
||||
// ResizeNearestNeighborV2D
|
||||
INPUT_MAP(ResizeNearestNeighborV2D) = {{1, INPUT_DESC(x)}};/*
|
||||
将x处理并存入对应INPUT_DESC结构体的相应变量中
|
||||
将x内容转为字符串变量并存储至结构体的name变量中
|
||||
引用Operator空间并将指针所指的类转为INPUT_DESC结构体
|
||||
将ResizeNearestNeighborV2D的类型与标准进行对比，后进行空间调整，并用指针input_map_为key存储相应内容
|
||||
*/
|
||||
ATTR_MAP(ResizeNearestNeighborV2D) = {
|
||||
{"size", ATTR_DESC(size, AnyTraits<std::vector<int64_t>>(), AnyTraits<std::vector<int64_t>>())},
|
||||
{"align_corners", ATTR_DESC(align_corners, AnyTraits<bool>())}};/*
|
||||
将size处理并存入对应ATTR_DESC结构体的相应变量中
|
||||
将size内容转为字符串变量并存储至结构体的name变量中
|
||||
引用Operator空间并将指针所指的类转为ATTR_DESC结构体
|
||||
//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
//存入MaxPool对应空间并用attr_map_指针保存
|
||||
//其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
//std:vector<>的作用为构建<>内类型的容量可变的数组
|
||||
将ResizeNearestNeighborV2D的类型与标准进行对比，后进行空间调整，并用指针attr_map_为key存储相应内容
|
||||
*/
|
||||
OUTPUT_MAP(ResizeNearestNeighborV2D) = {{0, OUTPUT_DESC(y)}};/*
|
||||
将y处理并存入对应OUTPUT_DESC结构体的相应变量中
|
||||
将x内容转为字符串变量并存储至结构体的name变量中
|
||||
引用Operator空间并将指针所指的类转为OUTPUT_DESC结构体
|
||||
将ResizeNearestNeighborV2D的类型与标准进行对比，后进行空间调整，并用指针output_map_为key存储相应内容
|
||||
*/
|
||||
REG_ADPT_DESC(ResizeNearestNeighborV2D, kNameResizeNearestNeighborD, ADPT_DESC(ResizeNearestNeighborV2D))/*
|
||||
将ResizeNearestNeighborV2D处理并存入对应ADPT_DESC结构体的相应变量中
|
||||
将ResizeNearestNeighborV2D内容转为字符串变量并存储至结构体的name变量中
|
||||
引用Operator空间并将指针所指的类转为ADPT_DESC结构体
|
||||
|
||||
再将ResizeNearestNeighborV2D处理并存入对应REG_ADPT_DESC结构体的相应变量中
|
||||
将ResizeNearestNeighborV2D内容转为字符串变量并存储至REG_ADPT_DESC的结构体的name变量中
|
||||
引用Operator空间并将指针所指的类转为REG_ADPT_DESC结构体
|
||||
*/
|
||||
|
||||
// ResizeNearestNeighborV2
|
||||
INPUT_MAP(ResizeNearestNeighborV2) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(size)}};
|
||||
ATTR_MAP(ResizeNearestNeighborV2) = {{"align_corners", ATTR_DESC(align_corners, AnyTraits<bool>())},
|
||||
{"half_pixel_centers", ATTR_DESC(half_pixel_centers, AnyTraits<bool>())}};
|
||||
OUTPUT_MAP(ResizeNearestNeighborV2) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(ResizeNearestNeighborV2, kNameResizeNearestNeighborV2, ADPT_DESC(ResizeNearestNeighborV2))
|
||||
|
||||
// ResizeNearestNeighborV2Grad
|
||||
INPUT_MAP(ResizeNearestNeighborV2Grad) = {{1, INPUT_DESC(grads)}, {2, INPUT_DESC(size)}};
|
||||
ATTR_MAP(ResizeNearestNeighborV2Grad) = {{"align_corners", ATTR_DESC(align_corners, AnyTraits<bool>())}};
|
||||
OUTPUT_MAP(ResizeNearestNeighborV2Grad) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(ResizeNearestNeighborV2Grad, kNameResizeNearestNeighborGrad, ADPT_DESC(ResizeNearestNeighborV2Grad))
|
||||
|
||||
// ResizeBilinearV2Grad
|
||||
INPUT_MAP(ResizeBilinearV2Grad) = {{1, INPUT_DESC(grads)}, {2, INPUT_DESC(original_image)}};
|
||||
ATTR_MAP(ResizeBilinearV2Grad) = {{"align_corners", ATTR_DESC(align_corners, AnyTraits<bool>())}};
|
||||
OUTPUT_MAP(ResizeBilinearV2Grad) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(ResizeBilinearV2Grad, kNameResizeBilinearGrad, ADPT_DESC(ResizeBilinearV2Grad))
|
||||
|
||||
// ResizeBilinearV2D
|
||||
INPUT_MAP(ResizeBilinearV2D) = {{1, INPUT_DESC(x)}};
|
||||
ATTR_MAP(ResizeBilinearV2D) = {
|
||||
{"size", ATTR_DESC(size, AnyTraits<std::vector<int64_t>>(), AnyTraits<std::vector<int64_t>>())},
|
||||
{"align_corners", ATTR_DESC(align_corners, AnyTraits<bool>())}};
|
||||
OUTPUT_MAP(ResizeBilinearV2D) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(ResizeBilinearV2D, kNameResizeBilinear, ADPT_DESC(ResizeBilinearV2D))
|
||||
|
||||
// CropAndResize
|
||||
INPUT_MAP(CropAndResize) = {
|
||||
{1, INPUT_DESC(x)}, {2, INPUT_DESC(boxes)}, {3, INPUT_DESC(box_index)}, {4, INPUT_DESC(crop_size)}};
|
||||
ATTR_MAP(CropAndResize) = {{"extrapolation_value", ATTR_DESC(extrapolation_value, AnyTraits<float>())},
|
||||
{"method", ATTR_DESC(method, AnyTraits<std::string>())}};
|
||||
OUTPUT_MAP(CropAndResize) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(CropAndResize, kNameCropAndResize, ADPT_DESC(CropAndResize))
|
||||
} // namespace mindspore::transform
|
||||
|
|
@ -0,0 +1,44 @@
|
|||
/**
|
||||
* Copyright 2019-2021 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_IMAGE_OPS_DECLARE_H_
|
||||
#define MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_IMAGE_OPS_DECLARE_H_
|
||||
|
||||
#include <string>
|
||||
#include "utils/hash_map.h"
|
||||
#include "transform/graph_ir/op_declare/op_declare_macro.h"
|
||||
#include "ops/image_ops.h"
|
||||
|
||||
namespace mindspore::transform {
|
||||
DECLARE_OP_ADAPTER(ResizeNearestNeighborV2D)//将ResizeNearestNeighborV2D收录后与标准进行比较，进行空间调整，用input_map_为key储存
|
||||
DECLARE_OP_USE_OUTPUT(ResizeNearestNeighborV2D)//将ResizeNearestNeighborV2D收录后与标准进行比较，进行空间调整，用output_map_为key储存
|
||||
|
||||
DECLARE_OP_ADAPTER(ResizeNearestNeighborV2)
|
||||
DECLARE_OP_USE_OUTPUT(ResizeNearestNeighborV2)
|
||||
|
||||
DECLARE_OP_ADAPTER(ResizeNearestNeighborV2Grad)
|
||||
DECLARE_OP_USE_OUTPUT(ResizeNearestNeighborV2Grad)
|
||||
|
||||
DECLARE_OP_ADAPTER(ResizeBilinearV2D)
|
||||
DECLARE_OP_USE_OUTPUT(ResizeBilinearV2D)
|
||||
|
||||
DECLARE_OP_ADAPTER(ResizeBilinearV2Grad)
|
||||
DECLARE_OP_USE_OUTPUT(ResizeBilinearV2Grad)
|
||||
|
||||
DECLARE_OP_ADAPTER(CropAndResize)
|
||||
DECLARE_OP_USE_OUTPUT(CropAndResize)
|
||||
} // namespace mindspore::transform
|
||||
#endif // MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_IMAGE_OPS_DECLARE_H_
|
||||
|
|
@ -0,0 +1,44 @@
|
|||
/**
|
||||
* Copyright 2021 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#include "transform/graph_ir/io_format_map.h"
|
||||
|
||||
namespace mindspore {
|
||||
namespace transform {
|
||||
//用io_format_map_指针储存数据
|
||||
mindspore::HashMap<std::string, std::string> IOFormatMap::io_format_map_ = {{"BasicLSTMCell", "ND"},
|
||||
{"BasicLSTMCellInputGrad", "ND"},
|
||||
{"BasicLSTMCellCStateGrad", "ND"},
|
||||
{"Dequant", "ND"},
|
||||
{"DynamicGRUV2", "ND"},
|
||||
{"DynamicGRUV2Grad", "ND"},
|
||||
{"DynamicRNN", "ND"},
|
||||
{"DynamicRNNGrad", "ND"},
|
||||
{"MatMul", "ND"},
|
||||
{"BatchMatMul", "ND"},
|
||||
{"BatchMatMulV2", "ND"},
|
||||
{"Quant", "ND"},
|
||||
{"BasicLSTMCellWeightGrad", "HWCN"},
|
||||
{"ExtractImagePatches", "NCHW"},
|
||||
{"Conv3D", "format"},
|
||||
{"MaxPool3D", "NCDHW"},
|
||||
{"Conv3DBackpropFilter", "format"},
|
||||
{"Conv3DBackpropInput", "format"},
|
||||
{"Conv3DTranspose", "format"}};
|
||||
mindspore::HashMap<std::string, std::string> &IOFormatMap::get() { return io_format_map_; }
|
||||
//采用返回io_format_map_指针的引用的方式，防止因连续赋值导致不必要的计算开销
|
||||
} // namespace transform
|
||||
} // namespace mindspore
|
||||
|
|
@ -0,0 +1,34 @@
|
|||
/**
|
||||
* Copyright 2021 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
#ifndef MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_IO_FORMAT_MAP_H_
|
||||
#define MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_IO_FORMAT_MAP_H_
|
||||
|
||||
#include <string>
|
||||
#include "utils/hash_map.h"
|
||||
|
||||
namespace mindspore {
|
||||
namespace transform {
|
||||
//定义IOFormatMap类
|
||||
class IOFormatMap {
|
||||
public:
|
||||
static mindspore::HashMap<std::string, std::string> &get();//定义方法get()，返回值为对象的引用
|
||||
|
||||
private:
|
||||
static mindspore::HashMap<std::string, std::string> io_format_map_;//定义私有指针io_format_map_
|
||||
};
|
||||
} // namespace transform
|
||||
} // namespace mindspore
|
||||
#endif // MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_IO_FORMAT_MAP_H_
|
||||
|
|
@ -0,0 +1,46 @@
|
|||
/**
|
||||
* Copyright 2019 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#include "transform/graph_ir/op_declare/logging_ops_declare.h"
|
||||
|
||||
namespace mindspore::transform {
|
||||
// Print
|
||||
INPUT_MAP(Print) = EMPTY_INPUT_MAP;//将Print的类型与标准进行对比，后进行空间调整，使原input_map_为储存相应内容的key为空
|
||||
DYN_INPUT_MAP(Print) = {{1, DYN_INPUT_DESC(x)}};/*
|
||||
将x处理并存入对应DYN_INPUT_DESC结构体的相应变量中
|
||||
将x内容转为字符串变量并存储至结构体的name变量中
|
||||
引用Operator空间并将指针所指的类转为DYN_INPUT_DESC结构体
|
||||
将Print的类型与标准进行对比，后进行空间调整，并用指针dyn_input_map_为key存储相应内容
|
||||
*/
|
||||
ATTR_MAP(Print) = EMPTY_ATTR_MAP;//将TensorScatterUpdate的类型与标准进行对比，后进行空间调整，使原attr_map_为储存相应内容的key为空
|
||||
REG_ADPT_DESC(Print, kNamePrint, ADPT_DESC(Print))/*
|
||||
将Print处理并存入对应ADPT_DESC结构体的相应变量中
|
||||
将Print内容转为字符串变量并存储至结构体的name变量中
|
||||
引用Operator空间并将指针所指的类转为ADPT_DESC结构体
|
||||
|
||||
再将Print处理并存入对应REG_ADPT_DESC结构体的相应变量中
|
||||
将Print内容转为字符串变量并存储至REG_ADPT_DESC的结构体的name变量中
|
||||
引用Operator空间并将指针所指的类转为REG_ADPT_DESC结构体
|
||||
*/
|
||||
|
||||
|
||||
#ifdef ENABLE_D
|
||||
INPUT_MAP(Assert) = {{1, INPUT_DESC(input_condition)}};
|
||||
DYN_INPUT_MAP(Assert) = {{2, DYN_INPUT_DESC(input_data)}};
|
||||
ATTR_MAP(Assert) = {{"summarize", ATTR_DESC(summarize, AnyTraits<int64_t>())}};
|
||||
REG_ADPT_DESC(Assert, kNameAssert, ADPT_DESC(Assert))
|
||||
#endif
|
||||
} // namespace mindspore::transform
|
||||
|
|
@ -0,0 +1,34 @@
|
|||
/**
|
||||
* Copyright 2019-2021 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_LOGGING_OPS_DECLARE_H_
|
||||
#define MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_LOGGING_OPS_DECLARE_H_
|
||||
|
||||
#include <string>
|
||||
#include "utils/hash_map.h"
|
||||
#include "transform/graph_ir/op_declare/op_declare_macro.h"
|
||||
#include "ops/logging_ops.h"
|
||||
|
||||
namespace mindspore::transform {
|
||||
DECLARE_OP_ADAPTER(Print)//将Print收录后与标准进行比较，进行空间调整，用input_map_为key储存
|
||||
DECLARE_OP_USE_DYN_INPUT(Print)//将Print收录后与标准进行比较，进行空间调整，用dyn_input_map_为key储存
|
||||
|
||||
#ifdef ENABLE_D
|
||||
DECLARE_OP_ADAPTER(Assert)
|
||||
DECLARE_OP_USE_DYN_INPUT(Assert)
|
||||
#endif
|
||||
} // namespace mindspore::transform
|
||||
#endif // MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_LOGGING_OPS_DECLARE_H_
|
||||
|
|
@ -0,0 +1,132 @@
|
|||
/**
|
||||
* Copyright 2021 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#include "transform/graph_ir/op_declare/math_ops_declare.h"
|
||||
#include <vector>
|
||||
|
||||
namespace mindspore::transform {
|
||||
// ActsULQ
|
||||
INPUT_MAP(ActsULQ) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(clamp_min)}, {3, INPUT_DESC(clamp_max)}};
|
||||
//以其中一句为例
|
||||
/*
|
||||
将x处理并存入对应INPUT_DESC结构体的相应变量中
|
||||
将x内容转为字符串变量并存储至结构体的name变量中
|
||||
引用Operator空间并将指针所指的类转为INPUT_DESC结构体
|
||||
将ActsULQ的类型与标准进行对比，后进行空间调整，并用指针input_map_为key存储相应内容
|
||||
*/
|
||||
ATTR_MAP(ActsULQ) = {{"fixed_min", ATTR_DESC(fixed_min, AnyTraits<bool>())},
|
||||
{"num_bits", ATTR_DESC(num_bits, AnyTraits<int64_t>())}};
|
||||
/*
|
||||
将fixed_min处理并存入对应ATTR_DESC结构体的相应变量中
|
||||
将fixed_min内容转为字符串变量并存储至结构体的name变量中
|
||||
引用Operator空间并将指针所指的类转为ATTR_DESC结构体
|
||||
//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
//存入MaxPool对应空间并用attr_map_指针保存
|
||||
//其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
将ActsULQ的类型与标准进行对比，后进行空间调整，并用指针attr_map_为key存储相应内容
|
||||
*/
|
||||
OUTPUT_MAP(ActsULQ) = {{0, OUTPUT_DESC(y)},
|
||||
{1, OUTPUT_DESC(clamp_min_mask)},
|
||||
{2, OUTPUT_DESC(clamp_max_mask)},
|
||||
{3, OUTPUT_DESC(x_clamped_loss)}};/*
|
||||
将y处理并存入对应INPUT_DESC结构体的相应变量中
|
||||
将x内容转为字符串变量并存储至结构体的name变量中
|
||||
引用Operator空间并将指针所指的类转为INPUT_DESC结构体
|
||||
将ActsULQ的类型与标准进行对比，后进行空间调整，并用指针output_map_为key存储相应内容
|
||||
*/
|
||||
REG_ADPT_DESC(ActsULQ, kNameActsULQ, ADPT_DESC(ActsULQ))/*
|
||||
将ActsULQ处理并存入对应ADPT_DESC结构体的相应变量中
|
||||
将ActsULQ内容转为字符串变量并存储至结构体的name变量中
|
||||
引用Operator空间并将指针所指的类转为ADPT_DESC结构体
|
||||
|
||||
再将ActsULQ处理并存入对应REG_ADPT_DESC结构体的相应变量中
|
||||
将ActsULQ内容转为字符串变量并存储至REG_ADPT_DESC的结构体的name变量中
|
||||
引用Operator空间并将指针所指的类转为REG_ADPT_DESC结构体
|
||||
*/
|
||||
|
||||
// ActsULQInputGrad
|
||||
INPUT_MAP(ActsULQInputGrad) = {
|
||||
{1, INPUT_DESC(y_grad)}, {2, INPUT_DESC(clamp_min_mask)}, {3, INPUT_DESC(clamp_max_mask)}};
|
||||
ATTR_MAP(ActsULQInputGrad) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(ActsULQInputGrad) = {{0, OUTPUT_DESC(x_grad)}};
|
||||
REG_ADPT_DESC(ActsULQInputGrad, kNameActsULQInputGrad, ADPT_DESC(ActsULQInputGrad))
|
||||
|
||||
// ActULQClampMaxGrad
|
||||
INPUT_MAP(ActULQClampMaxGrad) = {
|
||||
{1, INPUT_DESC(y_grad)}, {2, INPUT_DESC(clamp_max_mask)}, {3, INPUT_DESC(x_clamped_loss)}};
|
||||
ATTR_MAP(ActULQClampMaxGrad) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(ActULQClampMaxGrad) = {{0, OUTPUT_DESC(clamp_max_grad)}};
|
||||
REG_ADPT_DESC(ActULQClampMaxGrad, kNameActULQClampMaxGrad, ADPT_DESC(ActULQClampMaxGrad))
|
||||
|
||||
// ActULQClampMinGrad
|
||||
INPUT_MAP(ActULQClampMinGrad) = {
|
||||
{1, INPUT_DESC(y_grad)}, {2, INPUT_DESC(clamp_min_mask)}, {3, INPUT_DESC(x_clamped_loss)}};
|
||||
ATTR_MAP(ActULQClampMinGrad) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(ActULQClampMinGrad) = {{0, OUTPUT_DESC(clamp_min_grad)}};
|
||||
REG_ADPT_DESC(ActULQClampMinGrad, kNameActULQClampMinGrad, ADPT_DESC(ActULQClampMinGrad))
|
||||
|
||||
// HistogramFixedWidthD
|
||||
INPUT_MAP(HistogramFixedWidthD) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(range)}};
|
||||
ATTR_MAP(HistogramFixedWidthD) = {{"nbins", ATTR_DESC(nbins, AnyTraits<int64_t>())},
|
||||
{"dtype", ATTR_DESC(dtype, AnyTraits<int64_t>())}};
|
||||
OUTPUT_MAP(HistogramFixedWidthD) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(HistogramFixedWidthD, kNameHistogramFixedWidthD, ADPT_DESC(HistogramFixedWidthD))
|
||||
|
||||
// IFMR
|
||||
INPUT_MAP(IFMR) = {
|
||||
{1, INPUT_DESC(data)}, {2, INPUT_DESC(data_min)}, {3, INPUT_DESC(data_max)}, {4, INPUT_DESC(cumsum)}};
|
||||
ATTR_MAP(IFMR) = {{"min_percentile", ATTR_DESC(min_percentile, AnyTraits<float>())},
|
||||
{"max_percentile", ATTR_DESC(max_percentile, AnyTraits<float>())},
|
||||
{"search_range", ATTR_DESC(search_range, AnyTraits<std::vector<float>>())},
|
||||
{"search_step", ATTR_DESC(search_step, AnyTraits<float>())}};
|
||||
OUTPUT_MAP(IFMR) = {{0, OUTPUT_DESC(scale)}, {1, OUTPUT_DESC(offset)}};
|
||||
REG_ADPT_DESC(IFMR, kNameIFMR, ADPT_DESC(IFMR))
|
||||
|
||||
// NLLLoss
|
||||
INPUT_MAP(NLLLoss) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(target)}, {3, INPUT_DESC(weight)}};
|
||||
ATTR_MAP(NLLLoss) = {{"reduction", ATTR_DESC(reduction, AnyTraits<std::string>())}};
|
||||
OUTPUT_MAP(NLLLoss) = {{0, OUTPUT_DESC(y)}, {1, OUTPUT_DESC(total_weight)}};
|
||||
REG_ADPT_DESC(NLLLoss, kNameNLLLoss, ADPT_DESC(NLLLoss))
|
||||
|
||||
// NLLLossGrad
|
||||
INPUT_MAP(NLLLossGrad) = {{1, INPUT_DESC(x)},
|
||||
{2, INPUT_DESC(y_grad)},
|
||||
{3, INPUT_DESC(target)},
|
||||
{4, INPUT_DESC(weight)},
|
||||
{5, INPUT_DESC(total_weight)}};
|
||||
ATTR_MAP(NLLLossGrad) = {{"reduction", ATTR_DESC(reduction, AnyTraits<std::string>())}};
|
||||
OUTPUT_MAP(NLLLossGrad) = {{0, OUTPUT_DESC(x_grad)}};
|
||||
REG_ADPT_DESC(NLLLossGrad, kNameNLLLossGrad, ADPT_DESC(NLLLossGrad))
|
||||
|
||||
// Erf
|
||||
INPUT_MAP(Erf) = {{1, INPUT_DESC(x)}};
|
||||
ATTR_MAP(Erf) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(Erf) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(Erf, kNameErf, ADPT_DESC(Erf))
|
||||
|
||||
// Erfc
|
||||
INPUT_MAP(Erfc) = {{1, INPUT_DESC(x)}};
|
||||
ATTR_MAP(Erfc) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(Erfc) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(Erfc, kNameErfc, ADPT_DESC(Erfc))
|
||||
|
||||
// WtsARQ
|
||||
INPUT_MAP(WtsARQ) = {{1, INPUT_DESC(w)}, {2, INPUT_DESC(w_min)}, {3, INPUT_DESC(w_max)}};
|
||||
ATTR_MAP(WtsARQ) = {{"num_bits", ATTR_DESC(num_bits, AnyTraits<int64_t>())},
|
||||
{"offset_flag", ATTR_DESC(offset_flag, AnyTraits<bool>())}};
|
||||
OUTPUT_MAP(WtsARQ) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(WtsARQ, kNameWtsARQ, ADPT_DESC(WtsARQ))
|
||||
} // namespace mindspore::transform
|
||||
|
|
@ -0,0 +1,59 @@
|
|||
/**
|
||||
* Copyright 2021 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_MATH_OPS_DECLARE_H_
|
||||
#define MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_MATH_OPS_DECLARE_H_
|
||||
|
||||
#include <string>
|
||||
#include "utils/hash_map.h"
|
||||
#include "transform/graph_ir/op_declare/op_declare_macro.h"
|
||||
#include "ops/math_ops.h"
|
||||
|
||||
namespace mindspore::transform {
|
||||
DECLARE_OP_ADAPTER(ActsULQ)//将ActsULQ收录后与标准进行比较，进行空间调整，用input_map_为key储存
|
||||
DECLARE_OP_USE_OUTPUT(ActsULQ)//将ActsULQ收录后与标准进行比较，进行空间调整，用output_map_为key储存
|
||||
|
||||
DECLARE_OP_ADAPTER(ActsULQInputGrad)
|
||||
DECLARE_OP_USE_OUTPUT(ActsULQInputGrad)
|
||||
|
||||
DECLARE_OP_ADAPTER(ActULQClampMaxGrad)
|
||||
DECLARE_OP_USE_OUTPUT(ActULQClampMaxGrad)
|
||||
|
||||
DECLARE_OP_ADAPTER(ActULQClampMinGrad)
|
||||
DECLARE_OP_USE_OUTPUT(ActULQClampMinGrad)
|
||||
|
||||
DECLARE_OP_ADAPTER(HistogramFixedWidthD)
|
||||
DECLARE_OP_USE_OUTPUT(HistogramFixedWidthD)
|
||||
|
||||
DECLARE_OP_ADAPTER(IFMR)
|
||||
DECLARE_OP_USE_OUTPUT(IFMR)
|
||||
|
||||
DECLARE_OP_ADAPTER(NLLLoss)
|
||||
DECLARE_OP_USE_OUTPUT(NLLLoss)
|
||||
|
||||
DECLARE_OP_ADAPTER(NLLLossGrad)
|
||||
DECLARE_OP_USE_OUTPUT(NLLLossGrad)
|
||||
|
||||
DECLARE_OP_ADAPTER(Erf)
|
||||
DECLARE_OP_USE_OUTPUT(Erf)
|
||||
|
||||
DECLARE_OP_ADAPTER(Erfc)
|
||||
DECLARE_OP_USE_OUTPUT(Erfc)
|
||||
|
||||
DECLARE_OP_ADAPTER(WtsARQ)
|
||||
DECLARE_OP_USE_OUTPUT(WtsARQ)
|
||||
} // namespace mindspore::transform
|
||||
#endif // MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_MATH_OPS_DECLARE_H_
|
||||
|
|
@ -0,0 +1,180 @@
|
|||
/**
|
||||
* Copyright 2019-2021 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#include "transform/graph_ir/op_declare/matrix_calculation_ops_declare.h"
|
||||
|
||||
namespace mindspore::transform {
|
||||
// TensorScatterUpdate
|
||||
INPUT_MAP(TensorScatterUpdate) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(indices)}, {3, INPUT_DESC(updates)}};
|
||||
//以其中一句为例
|
||||
/*
|
||||
将x处理并存入对应INPUT_DESC结构体的相应变量中
|
||||
将x内容转为字符串变量并存储至结构体的name变量中
|
||||
引用Operator空间并将指针所指的类转为INPUT_DESC结构体
|
||||
将TensorScatterUpdate的类型与标准进行对比，后进行空间调整，并用指针input_map_为key存储相应内容
|
||||
*/
|
||||
ATTR_MAP(TensorScatterUpdate) = EMPTY_ATTR_MAP;//将TensorScatterUpdate的类型与标准进行对比，后进行空间调整，使原attr_map_为储存相应内容的key为空
|
||||
OUTPUT_MAP(TensorScatterUpdate) = {{0, OUTPUT_DESC(y)}};/*
|
||||
将y处理并存入对应OUTPUT_DESC结构体的相应变量中
|
||||
将y内容转为字符串变量并存储至结构体的name变量中
|
||||
引用Operator空间并将指针所指的类转为INPUT_DESC结构体
|
||||
将TensorScatterUpdate的类型与标准进行对比，后进行空间调整，并用指针output_map_为key存储相应内容
|
||||
*/
|
||||
REG_ADPT_DESC(TensorScatterUpdate, kNameTensorScatterUpdate, ADPT_DESC(TensorScatterUpdate))/*
|
||||
将TensorScatterUpdate处理并存入对应ADPT_DESC结构体的相应变量中
|
||||
将TensorScatterUpdate内容转为字符串变量并存储至结构体的name变量中
|
||||
引用Operator空间并将指针所指的类转为ADPT_DESC结构体
|
||||
|
||||
再将TensorScatterUpdate处理并存入对应REG_ADPT_DESC结构体的相应变量中
|
||||
将TensorScatterUpdate内容转为字符串变量并存储至REG_ADPT_DESC的结构体的name变量中
|
||||
引用Operator空间并将指针所指的类转为REG_ADPT_DESC结构体
|
||||
*/
|
||||
|
||||
// ScatterUpdate
|
||||
INPUT_MAP(ScatterUpdate) = {{1, INPUT_DESC(var)}, {2, INPUT_DESC(indices)}, {3, INPUT_DESC(updates)}};
|
||||
ATTR_MAP(ScatterUpdate) = {{"use_locking", ATTR_DESC(use_locking, AnyTraits<bool>())}};
|
||||
OUTPUT_MAP(ScatterUpdate) = {{0, OUTPUT_DESC(var)}};
|
||||
REG_ADPT_DESC(ScatterUpdate, kNameScatterUpdate, ADPT_DESC(ScatterUpdate))
|
||||
|
||||
// ScatterNdUpdate
|
||||
INPUT_MAP(ScatterNdUpdate) = {{1, INPUT_DESC(var)}, {2, INPUT_DESC(indices)}, {3, INPUT_DESC(updates)}};
|
||||
ATTR_MAP(ScatterNdUpdate) = {{"use_locking", ATTR_DESC(use_locking, AnyTraits<bool>())}};
|
||||
OUTPUT_MAP(ScatterNdUpdate) = {{0, OUTPUT_DESC(var)}};
|
||||
REG_ADPT_DESC(ScatterNdUpdate, kNameScatterNdUpdate, ADPT_DESC(ScatterNdUpdate))
|
||||
|
||||
// ScatterMax
|
||||
INPUT_MAP(ScatterMax) = {{1, INPUT_DESC(var)}, {2, INPUT_DESC(indices)}, {3, INPUT_DESC(updates)}};
|
||||
ATTR_MAP(ScatterMax) = {{"use_locking", ATTR_DESC(use_locking, AnyTraits<bool>())}};
|
||||
OUTPUT_MAP(ScatterMax) = {{0, OUTPUT_DESC(var)}};
|
||||
REG_ADPT_DESC(ScatterMax, kNameScatterMax, ADPT_DESC(ScatterMax))
|
||||
|
||||
// ScatterMin
|
||||
INPUT_MAP(ScatterMin) = {{1, INPUT_DESC(var)}, {2, INPUT_DESC(indices)}, {3, INPUT_DESC(updates)}};
|
||||
ATTR_MAP(ScatterMin) = {{"use_locking", ATTR_DESC(use_locking, AnyTraits<bool>())}};
|
||||
OUTPUT_MAP(ScatterMin) = {{0, OUTPUT_DESC(var)}};
|
||||
REG_ADPT_DESC(ScatterMin, kNameScatterMin, ADPT_DESC(ScatterMin))
|
||||
|
||||
// ScatterAdd
|
||||
INPUT_MAP(ScatterAdd) = {{1, INPUT_DESC(var)}, {2, INPUT_DESC(indices)}, {3, INPUT_DESC(updates)}};
|
||||
ATTR_MAP(ScatterAdd) = {{"use_locking", ATTR_DESC(use_locking, AnyTraits<bool>())}};
|
||||
OUTPUT_MAP(ScatterAdd) = {{0, OUTPUT_DESC(var)}};
|
||||
REG_ADPT_DESC(ScatterAdd, kNameScatterAdd, ADPT_DESC(ScatterAdd))
|
||||
|
||||
// ScatterSub
|
||||
INPUT_MAP(ScatterSub) = {{1, INPUT_DESC(var)}, {2, INPUT_DESC(indices)}, {3, INPUT_DESC(updates)}};
|
||||
ATTR_MAP(ScatterSub) = {{"use_locking", ATTR_DESC(use_locking, AnyTraits<bool>())}};
|
||||
OUTPUT_MAP(ScatterSub) = {{0, OUTPUT_DESC(var)}};
|
||||
REG_ADPT_DESC(ScatterSub, kNameScatterSub, ADPT_DESC(ScatterSub))
|
||||
|
||||
// ScatterMul
|
||||
INPUT_MAP(ScatterMul) = {{1, INPUT_DESC(var)}, {2, INPUT_DESC(indices)}, {3, INPUT_DESC(updates)}};
|
||||
ATTR_MAP(ScatterMul) = {{"use_locking", ATTR_DESC(use_locking, AnyTraits<bool>())}};
|
||||
OUTPUT_MAP(ScatterMul) = {{0, OUTPUT_DESC(var)}};
|
||||
REG_ADPT_DESC(ScatterMul, kNameScatterMul, ADPT_DESC(ScatterMul))
|
||||
|
||||
// ScatterDiv
|
||||
INPUT_MAP(ScatterDiv) = {{1, INPUT_DESC(var)}, {2, INPUT_DESC(indices)}, {3, INPUT_DESC(updates)}};
|
||||
ATTR_MAP(ScatterDiv) = {{"use_locking", ATTR_DESC(use_locking, AnyTraits<bool>())}};
|
||||
OUTPUT_MAP(ScatterDiv) = {{0, OUTPUT_DESC(var)}};
|
||||
REG_ADPT_DESC(ScatterDiv, kNameScatterDiv, ADPT_DESC(ScatterDiv))
|
||||
|
||||
// ScatterNdAdd
|
||||
INPUT_MAP(ScatterNdAdd) = {{1, INPUT_DESC(var)}, {2, INPUT_DESC(indices)}, {3, INPUT_DESC(updates)}};
|
||||
ATTR_MAP(ScatterNdAdd) = {{"use_locking", ATTR_DESC(use_locking, AnyTraits<bool>())}};
|
||||
OUTPUT_MAP(ScatterNdAdd) = {{0, OUTPUT_DESC(var)}};
|
||||
REG_ADPT_DESC(ScatterNdAdd, kNameScatterNdAdd, ADPT_DESC(ScatterNdAdd))
|
||||
|
||||
// ScatterNdSub
|
||||
INPUT_MAP(ScatterNdSub) = {{1, INPUT_DESC(var)}, {2, INPUT_DESC(indices)}, {3, INPUT_DESC(updates)}};
|
||||
ATTR_MAP(ScatterNdSub) = {{"use_locking", ATTR_DESC(use_locking, AnyTraits<bool>())}};
|
||||
OUTPUT_MAP(ScatterNdSub) = {{0, OUTPUT_DESC(var)}};
|
||||
REG_ADPT_DESC(ScatterNdSub, kNameScatterNdSub, ADPT_DESC(ScatterNdSub))
|
||||
|
||||
// MatMul
|
||||
INPUT_MAP(MatMul) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}, {3, INPUT_DESC(bias)}};
|
||||
ATTR_MAP(MatMul) = {{"transpose_x1", ATTR_DESC(transpose_x1, AnyTraits<bool>())},
|
||||
{"transpose_x2", ATTR_DESC(transpose_x2, AnyTraits<bool>())}};
|
||||
OUTPUT_MAP(MatMul) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(MatMul, kNameMatMul, ADPT_DESC(MatMul))
|
||||
|
||||
// MatMulV2
|
||||
INPUT_MAP(MatMulV2) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}, {3, INPUT_DESC(bias)}};
|
||||
ATTR_MAP(MatMulV2) = {{"transpose_a", ATTR_DESC(transpose_x1, AnyTraits<bool>())},
|
||||
{"transpose_b", ATTR_DESC(transpose_x2, AnyTraits<bool>())}};
|
||||
OUTPUT_MAP(MatMulV2) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(MatMulV2, prim::kPrimMatMul->name(), ADPT_DESC(MatMulV2))
|
||||
|
||||
// MatrixDiagD
|
||||
INPUT_MAP(MatrixDiagD) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(assist)}};
|
||||
ATTR_MAP(MatrixDiagD) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(MatrixDiagD) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(MatrixDiagD, kNameMatrixDiagD, ADPT_DESC(MatrixDiagD))
|
||||
|
||||
// MatrixDiagPartD
|
||||
INPUT_MAP(MatrixDiagPartD) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(assist)}};
|
||||
ATTR_MAP(MatrixDiagPartD) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(MatrixDiagPartD) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(MatrixDiagPartD, kNameMatrixDiagPartD, ADPT_DESC(MatrixDiagPartD))
|
||||
|
||||
// MatrixSetDiagD
|
||||
INPUT_MAP(MatrixSetDiagD) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(diagonal)}, {3, INPUT_DESC(assist)}};
|
||||
ATTR_MAP(MatrixSetDiagD) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(MatrixSetDiagD) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(MatrixSetDiagD, kNameMatrixSetDiagD, ADPT_DESC(MatrixSetDiagD))
|
||||
|
||||
// DiagPart
|
||||
INPUT_MAP(DiagPart) = {{1, INPUT_DESC(x)}};
|
||||
ATTR_MAP(DiagPart) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(DiagPart) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(DiagPart, kNameDiagPart, ADPT_DESC(DiagPart))
|
||||
|
||||
// BatchMatMul
|
||||
INPUT_MAP(BatchMatMul) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}};
|
||||
ATTR_MAP(BatchMatMul) = {{"transpose_x1", ATTR_DESC(adj_x1, AnyTraits<bool>())},
|
||||
{"transpose_x2", ATTR_DESC(adj_x2, AnyTraits<bool>())}};
|
||||
OUTPUT_MAP(BatchMatMul) = {{0, OUTPUT_DESC(y)}};
|
||||
|
||||
// BatchMatMul->BatchMatMulV2
|
||||
INPUT_MAP(BatchMatMulV2) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}};
|
||||
ATTR_MAP(BatchMatMulV2) = {{"transpose_x1", ATTR_DESC(adj_x1, AnyTraits<bool>())},
|
||||
{"transpose_x2", ATTR_DESC(adj_x2, AnyTraits<bool>())}};
|
||||
OUTPUT_MAP(BatchMatMulV2) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(BatchMatMul, kNameBatchMatMul, ADPT_DESC(BatchMatMul))
|
||||
REG_ADPT_DESC(BatchMatMulV2, kNameBatchMatMulV2, ADPT_DESC(BatchMatMulV2))
|
||||
|
||||
// L2Loss
|
||||
INPUT_MAP(L2Loss) = {{1, INPUT_DESC(x)}};
|
||||
ATTR_MAP(L2Loss) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(L2Loss) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(L2Loss, kNameL2Loss, ADPT_DESC(L2Loss))
|
||||
|
||||
// ScatterElements
|
||||
INPUT_MAP(ScatterElements) = {{1, INPUT_DESC(data)}, {2, INPUT_DESC(indices)}, {3, INPUT_DESC(updates)}};
|
||||
ATTR_MAP(ScatterElements) = {{"axis", ATTR_DESC(axis, AnyTraits<int64_t>())}};
|
||||
OUTPUT_MAP(ScatterElements) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(ScatterElements, kNameScatterElements, ADPT_DESC(ScatterElements))
|
||||
|
||||
// FullyConnection
|
||||
INPUT_MAP(FullyConnection) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(w)}, {3, INPUT_DESC(b)}, {4, INPUT_DESC(offset_w)}};
|
||||
|
||||
ATTR_MAP(FullyConnection) = {{"num_output", ATTR_DESC(num_output, AnyTraits<int64_t>())},
|
||||
{"transpose", ATTR_DESC(transpose, AnyTraits<bool>())},
|
||||
{"axis", ATTR_DESC(axis, AnyTraits<int64_t>())},
|
||||
{"offset_x", ATTR_DESC(offset_x, AnyTraits<int64_t>())}};
|
||||
|
||||
OUTPUT_MAP(FullyConnection) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(FullyConnection, kNameFullConnection, ADPT_DESC(FullyConnection))
|
||||
} // namespace mindspore::transform
|
||||
|
|
@ -0,0 +1,92 @@
|
|||
/**
|
||||
* Copyright 2019-2021 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_MATRIX_CALCULATION_OPS_DECLARE_H_
|
||||
#define MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_MATRIX_CALCULATION_OPS_DECLARE_H_
|
||||
|
||||
#include <string>
|
||||
#include "utils/hash_map.h"
|
||||
#include "transform/graph_ir/op_declare/op_declare_macro.h"
|
||||
#include "ops/matrix_calculation_ops.h"
|
||||
|
||||
namespace mindspore::transform {
|
||||
DECLARE_OP_ADAPTER(TensorScatterUpdate)//将TensorScatterUpdate收录后与标准进行比较，进行空间调整，用input_map_为key储存
|
||||
DECLARE_OP_USE_OUTPUT(TensorScatterUpdate)//将TensorScatterUpdate收录后与标准进行比较，进行空间调整，用output_map_为key储存
|
||||
|
||||
DECLARE_OP_ADAPTER(ScatterUpdate)
|
||||
DECLARE_OP_USE_OUTPUT(ScatterUpdate)
|
||||
|
||||
DECLARE_OP_ADAPTER(ScatterNdUpdate)
|
||||
DECLARE_OP_USE_OUTPUT(ScatterNdUpdate)
|
||||
|
||||
DECLARE_OP_ADAPTER(ScatterMax)
|
||||
DECLARE_OP_USE_OUTPUT(ScatterMax)
|
||||
|
||||
DECLARE_OP_ADAPTER(ScatterMin)
|
||||
DECLARE_OP_USE_OUTPUT(ScatterMin)
|
||||
|
||||
DECLARE_OP_ADAPTER(ScatterAdd)
|
||||
DECLARE_OP_USE_OUTPUT(ScatterAdd)
|
||||
|
||||
DECLARE_OP_ADAPTER(ScatterSub)
|
||||
DECLARE_OP_USE_OUTPUT(ScatterSub)
|
||||
|
||||
DECLARE_OP_ADAPTER(ScatterMul)
|
||||
DECLARE_OP_USE_OUTPUT(ScatterMul)
|
||||
|
||||
DECLARE_OP_ADAPTER(ScatterDiv)
|
||||
DECLARE_OP_USE_OUTPUT(ScatterDiv)
|
||||
|
||||
DECLARE_OP_ADAPTER(ScatterNdAdd)
|
||||
DECLARE_OP_USE_OUTPUT(ScatterNdAdd)
|
||||
|
||||
DECLARE_OP_ADAPTER(ScatterNdSub)
|
||||
DECLARE_OP_USE_OUTPUT(ScatterNdSub)
|
||||
|
||||
DECLARE_OP_ADAPTER(BatchMatMul)
|
||||
DECLARE_OP_USE_OUTPUT(BatchMatMul)
|
||||
|
||||
DECLARE_OP_ADAPTER(BatchMatMulV2)
|
||||
DECLARE_OP_USE_OUTPUT(BatchMatMulV2)
|
||||
|
||||
DECLARE_OP_ADAPTER(MatMul)
|
||||
DECLARE_OP_USE_OUTPUT(MatMul)
|
||||
|
||||
DECLARE_OP_ADAPTER(MatMulV2)
|
||||
DECLARE_OP_USE_OUTPUT(MatMulV2)
|
||||
|
||||
DECLARE_OP_ADAPTER(MatrixDiagD)
|
||||
DECLARE_OP_USE_OUTPUT(MatrixDiagD)
|
||||
|
||||
DECLARE_OP_ADAPTER(MatrixDiagPartD)
|
||||
DECLARE_OP_USE_OUTPUT(MatrixDiagPartD)
|
||||
|
||||
DECLARE_OP_ADAPTER(MatrixSetDiagD)
|
||||
DECLARE_OP_USE_OUTPUT(MatrixSetDiagD)
|
||||
|
||||
DECLARE_OP_ADAPTER(DiagPart)
|
||||
DECLARE_OP_USE_OUTPUT(DiagPart)
|
||||
|
||||
DECLARE_OP_ADAPTER(L2Loss)
|
||||
DECLARE_OP_USE_OUTPUT(L2Loss)
|
||||
|
||||
DECLARE_OP_ADAPTER(ScatterElements)
|
||||
DECLARE_OP_USE_OUTPUT(ScatterElements)
|
||||
|
||||
DECLARE_OP_ADAPTER(FullyConnection)
|
||||
DECLARE_OP_USE_OUTPUT(FullyConnection)
|
||||
} // namespace mindspore::transform
|
||||
#endif // MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_MATRIX_CALCULATION_OPS_DECLARE_H_
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,109 @@
|
|||
/**
|
||||
* Copyright 2019 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#include "transform/graph_ir/op_declare/nn_batch_norm_ops_declare.h"
|
||||
#include <vector>
|
||||
|
||||
namespace mindspore::transform {
|
||||
// BatchNorm
|
||||
INPUT_MAP(BatchNorm) = {{1, INPUT_DESC(x)},
|
||||
{2, INPUT_DESC(scale)},
|
||||
{3, INPUT_DESC(offset)},
|
||||
{4, INPUT_DESC(mean)},
|
||||
{5, INPUT_DESC(variance)}};
|
||||
ATTR_MAP(BatchNorm) = {{"format", ATTR_DESC(data_format, AnyTraits<std::string>())},
|
||||
{"epsilon", ATTR_DESC(epsilon, AnyTraits<float>())},
|
||||
{"is_training", ATTR_DESC(is_training, AnyTraits<bool>())}};
|
||||
OUTPUT_MAP(BatchNorm) = {{0, OUTPUT_DESC(y)},
|
||||
{1, OUTPUT_DESC(batch_mean)},
|
||||
{2, OUTPUT_DESC(batch_variance)},
|
||||
{3, OUTPUT_DESC(reserve_space_1)},
|
||||
{4, OUTPUT_DESC(reserve_space_2)}};
|
||||
// BNInference is BatchNorm for caffe
|
||||
//用这一部分做注释，宏定义用的比较全
|
||||
INPUT_MAP(BNInference) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(mean)}, {3, INPUT_DESC(variance)},
|
||||
{4, INPUT_DESC(momentum)}, {5, INPUT_DESC(scale)}, {6, INPUT_DESC(offset)}};
|
||||
//以其中一句为例
|
||||
/*
|
||||
将x处理并存入对应INPUT_DESC结构体的相应变量中
|
||||
将x内容转为字符串变量并存储至结构体的name变量中
|
||||
引用Operator空间并将指针所指的类转为INPUT_DESC结构体
|
||||
将BNInference的类型与标准进行对比，后进行空间调整，并用指针input_map_为key存储相应内容
|
||||
*/
|
||||
ATTR_MAP(BNInference) = {{"epsilon", ATTR_DESC(epsilon, AnyTraits<float>())},
|
||||
{"use_global_stats", ATTR_DESC(use_global_stats, AnyTraits<bool>())},
|
||||
{"mode", ATTR_DESC(mode, AnyTraits<int64_t>())}};
|
||||
/*
|
||||
将epsilon处理并存入对应ATTR_DESC结构体的相应变量中
|
||||
将epsilon内容转为字符串变量并存储至结构体的name变量中
|
||||
引用Operator空间并将指针所指的类转为ATTR_DESC结构体
|
||||
//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
//存入MaxPool对应空间并用attr_map_指针保存
|
||||
//其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
//std:vector<>的作用为构建<>内类型的容量可变的数组
|
||||
将BNInference的类型与标准进行对比，后进行空间调整，并用指针attr_map_为key存储相应内容
|
||||
*/
|
||||
OUTPUT_MAP(BNInference) = {{0, OUTPUT_DESC(y)}};/*
|
||||
将y处理并存入对应OUTPUT_DESC结构体的相应变量中
|
||||
将y内容转为字符串变量并存储至结构体的name变量中
|
||||
引用Operator空间并将指针所指的类转为INPUT_DESC结构体
|
||||
将BNInference的类型与标准进行对比，后进行空间调整，并用指针output_map_为key存储相应内容
|
||||
*/
|
||||
|
||||
REG_ADPT_DESC(BNInference, kNameBNInference, ADPT_DESC(BNInference))/*
|
||||
将NInference处理并存入对应ADPT_DESC结构体的相应变量中
|
||||
将NInference内容转为字符串变量并存储至结构体的name变量中
|
||||
引用Operator空间并将指针所指的类转为ADPT_DESC结构体
|
||||
|
||||
再将NInference处理并存入对应REG_ADPT_DESC结构体的相应变量中
|
||||
将NInference内容转为字符串变量并存储至REG_ADPT_DESC的结构体的name变量中
|
||||
引用Operator空间并将指针所指的类转为REG_ADPT_DESC结构体
|
||||
*/
|
||||
REG_ADPT_DESC(BatchNorm, kNameBatchNorm, ADPT_DESC(BatchNorm))
|
||||
REG_ADPT_DESC(FusedBatchNorm, kNameFusedBatchNorm, ADPT_DESC(BatchNorm))
|
||||
|
||||
// BatchNormGrad
|
||||
INPUT_MAP(BatchNormGrad) = {{1, INPUT_DESC(y_backprop)},
|
||||
{2, INPUT_DESC(x)},
|
||||
{3, INPUT_DESC(scale)},
|
||||
{4, INPUT_DESC(reserve_space_1)},
|
||||
{5, INPUT_DESC(reserve_space_2)}};
|
||||
ATTR_MAP(BatchNormGrad) = {{"format", ATTR_DESC(data_format, AnyTraits<std::string>())},
|
||||
{"epsilon", ATTR_DESC(epsilon, AnyTraits<float>())},
|
||||
{"is_training", ATTR_DESC(is_training, AnyTraits<bool>())}};
|
||||
OUTPUT_MAP(BatchNormGrad) = {{0, OUTPUT_DESC(x_backprop)},
|
||||
{1, OUTPUT_DESC(scale_backprop)},
|
||||
{2, OUTPUT_DESC(offset_backprop)},
|
||||
{3, OUTPUT_DESC(reserve_space_4)},
|
||||
{4, OUTPUT_DESC(reserve_space_5)}};
|
||||
REG_ADPT_DESC(BatchNormGrad, kNameBatchNormGrad, ADPT_DESC(BatchNormGrad))
|
||||
|
||||
// L2NormalizeGrad
|
||||
INPUT_MAP(L2NormalizeGrad) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(y)}, {3, INPUT_DESC(dy)}};
|
||||
ATTR_MAP(L2NormalizeGrad) = {
|
||||
{"axis", ATTR_DESC(dim, AnyTraits<std::vector<int64_t>>(), AnyTraits<std::vector<int64_t>>())},
|
||||
{"epsilon", ATTR_DESC(eps, AnyTraits<float>())}};
|
||||
OUTPUT_MAP(L2NormalizeGrad) = {{0, OUTPUT_DESC(dx)}};
|
||||
REG_ADPT_DESC(L2NormalizeGrad, kNameL2NormalizeGrad, ADPT_DESC(L2NormalizeGrad))
|
||||
|
||||
// L2Normalize
|
||||
INPUT_MAP(L2Normalize) = {{1, INPUT_DESC(x)}};
|
||||
ATTR_MAP(L2Normalize) = {
|
||||
{"axis", ATTR_DESC(axis, AnyTraits<std::vector<int64_t>>(), AnyTraits<std::vector<int64_t>>())},
|
||||
{"epsilon", ATTR_DESC(eps, AnyTraits<float>())}};
|
||||
OUTPUT_MAP(L2Normalize) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(L2Normalize, kNameL2Normalize, ADPT_DESC(L2Normalize))
|
||||
} // namespace mindspore::transform
|
||||
|
|
@ -0,0 +1,41 @@
|
|||
/**
|
||||
* Copyright 2019-2021 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_NN_BATCH_NORM_OPS_DECLARE_H_
|
||||
#define MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_NN_BATCH_NORM_OPS_DECLARE_H_
|
||||
|
||||
#include <string>
|
||||
#include "utils/hash_map.h"
|
||||
#include "transform/graph_ir/op_declare/op_declare_macro.h"
|
||||
#include "ops/nn_batch_norm_ops.h"
|
||||
|
||||
namespace mindspore::transform {
|
||||
DECLARE_OP_ADAPTER(BatchNorm)//将BatchNorm收录后与标准进行比较，进行空间调整，用input_map_为key储存
|
||||
DECLARE_OP_USE_OUTPUT(BatchNorm)//将BatchNorm收录后与标准进行比较，进行空间调整，用output_map_为key储存
|
||||
|
||||
DECLARE_OP_ADAPTER(BNInference)
|
||||
DECLARE_OP_USE_OUTPUT(BNInference)
|
||||
|
||||
DECLARE_OP_ADAPTER(BatchNormGrad)
|
||||
DECLARE_OP_USE_OUTPUT(BatchNormGrad)
|
||||
|
||||
DECLARE_OP_ADAPTER(L2Normalize)
|
||||
DECLARE_OP_USE_OUTPUT(L2Normalize)
|
||||
|
||||
DECLARE_OP_ADAPTER(L2NormalizeGrad)
|
||||
DECLARE_OP_USE_OUTPUT(L2NormalizeGrad)
|
||||
} // namespace mindspore::transform
|
||||
#endif // MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_NN_BATCH_NORM_OPS_DECLARE_H_
|
||||
|
|
@ -0,0 +1,239 @@
|
|||
/**
|
||||
* Copyright 2019 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#include "transform/graph_ir/op_declare/nn_calculation_ops_declare.h"
|
||||
#include <vector>
|
||||
|
||||
namespace mindspore::transform {
|
||||
// BiasAddGrad
|
||||
INPUT_MAP(BiasAddGrad) = {{1, INPUT_DESC(x)}};
|
||||
ATTR_MAP(BiasAddGrad) = {{"format", ATTR_DESC(data_format, AnyTraits<std::string>())}};
|
||||
OUTPUT_MAP(BiasAddGrad) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(BiasAddGrad, prim::kPrimBiasAddGrad->name(), ADPT_DESC(BiasAddGrad))
|
||||
|
||||
// Conv2D
|
||||
INPUT_MAP(Conv2D) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(filter)}, {3, INPUT_DESC(bias)}};
|
||||
ATTR_MAP(Conv2D) = {
|
||||
{"stride", ATTR_DESC(strides, AnyTraits<std::vector<int64_t>>(), AnyTraits<std::vector<int64_t>>())},
|
||||
{"pad_list", ATTR_DESC(pads, AnyTraits<std::vector<int64_t>>(), AnyTraits<std::vector<int64_t>>())},
|
||||
{"dilation", ATTR_DESC(dilations, AnyTraits<std::vector<int64_t>>(), AnyTraits<std::vector<int64_t>>())},
|
||||
{"format", ATTR_DESC(data_format, AnyTraits<std::string>())},
|
||||
{"group", ATTR_DESC(groups, AnyTraits<int64_t>())},
|
||||
};
|
||||
OUTPUT_MAP(Conv2D) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(Conv2D, prim::kPrimConv2D->name(), ADPT_DESC(Conv2D))
|
||||
|
||||
// Conv2DBackpropInputD
|
||||
//这一部分用到的宏定义比较全，我选择注释这一部分来概括一下
|
||||
INPUT_MAP(Conv2DBackpropInputD) = {{1, INPUT_DESC(out_backprop)}, {2, INPUT_DESC(filter)}};/*
|
||||
将name变量out_backprop和filter处理并存入对应InputDesc结构体的相应变量中
|
||||
将name变量内容转为字符串变量并存储至结构体的name变量中
|
||||
引用Operator空间并将指针所指的类转为InputDesc结构体
|
||||
将收录内容的类型与标准进行对比，后进行空间调整，并用指针input_map_为key存储相应内容
|
||||
*/
|
||||
INPUT_ATTR_MAP(Conv2DBackpropInputD) = {
|
||||
{3, ATTR_DESC(input_size, AnyTraits<std::vector<int64_t>>(), AnyTraits<std::vector<int64_t>>())}};/*
|
||||
将name变量input_size处理并存入对应AttrDesc结构体的相应变量中
|
||||
将name变量内容转为字符串变量并存储至结构体的name变量中
|
||||
引用Operator空间并将指针所指的类转为AttrDesc结构体
|
||||
|
||||
//其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
//存入MaxPool对应空间并用attr_map_指针保存
|
||||
|
||||
将收录内容的类型与标准进行对比，后进行空间调整，并用指针input_attr_map_为key存储相应内容
|
||||
*/
|
||||
ATTR_MAP(Conv2DBackpropInputD) = {
|
||||
{"pad_list", ATTR_DESC(pads, AnyTraits<std::vector<int64_t>>(), AnyTraits<std::vector<int64_t>>())},/*
|
||||
将name变量pads处理并存入对应AttrDesc结构体的相应变量中
|
||||
将name变量内容转为字符串变量并存储至结构体的name变量中
|
||||
引用Operator空间并将指针所指的类转为AttrDesc结构体
|
||||
//std:vector<>的作用为构建<>内类型的容量可变的数组
|
||||
//其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
//存入MaxPool对应空间并用attr_map_指针保存
|
||||
|
||||
将收录内容的类型与标准进行对比，后进行空间调整，并用指针attr_map_为key存储相应内容
|
||||
*/
|
||||
{"stride", ATTR_DESC(strides, AnyTraits<std::vector<int64_t>>())},
|
||||
{"dilation", ATTR_DESC(dilations, AnyTraits<std::vector<int64_t>>(), AnyTraits<std::vector<int64_t>>())},
|
||||
{"format", ATTR_DESC(data_format, AnyTraits<std::string>())},
|
||||
{"group", ATTR_DESC(groups, AnyTraits<int64_t>())},
|
||||
};
|
||||
OUTPUT_MAP(Conv2DBackpropInputD) = {{0, OUTPUT_DESC(y)}};/*
|
||||
将name变量y处理并存入对应OutputDesc结构体的相应变量中
|
||||
将name变量内容转为字符串变量并存储至结构体的name变量中
|
||||
引用Operator空间并将指针所指的类转为OutputDesc结构体
|
||||
|
||||
将收录内容的类型与标准进行对比，后进行空间调整，并用指针output_map_为key存储相应内容
|
||||
*/
|
||||
REG_ADPT_DESC(Conv2DBackpropInputD, prim::kPrimConv2DBackpropInput->name(), ADPT_DESC(Conv2DBackpropInputD))/*
|
||||
将Conv2DBackpropInputD处理并存入对应ADPT_DESC结构体的相应变量中
|
||||
将Conv2DBackpropInputD内容转为字符串变量并存储至结构体的name变量中
|
||||
引用Operator空间并将指针所指的类转为ADPT_DESC结构体
|
||||
|
||||
再将Conv2DBackpropInputD处理并存入对应REG_ADPT_DESC结构体的相应变量中
|
||||
将Conv2DBackpropInputD内容转为字符串变量并存储至REG_ADPT_DESC的结构体的name变量中
|
||||
引用Operator空间并将指针所指的类转为REG_ADPT_DESC结构体
|
||||
*/
|
||||
|
||||
// Conv2DBackpropInput for tf inference
|
||||
INPUT_MAP(Conv2DBackpropInput) = {{1, INPUT_DESC(input_size)}, {2, INPUT_DESC(filter)}, {3, INPUT_DESC(out_backprop)}};
|
||||
ATTR_MAP(Conv2DBackpropInput) = {
|
||||
{"stride", ATTR_DESC(strides, AnyTraits<std::vector<int64_t>>())},
|
||||
{"dilation", ATTR_DESC(dilations, AnyTraits<std::vector<int64_t>>(), AnyTraits<std::vector<int64_t>>())},
|
||||
{"pad_list", ATTR_DESC(pads, AnyTraits<std::vector<int64_t>>(), AnyTraits<std::vector<int64_t>>())},
|
||||
{"data_format", ATTR_DESC(data_format, AnyTraits<std::string>())},
|
||||
};
|
||||
OUTPUT_MAP(Conv2DBackpropInput) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(Conv2DBackpropInput, kNameConv2DBackpropInputV2, ADPT_DESC(Conv2DBackpropInput))
|
||||
|
||||
// Deconvolution for caffe inference
|
||||
INPUT_MAP(Deconvolution) = {
|
||||
{1, INPUT_DESC(x)}, {2, INPUT_DESC(filter)}, {3, INPUT_DESC(bias)}, {4, INPUT_DESC(offset_w)}};
|
||||
ATTR_MAP(Deconvolution) = {
|
||||
{"stride", ATTR_DESC(strides, AnyTraits<std::vector<int64_t>>(), AnyTraits<std::vector<int64_t>>())},
|
||||
{"pad_list", ATTR_DESC(pads, AnyTraits<std::vector<int64_t>>(), AnyTraits<std::vector<int64_t>>())},
|
||||
{"dilation", ATTR_DESC(dilations, AnyTraits<std::vector<int64_t>>(), AnyTraits<std::vector<int64_t>>())},
|
||||
{"group", ATTR_DESC(groups, AnyTraits<int64_t>())},
|
||||
{"format", ATTR_DESC(data_format, AnyTraits<string>())},
|
||||
{"offset", ATTR_DESC(offset_x, AnyTraits<int64_t>())}};
|
||||
OUTPUT_MAP(Deconvolution) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(Deconvolution, kNameDeconvolution, ADPT_DESC(Deconvolution))
|
||||
REG_ADPT_DESC(Conv2DTranspose, kConv2DTransposeOpName, ADPT_DESC(Conv2DBackpropInputD))
|
||||
|
||||
// Conv2DTransposeD for tf onnx inference
|
||||
INPUT_MAP(Conv2DTransposeD) = {
|
||||
{1, INPUT_DESC(x)}, {2, INPUT_DESC(filter)}, {3, INPUT_DESC(bias)}, {4, INPUT_DESC(offset_w)}};
|
||||
ATTR_MAP(Conv2DTransposeD) = {
|
||||
{"input_size", ATTR_DESC(input_size, AnyTraits<std::vector<int64_t>>(), AnyTraits<std::vector<int64_t>>())},
|
||||
{"stride", ATTR_DESC(strides, AnyTraits<std::vector<int64_t>>(), AnyTraits<std::vector<int64_t>>())},
|
||||
{"pad_list", ATTR_DESC(pads, AnyTraits<std::vector<int64_t>>(), AnyTraits<std::vector<int64_t>>())},
|
||||
{"dilation", ATTR_DESC(dilations, AnyTraits<std::vector<int64_t>>(), AnyTraits<std::vector<int64_t>>())},
|
||||
{"group", ATTR_DESC(groups, AnyTraits<int64_t>())},
|
||||
{"data_format", ATTR_DESC(data_format, AnyTraits<string>())},
|
||||
{"output_paddings", ATTR_DESC(output_padding, AnyTraits<std::vector<int64_t>>(), AnyTraits<std::vector<int64_t>>())},
|
||||
{"offset", ATTR_DESC(offset_x, AnyTraits<int64_t>())}};
|
||||
OUTPUT_MAP(Conv2DTransposeD) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(Conv2DTransposeD, kNameConv2DTransposeD, ADPT_DESC(Conv2DTransposeD))
|
||||
|
||||
// Conv2DBackpropFilterD
|
||||
INPUT_MAP(Conv2DBackpropFilterD) = {{1, INPUT_DESC(out_backprop)}, {2, INPUT_DESC(x)}};
|
||||
INPUT_ATTR_MAP(Conv2DBackpropFilterD) = {
|
||||
{3, ATTR_DESC(filter_size, AnyTraits<std::vector<int64_t>>(), AnyTraits<std::vector<int64_t>>())}};
|
||||
ATTR_MAP(Conv2DBackpropFilterD) = {
|
||||
{"pad_list", ATTR_DESC(pads, AnyTraits<std::vector<int64_t>>(), AnyTraits<std::vector<int64_t>>())},
|
||||
{"stride", ATTR_DESC(strides, AnyTraits<std::vector<int64_t>>(), AnyTraits<std::vector<int64_t>>())},
|
||||
{"dilation", ATTR_DESC(dilations, AnyTraits<std::vector<int64_t>>(), AnyTraits<std::vector<int64_t>>())},
|
||||
{"format", ATTR_DESC(data_format, AnyTraits<std::string>())},
|
||||
{"group", ATTR_DESC(groups, AnyTraits<int64_t>())},
|
||||
};
|
||||
OUTPUT_MAP(Conv2DBackpropFilterD) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(Conv2DBackpropFilterD, prim::kPrimConv2DBackpropFilter->name(), ADPT_DESC(Conv2DBackpropFilterD))
|
||||
|
||||
// Conv3DTransposeD
|
||||
INPUT_MAP(Conv3DTransposeD) = {
|
||||
{1, INPUT_DESC(x)}, {2, INPUT_DESC(filter)}, {3, INPUT_DESC(bias)}, {4, INPUT_DESC(offset_w)}};
|
||||
ATTR_MAP(Conv3DTransposeD) = {
|
||||
{"input_size", ATTR_DESC(input_size, AnyTraits<std::vector<int64_t>>(), AnyTraits<std::vector<int64_t>>())},
|
||||
{"strides", ATTR_DESC(strides, AnyTraits<std::vector<int64_t>>(), AnyTraits<std::vector<int64_t>>())},
|
||||
{"pad_list", ATTR_DESC(pads, AnyTraits<std::vector<int64_t>>(), AnyTraits<std::vector<int64_t>>())},
|
||||
{"dilations", ATTR_DESC(dilations, AnyTraits<std::vector<int64_t>>(), AnyTraits<std::vector<int64_t>>())},
|
||||
{"groups", ATTR_DESC(groups, AnyTraits<int64_t>())},
|
||||
{"format", ATTR_DESC(data_format, AnyTraits<std::string>())},
|
||||
{"output_padding", ATTR_DESC(output_padding, AnyTraits<std::vector<int64_t>>(), AnyTraits<std::vector<int64_t>>())},
|
||||
};
|
||||
OUTPUT_MAP(Conv3DTransposeD) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(Conv3DTransposeD, kNameConv3DTransposeD, ADPT_DESC(Conv3DTransposeD))
|
||||
|
||||
// Conv3D
|
||||
INPUT_MAP(Conv3D) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(filter)}, {3, INPUT_DESC(bias)}, {4, INPUT_DESC(offset_w)}};
|
||||
ATTR_MAP(Conv3D) = {
|
||||
{"strides", ATTR_DESC(strides, AnyTraits<std::vector<int64_t>>(), AnyTraits<std::vector<int64_t>>())},
|
||||
{"pad_list", ATTR_DESC(pads, AnyTraits<std::vector<int64_t>>(), AnyTraits<std::vector<int64_t>>())},
|
||||
{"dilations", ATTR_DESC(dilations, AnyTraits<std::vector<int64_t>>(), AnyTraits<std::vector<int64_t>>())},
|
||||
{"groups", ATTR_DESC(groups, AnyTraits<int64_t>())},
|
||||
{"format", ATTR_DESC(data_format, AnyTraits<std::string>())},
|
||||
{"offset_x", ATTR_DESC(offset_x, AnyTraits<int64_t>())},
|
||||
};
|
||||
OUTPUT_MAP(Conv3D) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(Conv3D, kNameConv3D, ADPT_DESC(Conv3D))
|
||||
|
||||
// Conv3DBackpropInputD
|
||||
INPUT_MAP(Conv3DBackpropInputD) = {{1, INPUT_DESC(out_backprop)}, {2, INPUT_DESC(filter)}};
|
||||
INPUT_ATTR_MAP(Conv3DBackpropInputD) = {
|
||||
{3, ATTR_DESC(input_size, AnyTraits<std::vector<int64_t>>(), AnyTraits<std::vector<int64_t>>())}};
|
||||
ATTR_MAP(Conv3DBackpropInputD) = {
|
||||
{"pad_list", ATTR_DESC(pads, AnyTraits<std::vector<int64_t>>(), AnyTraits<std::vector<int64_t>>())},
|
||||
{"strides", ATTR_DESC(strides, AnyTraits<std::vector<int64_t>>())},
|
||||
{"dilations", ATTR_DESC(dilations, AnyTraits<std::vector<int64_t>>(), AnyTraits<std::vector<int64_t>>())},
|
||||
{"format", ATTR_DESC(data_format, AnyTraits<std::string>())},
|
||||
{"groups", ATTR_DESC(groups, AnyTraits<int64_t>())},
|
||||
};
|
||||
OUTPUT_MAP(Conv3DBackpropInputD) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(Conv3DBackpropInputD, kNameConv3DBackpropInputD, ADPT_DESC(Conv3DBackpropInputD))
|
||||
|
||||
// Conv3DBackpropFilterD
|
||||
INPUT_MAP(Conv3DBackpropFilterD) = {{1, INPUT_DESC(out_backprop)}, {2, INPUT_DESC(x)}};
|
||||
INPUT_ATTR_MAP(Conv3DBackpropFilterD) = {
|
||||
{3, ATTR_DESC(filter_size, AnyTraits<std::vector<int64_t>>(), AnyTraits<std::vector<int64_t>>())}};
|
||||
ATTR_MAP(Conv3DBackpropFilterD) = {
|
||||
{"strides", ATTR_DESC(strides, AnyTraits<std::vector<int64_t>>(), AnyTraits<std::vector<int64_t>>())},
|
||||
{"pad_list", ATTR_DESC(pads, AnyTraits<std::vector<int64_t>>(), AnyTraits<std::vector<int64_t>>())},
|
||||
{"dilations", ATTR_DESC(dilations, AnyTraits<std::vector<int64_t>>(), AnyTraits<std::vector<int64_t>>())},
|
||||
{"groups", ATTR_DESC(groups, AnyTraits<int64_t>())},
|
||||
{"format", ATTR_DESC(data_format, AnyTraits<std::string>())},
|
||||
};
|
||||
OUTPUT_MAP(Conv3DBackpropFilterD) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(Conv3DBackpropFilterD, kNameConv3DBackpropFilterD, ADPT_DESC(Conv3DBackpropFilterD))
|
||||
|
||||
// DepthwiseConv2D
|
||||
INPUT_MAP(DepthwiseConv2D) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(filter)}, {3, INPUT_DESC(bias)}};
|
||||
ATTR_MAP(DepthwiseConv2D) = {
|
||||
{"stride", ATTR_DESC(strides, AnyTraits<std::vector<int64_t>>(), AnyTraits<std::vector<int64_t>>())},
|
||||
{"pad_list", ATTR_DESC(pads, AnyTraits<std::vector<int64_t>>(), AnyTraits<std::vector<int64_t>>())},
|
||||
{"dilation", ATTR_DESC(dilations, AnyTraits<std::vector<int64_t>>(), AnyTraits<std::vector<int64_t>>())},
|
||||
{"format", ATTR_DESC(data_format, AnyTraits<std::string>())},
|
||||
};
|
||||
OUTPUT_MAP(DepthwiseConv2D) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(DepthwiseConv2D, prim::kPrimDepthwiseConv2dNative->name(), ADPT_DESC(DepthwiseConv2D))
|
||||
|
||||
// DepthwiseConv2DBackpropInputD
|
||||
INPUT_MAP(DepthwiseConv2DBackpropInputD) = {{2, INPUT_DESC(filter)}, {3, INPUT_DESC(out_backprop)}};
|
||||
INPUT_ATTR_MAP(DepthwiseConv2DBackpropInputD) = {
|
||||
{1, ATTR_DESC(input_size, AnyTraits<std::vector<int64_t>>(), AnyTraits<std::vector<int64_t>>())}};
|
||||
ATTR_MAP(DepthwiseConv2DBackpropInputD) = {
|
||||
{"stride", ATTR_DESC(strides, AnyTraits<std::vector<int64_t>>(), AnyTraits<std::vector<int64_t>>())},
|
||||
{"pad_list", ATTR_DESC(pads, AnyTraits<std::vector<int64_t>>(), AnyTraits<std::vector<int64_t>>())},
|
||||
{"dilation", ATTR_DESC(dilations, AnyTraits<std::vector<int64_t>>(), AnyTraits<std::vector<int64_t>>())},
|
||||
};
|
||||
OUTPUT_MAP(DepthwiseConv2DBackpropInputD) = {{0, OUTPUT_DESC(input_grad)}};
|
||||
REG_ADPT_DESC(DepthwiseConv2DBackpropInputD, prim::kPrimDepthwiseConv2dNativeBackpropInput->name(),
|
||||
ADPT_DESC(DepthwiseConv2DBackpropInputD))
|
||||
|
||||
// DepthwiseConv2DBackpropFilterD
|
||||
INPUT_MAP(DepthwiseConv2DBackpropFilterD) = {{1, INPUT_DESC(input)}, {3, INPUT_DESC(out_backprop)}};
|
||||
INPUT_ATTR_MAP(DepthwiseConv2DBackpropFilterD) = {
|
||||
{2, ATTR_DESC(filter_size, AnyTraits<std::vector<int64_t>>(), AnyTraits<std::vector<int64_t>>())}};
|
||||
ATTR_MAP(DepthwiseConv2DBackpropFilterD) = {
|
||||
{"stride", ATTR_DESC(strides, AnyTraits<std::vector<int64_t>>(), AnyTraits<std::vector<int64_t>>())},
|
||||
{"pad_list", ATTR_DESC(pads, AnyTraits<std::vector<int64_t>>(), AnyTraits<std::vector<int64_t>>())},
|
||||
{"dilation", ATTR_DESC(dilations, AnyTraits<std::vector<int64_t>>(), AnyTraits<std::vector<int64_t>>())},
|
||||
};
|
||||
OUTPUT_MAP(DepthwiseConv2DBackpropFilterD) = {{0, OUTPUT_DESC(filter_grad)}};
|
||||
REG_ADPT_DESC(DepthwiseConv2DBackpropFilterD, prim::kPrimDepthwiseConv2dNativeBackpropFilter->name(),
|
||||
ADPT_DESC(DepthwiseConv2DBackpropFilterD))
|
||||
} // namespace mindspore::transform
|
||||
|
|
@ -0,0 +1,82 @@
|
|||
/**
|
||||
* Copyright 2019-2021 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_NN_CALCULATION_OPS_DECLARE_H_
|
||||
#define MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_NN_CALCULATION_OPS_DECLARE_H_
|
||||
|
||||
#include <string>
|
||||
#include "utils/hash_map.h"
|
||||
#include "transform/graph_ir/op_declare/op_declare_macro.h"
|
||||
#include "ops/nn_calculation_ops.h"
|
||||
|
||||
namespace mindspore::transform {
|
||||
DECLARE_OP_ADAPTER(BiasAddGrad)//函数功能为将BiasAddGrad的类型与标准进行对比，后进行空间调整，并用指针input_map_为key存储相应内容
|
||||
DECLARE_OP_USE_OUTPUT(BiasAddGrad)//函数功能为将BiasAddGrad的类型与标准进行对比，后进行空间调整，并用指针output_map_为key存储相应内容
|
||||
|
||||
DECLARE_OP_ADAPTER(Conv2D)
|
||||
DECLARE_OP_USE_ENUM(Conv2D)//函数功能为将Conv2D的类型与标准进行对比，后进行空间调整，并用指针enum_map_为key存储相应内容
|
||||
DECLARE_OP_USE_OUTPUT(Conv2D)
|
||||
|
||||
DECLARE_OP_ADAPTER(Conv2DBackpropInputD)
|
||||
DECLARE_OP_USE_ENUM(Conv2DBackpropInputD)
|
||||
DECLARE_OP_USE_INPUT_ATTR(Conv2DBackpropInputD)////函数功能为将Conv2DBackpropInputD的类型与标准进行对比，后进行空间调整，并用指针input_attr_map_为key存储相应内容
|
||||
DECLARE_OP_USE_OUTPUT(Conv2DBackpropInputD)
|
||||
|
||||
DECLARE_OP_ADAPTER(Conv2DBackpropInput)
|
||||
DECLARE_OP_USE_OUTPUT(Conv2DBackpropInput)
|
||||
|
||||
DECLARE_OP_ADAPTER(Conv2DBackpropFilterD)
|
||||
DECLARE_OP_USE_ENUM(Conv2DBackpropFilterD)
|
||||
DECLARE_OP_USE_INPUT_ATTR(Conv2DBackpropFilterD)
|
||||
DECLARE_OP_USE_OUTPUT(Conv2DBackpropFilterD)
|
||||
|
||||
DECLARE_OP_ADAPTER(Conv3DTransposeD)
|
||||
DECLARE_OP_USE_ENUM(Conv3DTransposeD)
|
||||
DECLARE_OP_USE_OUTPUT(Conv3DTransposeD)
|
||||
|
||||
DECLARE_OP_ADAPTER(Conv3D)
|
||||
DECLARE_OP_USE_ENUM(Conv3D)
|
||||
DECLARE_OP_USE_OUTPUT(Conv3D)
|
||||
|
||||
DECLARE_OP_ADAPTER(Conv3DBackpropInputD)
|
||||
DECLARE_OP_USE_ENUM(Conv3DBackpropInputD)
|
||||
DECLARE_OP_USE_INPUT_ATTR(Conv3DBackpropInputD)
|
||||
DECLARE_OP_USE_OUTPUT(Conv3DBackpropInputD)
|
||||
|
||||
DECLARE_OP_ADAPTER(Conv3DBackpropFilterD)
|
||||
DECLARE_OP_USE_ENUM(Conv3DBackpropFilterD)
|
||||
DECLARE_OP_USE_INPUT_ATTR(Conv3DBackpropFilterD)
|
||||
DECLARE_OP_USE_OUTPUT(Conv3DBackpropFilterD)
|
||||
|
||||
DECLARE_OP_ADAPTER(DepthwiseConv2D)
|
||||
DECLARE_OP_USE_ENUM(DepthwiseConv2D)
|
||||
DECLARE_OP_USE_OUTPUT(DepthwiseConv2D)
|
||||
|
||||
DECLARE_OP_ADAPTER(DepthwiseConv2DBackpropFilterD)
|
||||
DECLARE_OP_USE_INPUT_ATTR(DepthwiseConv2DBackpropFilterD)
|
||||
DECLARE_OP_USE_OUTPUT(DepthwiseConv2DBackpropFilterD)
|
||||
|
||||
DECLARE_OP_ADAPTER(DepthwiseConv2DBackpropInputD)
|
||||
DECLARE_OP_USE_INPUT_ATTR(DepthwiseConv2DBackpropInputD)
|
||||
DECLARE_OP_USE_OUTPUT(DepthwiseConv2DBackpropInputD)
|
||||
|
||||
DECLARE_OP_ADAPTER(Deconvolution)
|
||||
DECLARE_OP_USE_OUTPUT(Deconvolution)
|
||||
|
||||
DECLARE_OP_ADAPTER(Conv2DTransposeD)
|
||||
DECLARE_OP_USE_OUTPUT(Conv2DTransposeD)
|
||||
} // namespace mindspore::transform
|
||||
#endif // MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_NN_CALCULATION_OPS_DECLARE_H_
|
||||
|
|
@ -0,0 +1,100 @@
|
|||
/**
|
||||
* Copyright 2019-2021 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#include "transform/graph_ir/op_declare/nn_detect_ops_declare.h"
|
||||
#include <vector>
|
||||
|
||||
namespace mindspore::transform {
|
||||
// BoundingBoxEncode
|
||||
INPUT_MAP(BoundingBoxEncode) = {
|
||||
{1, INPUT_DESC(anchor_box)},
|
||||
{2, INPUT_DESC(ground_truth_box)},
|
||||
};//将BoundingBoxEncode类型与标准进行对比，后进行空间调整，并用指针input_map_为key存储相应内容
|
||||
ATTR_MAP(BoundingBoxEncode) = {
|
||||
{"means", ATTR_DESC(means, AnyTraits<std::vector<float>>(), AnyTraits<float>())},
|
||||
{"stds", ATTR_DESC(stds, AnyTraits<std::vector<float>>(), AnyTraits<float>())},
|
||||
};/*
|
||||
将means处理并存入对应 ATTR_DESC结构体的相应变量中
|
||||
将means内容转为字符串变量并存储至结构体的name变量中
|
||||
引用Operator空间并将指针所指的类转为 ATTR_DESC结构体
|
||||
//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
//存入MaxPool对应空间并用attr_map_指针保存
|
||||
//其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
//std:vector<>的作用为构建<>内类型的容量可变的数组
|
||||
*/
|
||||
//将BoundingBoxEncode类型与标准进行对比，后进行空间调整，并用指针attr_map_为key存储相应内容
|
||||
OUTPUT_MAP(BoundingBoxEncode) = {{0, OUTPUT_DESC(delats)}};//将delats处理并存入对应OUTPUT_DESC结构体的相应变量中
|
||||
//将delats内容转为字符串变量并存储至结构体的name变量中
|
||||
//引用Operator空间并将指针所指的类转为OUTPUT_DESC结构体
|
||||
//将BoundingBoxEncode类型与标准进行对比，后进行空间调整，并用指针output_map_为key存储相应内容
|
||||
REG_ADPT_DESC(BoundingBoxEncode, kNameBoundingBoxEncode, ADPT_DESC(BoundingBoxEncode))//将BoudingBoxEncode处理并存入对应RED_ADPT_DESC结构体的相应变量中
|
||||
//将BoudingBoxEncode内容转为字符串变量并存储至结构体的name变量中
|
||||
//引用Operator空间并将指针所指的类转为RED_ADPT_DESC结构体
|
||||
|
||||
|
||||
// BoundingBoxDecode
|
||||
INPUT_MAP(BoundingBoxDecode) = {
|
||||
{1, INPUT_DESC(rois)},
|
||||
{2, INPUT_DESC(deltas)},
|
||||
};
|
||||
ATTR_MAP(BoundingBoxDecode) = {
|
||||
{"means", ATTR_DESC(means, AnyTraits<std::vector<float>>(), AnyTraits<float>())},
|
||||
{"stds", ATTR_DESC(stds, AnyTraits<std::vector<float>>(), AnyTraits<float>())},
|
||||
{"max_shape", ATTR_DESC(max_shape, AnyTraits<std::vector<int64_t>>(), AnyTraits<std::vector<int64_t>>())},
|
||||
{"wh_ratio_clip", ATTR_DESC(wh_ratio_clip, AnyTraits<float>())},
|
||||
};
|
||||
OUTPUT_MAP(BoundingBoxDecode) = {{0, OUTPUT_DESC(bboxes)}};
|
||||
REG_ADPT_DESC(BoundingBoxDecode, kNameBoundingBoxDecode, ADPT_DESC(BoundingBoxDecode))
|
||||
|
||||
// Iou
|
||||
INPUT_MAP(Iou) = {{1, INPUT_DESC(bboxes)}, {2, INPUT_DESC(gtboxes)}};
|
||||
ATTR_MAP(Iou) = {{"mode", ATTR_DESC(mode, AnyTraits<std::string>())}};
|
||||
OUTPUT_MAP(Iou) = {{0, OUTPUT_DESC(overlap)}};
|
||||
REG_ADPT_DESC(Iou, kNameIOU, ADPT_DESC(Iou))
|
||||
|
||||
// CheckValid
|
||||
INPUT_MAP(CheckValid) = {{1, INPUT_DESC(bbox_tensor)}, {2, INPUT_DESC(img_metas)}};
|
||||
ATTR_MAP(CheckValid) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(CheckValid) = {{0, OUTPUT_DESC(valid_tensor)}};
|
||||
REG_ADPT_DESC(CheckValid, kNameCheckValid, ADPT_DESC(CheckValid))
|
||||
|
||||
// Sort
|
||||
INPUT_MAP(Sort) = {{1, INPUT_DESC(x)}};
|
||||
ATTR_MAP(Sort) = {{"axis", ATTR_DESC(axis, AnyTraits<int64_t>())},
|
||||
{"descending", ATTR_DESC(descending, AnyTraits<bool>())}};
|
||||
OUTPUT_MAP(Sort) = {{0, OUTPUT_DESC(y1)}, {1, OUTPUT_DESC(y2)}};
|
||||
REG_ADPT_DESC(Sort, kNameSort, ADPT_DESC(Sort))
|
||||
|
||||
// ROIAlign
|
||||
INPUT_MAP(ROIAlign) = {{1, INPUT_DESC(features)}, {2, INPUT_DESC(rois)}};
|
||||
OUTPUT_MAP(ROIAlign) = {{0, OUTPUT_DESC(y)}};
|
||||
ATTR_MAP(ROIAlign) = {{"pooled_height", ATTR_DESC(pooled_height, AnyTraits<int64_t>())},
|
||||
{"pooled_width", ATTR_DESC(pooled_width, AnyTraits<int64_t>())},
|
||||
{"spatial_scale", ATTR_DESC(spatial_scale, AnyTraits<float>())},
|
||||
{"sample_num", ATTR_DESC(sample_num, AnyTraits<int64_t>())},
|
||||
{"roi_end_mode", ATTR_DESC(roi_end_mode, AnyTraits<int64_t>())}};
|
||||
REG_ADPT_DESC(ROIAlign, kNameROIAlign, ADPT_DESC(ROIAlign))
|
||||
// ROIAlignGrad
|
||||
INPUT_MAP(ROIAlignGrad) = {{1, INPUT_DESC(ydiff)}, {2, INPUT_DESC(rois)}};
|
||||
OUTPUT_MAP(ROIAlignGrad) = {{0, OUTPUT_DESC(xdiff)}};
|
||||
ATTR_MAP(ROIAlignGrad) = {
|
||||
{"xdiff_shape", ATTR_DESC(xdiff_shape, AnyTraits<std::vector<int64_t>>(), AnyTraits<std::vector<int64_t>>())},
|
||||
{"pooled_height", ATTR_DESC(pooled_height, AnyTraits<int64_t>())},
|
||||
{"pooled_width", ATTR_DESC(pooled_width, AnyTraits<int64_t>())},
|
||||
{"spatial_scale", ATTR_DESC(spatial_scale, AnyTraits<float>())},
|
||||
{"sample_num", ATTR_DESC(sample_num, AnyTraits<int64_t>())}};
|
||||
REG_ADPT_DESC(ROIAlignGrad, kNameROIAlignGrad, ADPT_DESC(ROIAlignGrad))
|
||||
} // namespace mindspore::transform
|
||||
|
|
@ -0,0 +1,47 @@
|
|||
/**
|
||||
* Copyright 2019-2021 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_NN_DETECT_OPS_DECLARE_H_
|
||||
#define MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_NN_DETECT_OPS_DECLARE_H_
|
||||
|
||||
#include <string>
|
||||
#include "utils/hash_map.h"
|
||||
#include "transform/graph_ir/op_declare/op_declare_macro.h"
|
||||
#include "ops/nn_detect_ops.h"
|
||||
|
||||
namespace mindspore::transform {
|
||||
DECLARE_OP_ADAPTER(Iou)//将收录内容的类型与标准进行对比，后进行空间调整，并用指针input_map_为key存储相应内容
|
||||
DECLARE_OP_USE_OUTPUT(Iou)//将收录内容的类型与标准进行对比，后进行空间调整，并用指针output_map_为key存储相应内容
|
||||
//以下部分根据变量的不同对不同变量进行操作
|
||||
DECLARE_OP_ADAPTER(CheckValid)
|
||||
DECLARE_OP_USE_OUTPUT(CheckValid)
|
||||
|
||||
DECLARE_OP_ADAPTER(Sort)
|
||||
DECLARE_OP_USE_OUTPUT(Sort)
|
||||
|
||||
DECLARE_OP_ADAPTER(BoundingBoxEncode)
|
||||
DECLARE_OP_USE_OUTPUT(BoundingBoxEncode)
|
||||
|
||||
DECLARE_OP_ADAPTER(BoundingBoxDecode)
|
||||
DECLARE_OP_USE_OUTPUT(BoundingBoxDecode)
|
||||
|
||||
DECLARE_OP_ADAPTER(ROIAlign)
|
||||
DECLARE_OP_USE_OUTPUT(ROIAlign)
|
||||
|
||||
DECLARE_OP_ADAPTER(ROIAlignGrad)
|
||||
DECLARE_OP_USE_OUTPUT(ROIAlignGrad)
|
||||
} // namespace mindspore::transform
|
||||
#endif // MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_NN_DETECT_OPS_DECLARE_H_
|
||||
|
|
@ -0,0 +1,163 @@
|
|||
/**
|
||||
* Copyright 2019-2021 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#include "transform/graph_ir/op_declare/nn_norm_ops_declare.h"
|
||||
#include <vector>
|
||||
|
||||
namespace mindspore::transform {
|
||||
// SoftmaxV2
|
||||
INPUT_MAP(SoftmaxV2) = {{1, INPUT_DESC(x)}};//将SoftmaxV2与标准进行比较，以input_map_为key并构造名为x的INPUT_DESC结构体并与标准比较其容量
|
||||
ATTR_MAP(SoftmaxV2) = {
|
||||
{"axis", ATTR_DESC(axes, AnyTraits<std::vector<int64_t>>(), AnyTraits<std::vector<int64_t>>())},
|
||||
};//将SoftmaxV2与标准进行比较，以attr_map_为key并构造储存了name变量axes的ATTR_DESC结构体
|
||||
//其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
//std:vector<>的作用为构建<>内类型的容量可变的数组
|
||||
OUTPUT_MAP(SoftmaxV2) = {{0, OUTPUT_DESC(y)}};//将SoftmaxV2与标准进行比较，以output_map_为key，并构造储存了name变量y的OUTPUT_DESC结构体并与标准比较其容量
|
||||
REG_ADPT_DESC(SoftmaxV2, kNameSoftmax, ADPT_DESC(SoftmaxV2))//在ADPT_DESC结构体的基础上创建名为kNameSoftmax的RED_ADPT_DESC结构体
|
||||
//将SoftmaxV2转为字符变量存入结构体的name变量中
|
||||
//并与标准比较其容量，返回对应内容
|
||||
//以下部分根据引用的不同变量对其进行相应操作
|
||||
// SoftmaxGrad
|
||||
INPUT_MAP(SoftmaxGrad) = {{1, INPUT_DESC(softmax)}, {2, INPUT_DESC(grad_softmax)}};
|
||||
OUTPUT_MAP(SoftmaxGrad) = {{0, OUTPUT_DESC(grad_x)}};
|
||||
ATTR_MAP(SoftmaxGrad) = EMPTY_ATTR_MAP;
|
||||
REG_ADPT_DESC(SoftmaxGrad, kNameSoftmaxGrad, ADPT_DESC(SoftmaxGrad))
|
||||
|
||||
// SoftmaxCrossEntropyWithLogits
|
||||
INPUT_MAP(SoftmaxCrossEntropyWithLogits) = {{1, INPUT_DESC(features)}, {2, INPUT_DESC(labels)}};
|
||||
ATTR_MAP(SoftmaxCrossEntropyWithLogits) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(SoftmaxCrossEntropyWithLogits) = {{0, OUTPUT_DESC(loss)}, {1, OUTPUT_DESC(backprop)}};
|
||||
REG_ADPT_DESC(SoftmaxCrossEntropyWithLogits, prim::kPrimSoftmaxCrossEntropyWithLogits->name(),
|
||||
ADPT_DESC(SoftmaxCrossEntropyWithLogits))
|
||||
|
||||
// SmoothL1Loss
|
||||
INPUT_MAP(SmoothL1Loss) = {{1, INPUT_DESC(predict)}, {2, INPUT_DESC(label)}};
|
||||
ATTR_MAP(SmoothL1Loss) = {{"beta", ATTR_DESC(sigma, AnyTraits<float>())}};
|
||||
OUTPUT_MAP(SmoothL1Loss) = {{0, OUTPUT_DESC(loss)}};
|
||||
REG_ADPT_DESC(SmoothL1Loss, kNameSmoothL1Loss, ADPT_DESC(SmoothL1Loss))
|
||||
|
||||
// SmoothL1LossGrad
|
||||
INPUT_MAP(SmoothL1LossGrad) = {{1, INPUT_DESC(predict)}, {2, INPUT_DESC(label)}, {3, INPUT_DESC(dout)}};
|
||||
ATTR_MAP(SmoothL1LossGrad) = {{"beta", ATTR_DESC(sigma, AnyTraits<float>())}};
|
||||
OUTPUT_MAP(SmoothL1LossGrad) = {{0, OUTPUT_DESC(gradient)}};
|
||||
REG_ADPT_DESC(SmoothL1LossGrad, kNameSmoothL1LossGrad, ADPT_DESC(SmoothL1LossGrad))
|
||||
|
||||
// SigmoidCrossEntropyWithLogits
|
||||
INPUT_MAP(SigmoidCrossEntropyWithLogits) = {{1, INPUT_DESC(predict)}, {2, INPUT_DESC(target)}};
|
||||
ATTR_MAP(SigmoidCrossEntropyWithLogits) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(SigmoidCrossEntropyWithLogits) = {{0, OUTPUT_DESC(loss)}};
|
||||
REG_ADPT_DESC(SigmoidCrossEntropyWithLogits, kNameSigmoidCrossEntropyWithLogits,
|
||||
ADPT_DESC(SigmoidCrossEntropyWithLogits))
|
||||
|
||||
// SigmoidCrossEntropyWithLogitsGrad
|
||||
INPUT_MAP(SigmoidCrossEntropyWithLogitsGrad) = {
|
||||
{1, INPUT_DESC(predict)}, {2, INPUT_DESC(target)}, {3, INPUT_DESC(dout)}};
|
||||
ATTR_MAP(SigmoidCrossEntropyWithLogitsGrad) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(SigmoidCrossEntropyWithLogitsGrad) = {{0, OUTPUT_DESC(gradient)}};
|
||||
REG_ADPT_DESC(SigmoidCrossEntropyWithLogitsGrad, kNameSigmoidCrossEntropyWithLogitsGrad,
|
||||
ADPT_DESC(SigmoidCrossEntropyWithLogitsGrad))
|
||||
|
||||
// SigmoidCrossEntropyWithLogitsV2
|
||||
INPUT_MAP(SigmoidCrossEntropyWithLogitsV2) = {
|
||||
{1, INPUT_DESC(predict)}, {2, INPUT_DESC(target)}, {3, INPUT_DESC(weight)}, {4, INPUT_DESC(pos_weight)}};
|
||||
ATTR_MAP(SigmoidCrossEntropyWithLogitsV2) = {{"reduction", ATTR_DESC(reduction, AnyTraits<std::string>())}};
|
||||
OUTPUT_MAP(SigmoidCrossEntropyWithLogitsV2) = {{0, OUTPUT_DESC(loss)}};
|
||||
REG_ADPT_DESC(SigmoidCrossEntropyWithLogitsV2, kNameSigmoidCrossEntropyWithLogitsV2,
|
||||
ADPT_DESC(SigmoidCrossEntropyWithLogitsV2))
|
||||
|
||||
// LogSoftmaxGrad
|
||||
INPUT_MAP(LogSoftmaxGrad) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(grad)}};
|
||||
ATTR_MAP(LogSoftmaxGrad) = {
|
||||
{"axis", ATTR_DESC(axis, AnyTraits<std::vector<int64_t>>(), AnyTraits<std::vector<int64_t>>())}};
|
||||
OUTPUT_MAP(LogSoftmaxGrad) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(LogSoftmaxGrad, prim::kPrimLogSoftmaxGrad->name(), ADPT_DESC(LogSoftmaxGrad))
|
||||
|
||||
// LogSoftmaxV2
|
||||
INPUT_MAP(LogSoftmaxV2) = {{1, INPUT_DESC(logits)}};
|
||||
ATTR_MAP(LogSoftmaxV2) = {
|
||||
{"axis", ATTR_DESC(axes, AnyTraits<std::vector<int64_t>>(), AnyTraits<std::vector<int64_t>>())}};
|
||||
OUTPUT_MAP(LogSoftmaxV2) = {{0, OUTPUT_DESC(logsoftmax)}};
|
||||
REG_ADPT_DESC(LogSoftmaxV2, prim::kPrimLogSoftmax->name(), ADPT_DESC(LogSoftmaxV2))
|
||||
|
||||
// LayerNorm
|
||||
INPUT_MAP(LayerNorm) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(gamma)}, {3, INPUT_DESC(beta)}};
|
||||
ATTR_MAP(LayerNorm) = {{"begin_norm_axis", ATTR_DESC(begin_norm_axis, AnyTraits<int64_t>())},
|
||||
{"begin_params_axis", ATTR_DESC(begin_params_axis, AnyTraits<int64_t>())},
|
||||
{"epsilon", ATTR_DESC(epsilon, AnyTraits<float>())}};//将begin_norm_axis变量处理并存入结构体对应的变量中，转为字符串并存至name变量中，
|
||||
//收录begin_+norm_axis变量和结构体，并与标准进行比较，进行空间调整并用attr_map_为key储存相应内容
|
||||
OUTPUT_MAP(LayerNorm) = {{0, OUTPUT_DESC(y)}, {1, OUTPUT_DESC(mean)}, {2, OUTPUT_DESC(variance)}};
|
||||
REG_ADPT_DESC(LayerNorm, prim::kPrimLayerNorm->name(), ADPT_DESC(LayerNorm))
|
||||
|
||||
// LayerNormGrad
|
||||
INPUT_MAP(LayerNormGrad) = {
|
||||
{1, INPUT_DESC(x)}, {2, INPUT_DESC(dy)}, {3, INPUT_DESC(variance)}, {4, INPUT_DESC(mean)}, {5, INPUT_DESC(gamma)}};
|
||||
ATTR_MAP(LayerNormGrad) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(LayerNormGrad) = {{0, OUTPUT_DESC(pd_x)}, {1, OUTPUT_DESC(pd_gamma)}, {2, OUTPUT_DESC(pd_beta)}};
|
||||
REG_ADPT_DESC(LayerNormGrad, prim::kPrimLayerNormGrad->name(), ADPT_DESC(LayerNormGrad))
|
||||
|
||||
// LRN
|
||||
INPUT_MAP(LRN) = {{1, INPUT_DESC(x)}};
|
||||
ATTR_MAP(LRN) = {{"depth_radius", ATTR_DESC(depth_radius, AnyTraits<int64_t>())},
|
||||
{"bias", ATTR_DESC(bias, AnyTraits<float>())},
|
||||
{"alpha", ATTR_DESC(alpha, AnyTraits<float>())},
|
||||
{"beta", ATTR_DESC(beta, AnyTraits<float>())},
|
||||
{"norm_region", ATTR_DESC(norm_region, AnyTraits<string>())}};
|
||||
OUTPUT_MAP(LRN) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(LRN, kNameLRN, ADPT_DESC(LRN))
|
||||
|
||||
// LRNGrad
|
||||
INPUT_MAP(LRNGrad) = {{1, INPUT_DESC(grads)}, {2, INPUT_DESC(x)}, {3, INPUT_DESC(y)}};
|
||||
ATTR_MAP(LRNGrad) = {{"depth_radius", ATTR_DESC(depth_radius, AnyTraits<int64_t>())},
|
||||
{"bias", ATTR_DESC(bias, AnyTraits<float>())},
|
||||
{"alpha", ATTR_DESC(alpha, AnyTraits<float>())},
|
||||
{"beta", ATTR_DESC(beta, AnyTraits<float>())}};
|
||||
OUTPUT_MAP(LRNGrad) = {{0, OUTPUT_DESC(z)}};
|
||||
REG_ADPT_DESC(LRNGrad, kNameLRNGrad, ADPT_DESC(LRNGrad))
|
||||
|
||||
// DropoutDoMask
|
||||
INPUT_MAP(DropOutDoMask) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(mask)}, {3, INPUT_DESC(keep_prob)}};
|
||||
ATTR_MAP(DropOutDoMask) = EMPTY_ATTR_MAP;
|
||||
OUTPUT_MAP(DropOutDoMask) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(DropOutDoMask, kNameDropoutDoMask, ADPT_DESC(DropOutDoMask))
|
||||
|
||||
// BinaryCrossEntropy
|
||||
INPUT_MAP(BinaryCrossEntropy) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(y)}, {3, INPUT_DESC(weight)}};
|
||||
ATTR_MAP(BinaryCrossEntropy) = {{"reduction", ATTR_DESC(reduction, AnyTraits<std::string>())}};
|
||||
OUTPUT_MAP(BinaryCrossEntropy) = {{0, OUTPUT_DESC(output)}};
|
||||
REG_ADPT_DESC(BinaryCrossEntropy, kNameBinaryCrossEntropy, ADPT_DESC(BinaryCrossEntropy))
|
||||
|
||||
// BinaryCrossEntropyGrad
|
||||
INPUT_MAP(BinaryCrossEntropyGrad) = {
|
||||
{1, INPUT_DESC(x)}, {2, INPUT_DESC(y)}, {3, INPUT_DESC(grad_output)}, {4, INPUT_DESC(weight)}};
|
||||
ATTR_MAP(BinaryCrossEntropyGrad) = {{"reduction", ATTR_DESC(reduction, AnyTraits<std::string>())}};
|
||||
OUTPUT_MAP(BinaryCrossEntropyGrad) = {{0, OUTPUT_DESC(output)}};
|
||||
REG_ADPT_DESC(BinaryCrossEntropyGrad, kNameBinaryCrossEntropyGrad, ADPT_DESC(BinaryCrossEntropyGrad))
|
||||
|
||||
// Centralization
|
||||
INPUT_MAP(Centralization) = {{1, INPUT_DESC(x)}};
|
||||
ATTR_MAP(Centralization) = {{"axes", ATTR_DESC(axes, AnyTraits<std::vector<int64_t>>())}};
|
||||
OUTPUT_MAP(Centralization) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(Centralization, kNameCentralization, ADPT_DESC(Centralization))
|
||||
|
||||
// Scale
|
||||
INPUT_MAP(Scale) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(scale)}, {3, INPUT_DESC(bias)}};
|
||||
ATTR_MAP(Scale) = {{"axis", ATTR_DESC(axis, AnyTraits<int64_t>())},
|
||||
{"num_axes", ATTR_DESC(num_axes, AnyTraits<int64_t>())},
|
||||
{"scale_from_blob", ATTR_DESC(scale_from_blob, AnyTraits<bool>())}};
|
||||
|
||||
OUTPUT_MAP(Scale) = {{0, OUTPUT_DESC(y)}};
|
||||
REG_ADPT_DESC(Scale, kNameScale, ADPT_DESC(Scale))
|
||||
} // namespace mindspore::transform
|
||||
|
|
@ -0,0 +1,83 @@
|
|||
/**
|
||||
* Copyright 2019-2021 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_IMAGE_OPS_DECLARE_H_
|
||||
#define MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_IMAGE_OPS_DECLARE_H_
|
||||
|
||||
#include <string>
|
||||
#include "utils/hash_map.h"
|
||||
#include "transform/graph_ir/op_declare/op_declare_macro.h"
|
||||
#include "ops/nn_norm_ops.h"
|
||||
|
||||
namespace mindspore::transform {
|
||||
DECLARE_OP_ADAPTER(SmoothL1Loss)
|
||||
DECLARE_OP_USE_OUTPUT(SmoothL1Loss)
|
||||
|
||||
DECLARE_OP_ADAPTER(SmoothL1LossGrad)
|
||||
DECLARE_OP_USE_OUTPUT(SmoothL1LossGrad)
|
||||
|
||||
DECLARE_OP_ADAPTER(SigmoidCrossEntropyWithLogits)
|
||||
DECLARE_OP_USE_OUTPUT(SigmoidCrossEntropyWithLogits)
|
||||
|
||||
DECLARE_OP_ADAPTER(SigmoidCrossEntropyWithLogitsGrad)
|
||||
DECLARE_OP_USE_OUTPUT(SigmoidCrossEntropyWithLogitsGrad)
|
||||
|
||||
DECLARE_OP_ADAPTER(SigmoidCrossEntropyWithLogitsV2)
|
||||
DECLARE_OP_USE_OUTPUT(SigmoidCrossEntropyWithLogitsV2)
|
||||
|
||||
DECLARE_OP_ADAPTER(LogSoftmaxGrad)
|
||||
DECLARE_OP_USE_OUTPUT(LogSoftmaxGrad)
|
||||
|
||||
DECLARE_OP_ADAPTER(LogSoftmaxV2)
|
||||
DECLARE_OP_USE_OUTPUT(LogSoftmaxV2)
|
||||
|
||||
DECLARE_OP_ADAPTER(LayerNorm)
|
||||
DECLARE_OP_USE_OUTPUT(LayerNorm)
|
||||
|
||||
DECLARE_OP_ADAPTER(LayerNormGrad)
|
||||
DECLARE_OP_USE_OUTPUT(LayerNormGrad)
|
||||
|
||||
DECLARE_OP_ADAPTER(LRN)
|
||||
DECLARE_OP_USE_OUTPUT(LRN)
|
||||
|
||||
DECLARE_OP_ADAPTER(LRNGrad)
|
||||
DECLARE_OP_USE_OUTPUT(LRNGrad)
|
||||
|
||||
DECLARE_OP_ADAPTER(DropOutDoMask)
|
||||
DECLARE_OP_USE_OUTPUT(DropOutDoMask)
|
||||
|
||||
DECLARE_OP_ADAPTER(SoftmaxCrossEntropyWithLogits)
|
||||
DECLARE_OP_USE_OUTPUT(SoftmaxCrossEntropyWithLogits)
|
||||
|
||||
DECLARE_OP_ADAPTER(SoftmaxV2)
|
||||
DECLARE_OP_USE_OUTPUT(SoftmaxV2)
|
||||
|
||||
DECLARE_OP_ADAPTER(SoftmaxGrad)
|
||||
DECLARE_OP_USE_OUTPUT(SoftmaxGrad)
|
||||
|
||||
DECLARE_OP_ADAPTER(BinaryCrossEntropy)
|
||||
DECLARE_OP_USE_OUTPUT(BinaryCrossEntropy)
|
||||
|
||||
DECLARE_OP_ADAPTER(BinaryCrossEntropyGrad)
|
||||
DECLARE_OP_USE_OUTPUT(BinaryCrossEntropyGrad)
|
||||
|
||||
DECLARE_OP_ADAPTER(Centralization)
|
||||
DECLARE_OP_USE_OUTPUT(Centralization)
|
||||
|
||||
DECLARE_OP_ADAPTER(Scale)
|
||||
DECLARE_OP_USE_OUTPUT(Scale)
|
||||
} // namespace mindspore::transform
|
||||
#endif // MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_IMAGE_OPS_DECLARE_H_
|
||||
|
|
@ -0,0 +1,232 @@
|
|||
/**
|
||||
* Copyright 2019-2021 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#include "transform/graph_ir/op_declare/nn_pooling_ops_declare.h"//按照路径寻找以下文件，导入到本文件
|
||||
#include <vector>//提供vector数组构建函数模版等
|
||||
|
||||
namespace mindspore::transform {//创建名为transform的空间，其空间处于空间mindspore下
|
||||
// MaxPool最大池化
|
||||
INPUT_MAP(MaxPool) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入MaxPool对应空间内并用input_map指针保存
|
||||
ATTR_MAP(MaxPool) = {{"kernel_size", ATTR_DESC(ksize, AnyTraits<int64_t>(), AnyTraits<std::vector<int64_t>>())},
|
||||
{"strides", ATTR_DESC(strides, AnyTraits<int64_t>(), AnyTraits<std::vector<int64_t>>())},
|
||||
{"pad_mode", ATTR_DESC(padding, AnyTraits<std::string>())},
|
||||
{"format", ATTR_DESC(data_format, AnyTraits<std::string>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
// 存入MaxPool对应空间并用attr_map_指针保存
|
||||
// 其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
// std:vector<>的作用为构建<>内类型的容量可变的数组
|
||||
OUTPUT_MAP(MaxPool) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入MaxPool对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(MaxPool, kNameMaxPool, ADPT_DESC(MaxPool))//构造指向MaxPool的指针并储存，创建结构体RegAdptDescMaxPool
|
||||
|
||||
// MaxPool3D
|
||||
INPUT_MAP(MaxPool3D) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入MaxPool3D对应空间内并用input_map指针保存
|
||||
ATTR_MAP(MaxPool3D) = {{"kernel_size", ATTR_DESC(ksize, AnyTraits<int64_t>(), AnyTraits<std::vector<int64_t>>())},
|
||||
{"strides", ATTR_DESC(strides, AnyTraits<int64_t>(), AnyTraits<std::vector<int64_t>>())},
|
||||
{"pad_mode", ATTR_DESC(padding, AnyTraits<std::string>())},
|
||||
{"pad_list", ATTR_DESC(pads, AnyTraits<int64_t>(), AnyTraits<std::vector<int64_t>>())},
|
||||
{"dilation", ATTR_DESC(dilation, AnyTraits<int64_t>(), AnyTraits<std::vector<int64_t>>())},
|
||||
{"ceil_mode", ATTR_DESC(ceil_mode, AnyTraits<int64_t>())},
|
||||
{"format", ATTR_DESC(data_format, AnyTraits<std::string>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
// 存入MaxPool3D对应空间并用attr_map_指针保存
|
||||
// 其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
// std:vector<>的作用为构建<>内类型的容量可变的数组
|
||||
OUTPUT_MAP(MaxPool3D) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入MaxPool3D对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(MaxPool3D, kNameMaxPool3D, ADPT_DESC(MaxPool3D))//构造指向MaxPool3D的指针并储存，创建结构体RegAdptDescMaxPool3D
|
||||
|
||||
// MaxPool3DGrad
|
||||
INPUT_MAP(MaxPool3DGrad) = {{1, INPUT_DESC(orig_x)}, {2, INPUT_DESC(orig_y)}, {3, INPUT_DESC(grads)}};
|
||||
//将变量orig_x、orig_y、grads处理并存入对应InputDesc结构体的相应变量中，存入MaxPool3DGrad对应空间内并用input_map指针保存
|
||||
ATTR_MAP(MaxPool3DGrad) = {{"kernel_size", ATTR_DESC(ksize, AnyTraits<int64_t>(), AnyTraits<std::vector<int64_t>>())},
|
||||
{"strides", ATTR_DESC(strides, AnyTraits<int64_t>(), AnyTraits<std::vector<int64_t>>())},
|
||||
{"pad_list", ATTR_DESC(pads, AnyTraits<int64_t>(), AnyTraits<std::vector<int64_t>>())},
|
||||
{"format", ATTR_DESC(data_format, AnyTraits<std::string>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
// 存入MaxPool3DGrad对应空间并用attr_map_指针保存
|
||||
// 其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
// std:vector<>的作用为构建<>内类型的容量可变的数组
|
||||
OUTPUT_MAP(MaxPool3DGrad) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入MaxPool3DGrad对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(MaxPool3DGrad, kNameMaxPool3DGrad, ADPT_DESC(MaxPool3DGrad))//构造指向MaxPool3DGrad的指针并储存，创建结构体RegAdptDescMaxPool3DGrad
|
||||
|
||||
// MaxPool3DGradGrad
|
||||
INPUT_MAP(MaxPool3DGradGrad) = {{1, INPUT_DESC(orig_x)}, {2, INPUT_DESC(orig_y)}, {3, INPUT_DESC(grads)}};
|
||||
//将变量orig_x、orig_y、grads处理并存入对应InputDesc结构体的相应变量中，存入MaxPool3DGradGrad对应空间内并用input_map指针保存
|
||||
ATTR_MAP(MaxPool3DGradGrad) = {
|
||||
{"kernel_size", ATTR_DESC(ksize, AnyTraits<int64_t>(), AnyTraits<std::vector<int64_t>>())},
|
||||
{"strides", ATTR_DESC(strides, AnyTraits<int64_t>(), AnyTraits<std::vector<int64_t>>())},
|
||||
{"pad_list", ATTR_DESC(pads, AnyTraits<int64_t>(), AnyTraits<std::vector<int64_t>>())},
|
||||
{"format", ATTR_DESC(data_format, AnyTraits<std::string>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
// 存入MaxPool3DGradGrad对应空间并用attr_map_指针保存
|
||||
// 其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
// std:vector<>的作用为构建<>内类型的容量可变的数组
|
||||
OUTPUT_MAP(MaxPool3DGradGrad) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入MaxPool3DGradGrad对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(MaxPool3DGradGrad, kNameMaxPool3DGradGrad, ADPT_DESC(MaxPool3DGradGrad))//构造指向MaxPool3DGrad的指针并储存，创建结构体RegAdptDescMaxPool3DGradGrad
|
||||
|
||||
// AvgPool平均池化
|
||||
INPUT_MAP(AvgPool) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入AvgPool对应空间内并用input_map指针保存
|
||||
ATTR_MAP(AvgPool) = {{"kernel_size", ATTR_DESC(ksize, AnyTraits<int64_t>(), AnyTraits<std::vector<int64_t>>())},
|
||||
{"strides", ATTR_DESC(strides, AnyTraits<int64_t>(), AnyTraits<std::vector<int64_t>>())},
|
||||
{"pad_mode", ATTR_DESC(padding, AnyTraits<std::string>())},
|
||||
{"format", ATTR_DESC(data_format, AnyTraits<std::string>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
// 存入AvgPool对应空间并用attr_map_指针保存
|
||||
// 其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
// std:vector<>的作用为构建<>内类型的容量可变的数组
|
||||
OUTPUT_MAP(AvgPool) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入AvgPool对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(AvgPool, kNameAvgPool, ADPT_DESC(AvgPool))//构造指向AvgPool的指针并储存，创建结构体RegAdptDescAvgPool
|
||||
|
||||
// MaxPoolGrad
|
||||
INPUT_MAP(MaxPoolGrad) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}, {3, INPUT_DESC(grad)}};
|
||||
//将变量x1、x2、x3处理并存入对应InputDesc结构体的相应变量中，存入MaxPoolGrad对应空间内并用input_map指针保存
|
||||
ATTR_MAP(MaxPoolGrad) = {{"kernel_size", ATTR_DESC(ksize, AnyTraits<int64_t>(), AnyTraits<std::vector<int64_t>>())},
|
||||
{"strides", ATTR_DESC(strides, AnyTraits<int64_t>(), AnyTraits<std::vector<int64_t>>())},
|
||||
{"pad_mode", ATTR_DESC(padding, AnyTraits<std::string>())},
|
||||
{"format", ATTR_DESC(data_format, AnyTraits<std::string>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
// 存入MaxPoolGrad对应空间并用attr_map_指针保存
|
||||
// 其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
// std:vector<>的作用为构建<>内类型的容量可变的数组
|
||||
OUTPUT_MAP(MaxPoolGrad) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入MaxPoolGrad对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(MaxPoolGrad, kNameMaxPoolGrad, ADPT_DESC(MaxPoolGrad))//构造指向MaxPoolGrad的指针并储存，创建结构体RegAdptDescMaxPoolGrad
|
||||
|
||||
// MaxPoolGradGrad
|
||||
INPUT_MAP(MaxPoolGradGrad) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}, {3, INPUT_DESC(grad)}};
|
||||
//将变量x1、x2、x3处理并存入对应InputDesc结构体的相应变量中，存入MaxPoolGraddGrad对应空间内并用input_map指针保存
|
||||
ATTR_MAP(MaxPoolGradGrad) = {{"kernel_size", ATTR_DESC(ksize, AnyTraits<int64_t>(), AnyTraits<std::vector<int64_t>>())},
|
||||
{"strides", ATTR_DESC(strides, AnyTraits<int64_t>(), AnyTraits<std::vector<int64_t>>())},
|
||||
{"pad_mode", ATTR_DESC(padding, AnyTraits<std::string>())},
|
||||
{"format", ATTR_DESC(data_format, AnyTraits<std::string>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
// 存入MaxPoolGradGrad对应空间并用attr_map_指针保存
|
||||
// 其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
// std:vector<>的作用为构建<>内类型的容量可变的数组
|
||||
OUTPUT_MAP(MaxPoolGradGrad) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入MaxPoolGradGrad对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(MaxPoolGradGrad, kNameMaxPoolGradGrad, ADPT_DESC(MaxPoolGradGrad))//构造指向MaxPoolGradGrad的指针并储存，创建结构体RegAdptDescMaxPoolGradGrad
|
||||
|
||||
// avgpoolgrad
|
||||
INPUT_MAP(AvgPoolGrad) = {{1, INPUT_DESC(orig_input_shape)}, {2, INPUT_DESC(input_grad)}};
|
||||
//将变量orig_input_shape、input_grad处理并存入对应InputDesc结构体的相应变量中，存入avgpoolgrad对应空间内并用input_map指针保存
|
||||
ATTR_MAP(AvgPoolGrad) = {{"kernel_size", ATTR_DESC(ksize, AnyTraits<int64_t>(), AnyTraits<std::vector<int64_t>>())},
|
||||
{"strides", ATTR_DESC(strides, AnyTraits<int64_t>(), AnyTraits<std::vector<int64_t>>())},
|
||||
{"pad_mode", ATTR_DESC(padding, AnyTraits<std::string>())},
|
||||
{"format", ATTR_DESC(data_format, AnyTraits<std::string>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
// 存入AvgPoolGrad对应空间并用attr_map_指针保存
|
||||
// 其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
// std:vector<>的作用为构建<>内类型的容量可变的数组
|
||||
OUTPUT_MAP(AvgPoolGrad) = {{0, OUTPUT_DESC(out_grad)}};//将变量out_grad处理并存入对应OutputDesc结构体的相应变量中，存入AvgPoolGrad对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(AvgPoolGrad, kNameAvgPoolGrad, ADPT_DESC(AvgPoolGrad))//构造指向AvgPoolGrad的指针并储存，创建结构体RegAdptDescAvgPoolGrad
|
||||
|
||||
// MaxPoolWithArgmax
|
||||
INPUT_MAP(MaxPoolWithArgmax) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入MaxPoolWithArgmax对应空间内并用input_map指针保存
|
||||
ATTR_MAP(MaxPoolWithArgmax) = {
|
||||
{"kernel_size", ATTR_DESC(ksize, AnyTraits<int64_t>(), AnyTraits<std::vector<int64_t>>())},
|
||||
{"strides", ATTR_DESC(strides, AnyTraits<int64_t>(), AnyTraits<std::vector<int64_t>>())},
|
||||
{"pad_mode", ATTR_DESC(padding, AnyTraits<std::string>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
// 存入MaxPoolWithArgmax对应空间并用attr_map_指针保存
|
||||
// 其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
// std:vector<>的作用为构建<>内类型的容量可变的数组
|
||||
OUTPUT_MAP(MaxPoolWithArgmax) = {{0, OUTPUT_DESC(y)}, {1, OUTPUT_DESC(argmax)}};
|
||||
//将变量y、argmax处理并存入对应OutputDesc结构体的相应变量中，存入MaxPoolWithArgmax对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(MaxPoolWithArgmax, kNameMaxPoolWithArgmax, ADPT_DESC(MaxPoolWithArgmax))
|
||||
//构造指向MaxPoolWithArgmax的指针并储存，创建结构体RegAdptDescMaxPoolWithArgmax
|
||||
|
||||
// MaxPoolGradWithArgmax
|
||||
INPUT_MAP(MaxPoolGradWithArgmax) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(grad)}, {3, INPUT_DESC(argmax)}};
|
||||
//将变量x、grad、argmax处理并存入对应InputDesc结构体的相应变量中，存入MaxPoolGradWithArgmax对应空间内并用input_map指针保存
|
||||
ATTR_MAP(MaxPoolGradWithArgmax) = {
|
||||
{"kernel_size", ATTR_DESC(ksize, AnyTraits<int64_t>(), AnyTraits<std::vector<int64_t>>())},
|
||||
{"strides", ATTR_DESC(strides, AnyTraits<int64_t>(), AnyTraits<std::vector<int64_t>>())},
|
||||
{"pad_mode", ATTR_DESC(padding, AnyTraits<std::string>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
// 存入MaxPoolGradWithArgmax对应空间并用attr_map_指针保存
|
||||
// 其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
// std:vector<>的作用为构建<>内类型的容量可变的数组
|
||||
OUTPUT_MAP(MaxPoolGradWithArgmax) = {{0, OUTPUT_DESC(y)}};
|
||||
//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入MaxPoolGradWithArgmax对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(MaxPoolGradWithArgmax, kNameMaxPoolGradWithArgmax, ADPT_DESC(MaxPoolGradWithArgmax))
|
||||
//构造指向MaxPoolGradWithArgmax的指针并储存，创建结构体RegAdptDescMaxPoolGradWithArgmax
|
||||
|
||||
// MaxPoolGradGradWithArgmax
|
||||
INPUT_MAP(MaxPoolGradGradWithArgmax) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(grad)}, {3, INPUT_DESC(argmax)}};
|
||||
//将变量x、grad、argmax处理并存入对应InputDesc结构体的相应变量中，存入MaxPoolGradGradWithArgmax对应空间内并用input_map指针保存
|
||||
ATTR_MAP(MaxPoolGradGradWithArgmax) = {
|
||||
{"kernel_size", ATTR_DESC(ksize, AnyTraits<int64_t>(), AnyTraits<std::vector<int64_t>>())},
|
||||
{"strides", ATTR_DESC(strides, AnyTraits<int64_t>(), AnyTraits<std::vector<int64_t>>())},
|
||||
{"pad_mode", ATTR_DESC(padding, AnyTraits<std::string>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
// 存入MaxPoolGradGradWithArgmax对应空间并用attr_map_指针保存
|
||||
// 其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
// std:vector<>的作用为构建<>内类型的容量可变的数组
|
||||
OUTPUT_MAP(MaxPoolGradGradWithArgmax) = {{0, OUTPUT_DESC(y)}};
|
||||
//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入MaxPoolGradGradWithArgmax对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(MaxPoolGradGradWithArgmax, kNameMaxPoolGradGradWithArgmax, ADPT_DESC(MaxPoolGradGradWithArgmax))
|
||||
//构造指向MaxPoolGradGradWithArgmax的指针并储存，创建结构体RegAdptDescMaxPoolGradGradWithArgmax
|
||||
|
||||
// Pooling
|
||||
INPUT_MAP(Pooling) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入Pooling对应空间内并用input_map指针保存
|
||||
ATTR_MAP(Pooling) = {{"mode", ATTR_DESC(mode, AnyTraits<int64_t>())},
|
||||
{"global", ATTR_DESC(global_pooling, AnyTraits<bool>())},
|
||||
{"kernel_size", ATTR_DESC(window, AnyTraits<int64_t>(), AnyTraits<std::vector<int64_t>>())},
|
||||
{"strides", ATTR_DESC(stride, AnyTraits<int64_t>(), AnyTraits<std::vector<int64_t>>())},
|
||||
{"pad", ATTR_DESC(pad, AnyTraits<int64_t>(), AnyTraits<std::vector<int64_t>>())},
|
||||
{"dilation", ATTR_DESC(dilation, AnyTraits<int64_t>(), AnyTraits<std::vector<int64_t>>())},
|
||||
{"round_mode", ATTR_DESC(ceil_mode, AnyTraits<int64_t>())},
|
||||
{"format", ATTR_DESC(data_format, AnyTraits<std::string>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
// 存入Pooling对应空间并用attr_map_指针保存
|
||||
// 其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
// std:vector<>的作用为构建<>内类型的容量可变的数组
|
||||
OUTPUT_MAP(Pooling) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入Pooling对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(Pooling, kNamePooling, ADPT_DESC(Pooling))//构造指向Pooling的指针并储存，创建结构体RegAdptDescPooling
|
||||
|
||||
// MaxPoolV3
|
||||
INPUT_MAP(MaxPoolV3) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入MaxPoolV3对应空间内并用input_map指针保存
|
||||
ATTR_MAP(MaxPoolV3) = {{"kernel_size", ATTR_DESC(ksize, AnyTraits<int64_t>(), AnyTraits<std::vector<int64_t>>())},
|
||||
{"strides", ATTR_DESC(strides, AnyTraits<int64_t>(), AnyTraits<std::vector<int64_t>>())},
|
||||
{"padding_mode", ATTR_DESC(padding_mode, AnyTraits<std::string>())},
|
||||
{"pad", ATTR_DESC(pads, AnyTraits<int64_t>(), AnyTraits<std::vector<int64_t>>())},
|
||||
{"format", ATTR_DESC(data_format, AnyTraits<std::string>())},
|
||||
{"global", ATTR_DESC(global_pooling, AnyTraits<bool>())},
|
||||
{"ceil_mode", ATTR_DESC(ceil_mode, AnyTraits<bool>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
// 存入MaxPoolV3对应空间并用attr_map_指针保存
|
||||
// 其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
// std:vector<>的作用为构建<>内类型的容量可变的数组
|
||||
OUTPUT_MAP(MaxPoolV3) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入MaxPoolV3对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(MaxPoolV3, kNameMaxPoolV3, ADPT_DESC(MaxPoolV3))//构造指向MaxPoolV3的指针并储存，创建结构体RegAdptDescMaxPoolV3
|
||||
|
||||
// AvgPoolV2
|
||||
INPUT_MAP(AvgPoolV2) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入AvgPoolV2对应空间内并用input_map指针保存
|
||||
ATTR_MAP(AvgPoolV2) = {{"kernel_size", ATTR_DESC(ksize, AnyTraits<int64_t>(), AnyTraits<std::vector<int64_t>>())},
|
||||
{"strides", ATTR_DESC(strides, AnyTraits<int64_t>(), AnyTraits<std::vector<int64_t>>())},
|
||||
{"padding_mode", ATTR_DESC(padding_mode, AnyTraits<std::string>())},
|
||||
{"pad", ATTR_DESC(pads, AnyTraits<int64_t>(), AnyTraits<std::vector<int64_t>>())},
|
||||
{"format", ATTR_DESC(data_format, AnyTraits<std::string>())},
|
||||
{"global", ATTR_DESC(global_pooling, AnyTraits<bool>())},
|
||||
{"ceil_mode", ATTR_DESC(ceil_mode, AnyTraits<bool>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
// 存入AvgPoolV2对应空间并用attr_map_指针保存
|
||||
// 其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
// std:vector<>的作用为构建<>内类型的容量可变的数组
|
||||
OUTPUT_MAP(AvgPoolV2) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入AvgPoolV2对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(AvgPoolV2, kNameAvgPoolV2, ADPT_DESC(AvgPoolV2))//构造指向AvgPoolV2的指针并储存，创建结构体RegAdptDescAvgPoolV2
|
||||
|
||||
// GlobalAveragePool
|
||||
INPUT_MAP(GlobalAveragePool) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入GlobalAveragePool对应空间内并用input_map指针保存
|
||||
ATTR_MAP(GlobalAveragePool) = EMPTY_ATTR_MAP;//将空变量存入AvgPoolV2对应空间并用attr_map_指针保存
|
||||
OUTPUT_MAP(GlobalAveragePool) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入GlobalAveragePool对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(GlobalAveragePool, kNameGlobalAvgPool, ADPT_DESC(GlobalAveragePool))//构造指向GlobalAveragePool的指针并储存，创建结构体RegAdptDescGlobalAveragePool
|
||||
|
||||
// Upsample
|
||||
INPUT_MAP(Upsample) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入Upsample对应空间内并用input_map指针保存
|
||||
ATTR_MAP(Upsample) = {{"scale", ATTR_DESC(scale, AnyTraits<float>())},
|
||||
{"stride_h", ATTR_DESC(stride_h, AnyTraits<int64_t>())},
|
||||
{"stride_w", ATTR_DESC(stride_w, AnyTraits<int64_t>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
// 存入Upsample对应空间并用attr_map_指针保存
|
||||
// 其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
// std:vector<>的作用为构建<>内类型的容量可变的数组
|
||||
OUTPUT_MAP(Upsample) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入Upsample对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(Upsample, kNameUpsample, ADPT_DESC(Upsample))//构造指向Upsample的指针并储存，创建结构体RegAdptDescUpsample
|
||||
} // namespace mindspore::transform
|
||||
|
|
@ -0,0 +1,76 @@
|
|||
/**
|
||||
* Copyright 2019-2021 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_NN_POOLING_OPS_DECLARE_H_//判断宏是否被定义，如果宏没有定义，则编译下面代码
|
||||
#define MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_NN_POOLING_OPS_DECLARE_H_//定义预处理宏_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_NN_POOLING_OPS_DECLARE_H_
|
||||
|
||||
#include <string>//导入标准库中的字符串类及相关操作
|
||||
#include "utils/hash_map.h"//按照路径寻找以下文件，导入到本文件,以下同理
|
||||
#include "transform/graph_ir/op_declare/op_declare_macro.h"
|
||||
#include "ops/nn_ops.h"
|
||||
#include "ops/nn_pooling_ops.h"
|
||||
|
||||
namespace mindspore::transform {//创建名为transform的空间，其空间处于空间mindspore下
|
||||
//对实现MaxPoolWithArgmax的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(MaxPoolWithArgmax)
|
||||
DECLARE_OP_USE_OUTPUT(MaxPoolWithArgmax)
|
||||
//对实现MaxPoolGradWithArgmax的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(MaxPoolGradWithArgmax)
|
||||
DECLARE_OP_USE_OUTPUT(MaxPoolGradWithArgmax)
|
||||
//对实现MaxPoolGradGradWithArgmax的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(MaxPoolGradGradWithArgmax)
|
||||
DECLARE_OP_USE_OUTPUT(MaxPoolGradGradWithArgmax)
|
||||
//对实现MaxPool的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(MaxPool)
|
||||
DECLARE_OP_USE_OUTPUT(MaxPool)
|
||||
//对实现MaxPoolGrad的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(MaxPoolGrad)
|
||||
DECLARE_OP_USE_OUTPUT(MaxPoolGrad)
|
||||
//对实现MaxPoolGradGrad的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(MaxPoolGradGrad)
|
||||
DECLARE_OP_USE_OUTPUT(MaxPoolGradGrad)
|
||||
//对实现MaxPool3D的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(MaxPool3D)
|
||||
DECLARE_OP_USE_OUTPUT(MaxPool3D)
|
||||
//对实现MaxPool3DGrad的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(MaxPool3DGrad)
|
||||
DECLARE_OP_USE_OUTPUT(MaxPool3DGrad)
|
||||
//对实现MaxPool3DGradGrad的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(MaxPool3DGradGrad)
|
||||
DECLARE_OP_USE_OUTPUT(MaxPool3DGradGrad)
|
||||
//对实现AvgPool的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(AvgPool)
|
||||
DECLARE_OP_USE_OUTPUT(AvgPool)
|
||||
//对实现AvgPoolGrad的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(AvgPoolGrad)
|
||||
DECLARE_OP_USE_OUTPUT(AvgPoolGrad)
|
||||
//对实现Pooling的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(Pooling)
|
||||
DECLARE_OP_USE_OUTPUT(Pooling)
|
||||
//对实现MaxPoolV3的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(MaxPoolV3)
|
||||
DECLARE_OP_USE_OUTPUT(MaxPoolV3)
|
||||
//对实现AvgPoolV2的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(AvgPoolV2)
|
||||
DECLARE_OP_USE_OUTPUT(AvgPoolV2)
|
||||
//对实现GlobalAveragePool的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(GlobalAveragePool)
|
||||
DECLARE_OP_USE_OUTPUT(GlobalAveragePool)
|
||||
//对实现Upsample的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(Upsample)
|
||||
DECLARE_OP_USE_OUTPUT(Upsample)
|
||||
} // namespace mindspore::transform
|
||||
#endif // MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_NN_POOLING_OPS_DECLARE_H_
|
||||
|
|
@ -0,0 +1,319 @@
|
|||
/**
|
||||
* Copyright 2019-2021 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#include "transform/graph_ir/op_declare/nn_training_ops_declare.h"//按照路径寻找以下文件，导入到本文件
|
||||
|
||||
namespace mindspore::transform {//创建名为transform的空间，其空间处于空间mindspore下
|
||||
// ApplyMomentum
|
||||
INPUT_MAP(ApplyMomentum) = {
|
||||
{1, INPUT_DESC(var)}, {2, INPUT_DESC(accum)}, {3, INPUT_DESC(lr)}, {4, INPUT_DESC(grad)}, {5, INPUT_DESC(momentum)}};
|
||||
//将变量var、accum、lr、grad、momentum处理并存入对应InputDesc结构体的相应变量中，存入ApplyMomentum对应空间内并用input_map指针保存
|
||||
ATTR_MAP(ApplyMomentum) = {{"use_nesterov", ATTR_DESC(use_nesterov, AnyTraits<bool>())},
|
||||
{"use_locking", ATTR_DESC(use_locking, AnyTraits<bool>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
// 存入ApplyMomentum对应空间并用attr_map_指针保存
|
||||
// 其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
OUTPUT_MAP(ApplyMomentum) = {{0, OUTPUT_DESC(var)}};//将变量var处理并存入对应OutputDesc结构体的相应变量中，存入ApplyMomentum对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(ApplyMomentum, kNameApplyMomentum, ADPT_DESC(ApplyMomentum))
|
||||
//构造指向ApplyMomentum的指针并储存，创建结构体RegAdptDescApplyMomentum
|
||||
|
||||
// LarsV2Update
|
||||
INPUT_MAP(LarsV2Update) = {{1, INPUT_DESC(w)},
|
||||
{2, INPUT_DESC(g)},
|
||||
{3, INPUT_DESC(w_square_sum)},
|
||||
{4, INPUT_DESC(g_square_sum)},
|
||||
{5, INPUT_DESC(weight_decay)},
|
||||
{6, INPUT_DESC(learning_rate)}};
|
||||
//将变量w、g、w_square_sum、g_square_sum、weight_decay、learning_rate处理并存入对应InputDesc结构体的相应变量中，存入arsV2Update对应空间内并用input_map指针保存
|
||||
ATTR_MAP(LarsV2Update) = {{"epsilon", ATTR_DESC(epsilon, AnyTraits<float>())},
|
||||
{"hyperpara", ATTR_DESC(hyperpara, AnyTraits<float>())},
|
||||
{"use_clip", ATTR_DESC(use_clip, AnyTraits<bool>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
// 存入LarsV2Update对应空间并用attr_map_指针保存
|
||||
// 其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
OUTPUT_MAP(LarsV2Update) = {{0, OUTPUT_DESC(g_new)}};//将变量g_new处理并存入对应OutputDesc结构体的相应变量中，存入LarsV2Update对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(LarsV2Update, kNameLARSUpdate, ADPT_DESC(LarsV2Update))//构造指向LarsV2Update的指针并储存，创建结构体RegAdptDescLarsV2Update
|
||||
|
||||
// ApplyAdam
|
||||
INPUT_MAP(ApplyAdam) = {{1, INPUT_DESC(var)}, {2, INPUT_DESC(m)}, {3, INPUT_DESC(v)},
|
||||
{4, INPUT_DESC(beta1_power)}, {5, INPUT_DESC(beta2_power)}, {6, INPUT_DESC(lr)},
|
||||
{7, INPUT_DESC(beta1)}, {8, INPUT_DESC(beta2)}, {9, INPUT_DESC(epsilon)},
|
||||
{10, INPUT_DESC(grad)}};
|
||||
//将变量var、m、v、beta1_power、beta2_power、lr、beta1、beta2、epsilon、grad处理并存入对应InputDesc结构体的相应变量中，存入ApplyAdam对应空间内并用input_map指针保存
|
||||
ATTR_MAP(ApplyAdam) = {{"use_locking", ATTR_DESC(use_locking, AnyTraits<bool>())},
|
||||
{"use_nesterov", ATTR_DESC(use_nesterov, AnyTraits<bool>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
// 存入ApplyAdam对应空间并用attr_map_指针保存
|
||||
// 其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
OUTPUT_MAP(ApplyAdam) = {{0, OUTPUT_DESC(var)}};//将变量var处理并存入对应OutputDesc结构体的相应变量中，存入ApplyAdam对应空间内并用output_map_指针保存
|
||||
|
||||
// ApplyAdamD
|
||||
INPUT_MAP(ApplyAdamD) = {{1, INPUT_DESC(var)}, {2, INPUT_DESC(m)}, {3, INPUT_DESC(v)},
|
||||
{4, INPUT_DESC(beta1_power)}, {5, INPUT_DESC(beta2_power)}, {6, INPUT_DESC(lr)},
|
||||
{7, INPUT_DESC(beta1)}, {8, INPUT_DESC(beta2)}, {9, INPUT_DESC(epsilon)},
|
||||
{10, INPUT_DESC(grad)}};
|
||||
//将变量var、m、v、beta1_power、beta2_power、lr、beta1、beta2、epsilon、grad处理并存入对应InputDesc结构体的相应变量中，存入ApplyAdamD对应空间内并用input_map指针保存
|
||||
ATTR_MAP(ApplyAdamD) = {{"use_locking", ATTR_DESC(use_locking, AnyTraits<bool>())},
|
||||
{"use_nesterov", ATTR_DESC(use_nesterov, AnyTraits<bool>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
// 存入ApplyAdamD对应空间并用attr_map_指针保存
|
||||
// 其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
OUTPUT_MAP(ApplyAdamD) = {{0, OUTPUT_DESC(var)}, {1, OUTPUT_DESC(m)}, {2, OUTPUT_DESC(v)}};
|
||||
//将变量var、m、v处理并存入对应OutputDesc结构体的相应变量中，存入ApplyAdamD对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(ApplyAdamD, kNameApplyAdam, ADPT_DESC(ApplyAdamD))//构造指向ApplyAdamD的指针并储存，创建结构体RegAdptDescApplyAdamD
|
||||
REG_ADPT_DESC(ApplyAdam, kNameApplyAdam, ADPT_DESC(ApplyAdam))//构造指向ApplyAdam的指针并储存，创建结构体RegAdptDescApplyAdam
|
||||
|
||||
// ApplyAdagradD
|
||||
INPUT_MAP(ApplyAdagradD) = {{1, INPUT_DESC(var)}, {2, INPUT_DESC(accum)}, {3, INPUT_DESC(lr)}, {4, INPUT_DESC(grad)}};
|
||||
//将变量var、accum、lr、grad处理并存入对应InputDesc结构体的相应变量中，存入ApplyAdagradD对应空间内并用input_map指针保存
|
||||
ATTR_MAP(ApplyAdagradD) = {{"update_slots", ATTR_DESC(update_slots, AnyTraits<bool>())},
|
||||
{"use_locking", ATTR_DESC(use_locking, AnyTraits<bool>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
// 存入ApplyAdagradD对应空间并用attr_map_指针保存
|
||||
// 其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
OUTPUT_MAP(ApplyAdagradD) = {{0, OUTPUT_DESC(var)}, {1, OUTPUT_DESC(accum)}};
|
||||
//将变量var、accum处理并存入对应OutputDesc结构体的相应变量中，存入ApplyAdagradD对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(ApplyAdagradD, kNameApplyAdagrad, ADPT_DESC(ApplyAdagradD))//构造指向ApplyAdagradD的指针并储存，创建结构体RegAdptDescApplyAdagradD
|
||||
|
||||
// ApplyAdagradV2D
|
||||
INPUT_MAP(ApplyAdagradV2D) = {{1, INPUT_DESC(var)}, {2, INPUT_DESC(accum)}, {3, INPUT_DESC(lr)}, {4, INPUT_DESC(grad)}};
|
||||
//将变量var、accum、lr、grad处理并存入对应InputDesc结构体的相应变量中，存入ApplyAdagradV2D对应空间内并用input_map指针保存
|
||||
ATTR_MAP(ApplyAdagradV2D) = {{"epsilon", ATTR_DESC(epsilon, AnyTraits<float>())},
|
||||
{"update_slots", ATTR_DESC(update_slots, AnyTraits<bool>())},
|
||||
{"use_locking", ATTR_DESC(use_locking, AnyTraits<bool>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
// 存入ApplyAdagradV2D对应空间并用attr_map_指针保存
|
||||
// 其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
OUTPUT_MAP(ApplyAdagradV2D) = {{0, OUTPUT_DESC(var)}, {1, OUTPUT_DESC(accum)}};
|
||||
//将变量var、accum处理并存入对应OutputDesc结构体的相应变量中，存入ApplyAdagradV2D对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(ApplyAdagradV2D, kNameApplyAdagradV2D, ADPT_DESC(ApplyAdagradV2D))//构造指向ApplyAdagradV2D的指针并储存，创建结构体RegAdptDescApplyAdagradV2D
|
||||
|
||||
// ApplyAddSignD
|
||||
INPUT_MAP(ApplyAddSignD) = {{1, INPUT_DESC(var)}, {2, INPUT_DESC(m)}, {3, INPUT_DESC(lr)},
|
||||
{4, INPUT_DESC(alpha)}, {5, INPUT_DESC(sign_decay)}, {6, INPUT_DESC(beta)},
|
||||
{7, INPUT_DESC(grad)}};
|
||||
//将变量var、m、lr、alpha、sign_decay、beta、grad处理并存入对应InputDesc结构体的相应变量中，存入ApplyAddSignD对应空间内并用input_map指针保存
|
||||
ATTR_MAP(ApplyAddSignD) = {{"use_locking", ATTR_DESC(use_locking, AnyTraits<bool>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
// 存入ApplyAddSignD对应空间并用attr_map_指针保存
|
||||
// 其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
OUTPUT_MAP(ApplyAddSignD) = {{0, OUTPUT_DESC(var)}, {1, OUTPUT_DESC(m)}};
|
||||
//将变量var、m处理并存入对应OutputDesc结构体的相应变量中，存入ApplyAddSignD对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(ApplyAddSignD, kNameApplyAddSignD, ADPT_DESC(ApplyAddSignD))//构造指向ApplyAddSignD的指针并储存，创建结构体RegAdptDescApplyAddSignD
|
||||
|
||||
// SparseApplyAdagradV2D
|
||||
INPUT_MAP(SparseApplyAdagradV2D) = {
|
||||
{1, INPUT_DESC(var)}, {2, INPUT_DESC(accum)}, {3, INPUT_DESC(grad)}, {4, INPUT_DESC(indices)}};
|
||||
//将变量var、accum、grad、indices处理并存入对应InputDesc结构体的相应变量中，存入SparseApplyAdagradV2D对应空间内并用input_map指针保存
|
||||
ATTR_MAP(SparseApplyAdagradV2D) = {{"lr", ATTR_DESC(lr, AnyTraits<float>())},
|
||||
{"epsilon", ATTR_DESC(epsilon, AnyTraits<float>())},
|
||||
{"update_slots", ATTR_DESC(update_slots, AnyTraits<bool>())},
|
||||
{"use_locking", ATTR_DESC(use_locking, AnyTraits<bool>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
// 存入SparseApplyAdagradV2D对应空间并用attr_map_指针保存
|
||||
// 其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
OUTPUT_MAP(SparseApplyAdagradV2D) = {{0, OUTPUT_DESC(var)}, {1, OUTPUT_DESC(accum)}};
|
||||
//将变量var、accum处理并存入对应OutputDesc结构体的相应变量中，存入SparseApplyAdagradV2D对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(SparseApplyAdagradV2D, kNameSparseApplyAdagradV2D, ADPT_DESC(SparseApplyAdagradV2D))
|
||||
//构造指向SparseApplyAdagradV2D的指针并储存，创建结构体RegAdptDescSparseApplyAdagradV2D
|
||||
|
||||
// DataFormatDimMap
|
||||
INPUT_MAP(DataFormatDimMap) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入DataFormatDimMap对应空间内并用input_map指针保存
|
||||
ATTR_MAP(DataFormatDimMap) = {{"src_format", ATTR_DESC(src_format, AnyTraits<std::string>())},
|
||||
{"dst_format", ATTR_DESC(dst_format, AnyTraits<std::string>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
// 存入DataFormatDimMap对应空间并用attr_map_指针保存
|
||||
// 其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
OUTPUT_MAP(DataFormatDimMap) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入DataFormatDimMap对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(DataFormatDimMap, kNameDataFormatDimMap, ADPT_DESC(DataFormatDimMap))
|
||||
//构造指向DataFormatDimMap的指针并储存，创建结构体RegAdptDescDataFormatDimMap
|
||||
|
||||
// ApplyAdadeltaD
|
||||
INPUT_MAP(ApplyAdadeltaD) = {{1, INPUT_DESC(var)}, {2, INPUT_DESC(accum)}, {3, INPUT_DESC(accum_update)},
|
||||
{4, INPUT_DESC(lr)}, {5, INPUT_DESC(rho)}, {6, INPUT_DESC(epsilon)},
|
||||
{7, INPUT_DESC(grad)}};
|
||||
//将变量var、accum、accum_update、lr、rho、epsilon、grad处理并存入对应InputDesc结构体的相应变量中，存入ApplyAdadeltaD对应空间内并用input_map指针保存
|
||||
ATTR_MAP(ApplyAdadeltaD) = {{"use_locking", ATTR_DESC(use_locking, AnyTraits<bool>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
// 存入ApplyAdadeltaD对应空间并用attr_map_指针保存
|
||||
// 其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
OUTPUT_MAP(ApplyAdadeltaD) = {{0, OUTPUT_DESC(var)}, {1, OUTPUT_DESC(accum)}, {2, OUTPUT_DESC(accum_update)}};
|
||||
//将变量var、accum、accum_update处理并存入对应OutputDesc结构体的相应变量中，存入ApplyAdadeltaD对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(ApplyAdadeltaD, kNameApplyAdadelta, ADPT_DESC(ApplyAdadeltaD))
|
||||
//构造指向ApplyAdadeltaD的指针并储存，创建结构体RegAdptDescApplyAdadeltaD
|
||||
|
||||
// ApplyAdaMaxD
|
||||
INPUT_MAP(ApplyAdaMaxD) = {{1, INPUT_DESC(var)}, {2, INPUT_DESC(m)}, {3, INPUT_DESC(v)},
|
||||
{4, INPUT_DESC(beta1_power)}, {5, INPUT_DESC(lr)}, {6, INPUT_DESC(beta1)},
|
||||
{7, INPUT_DESC(beta2)}, {8, INPUT_DESC(epsilon)}, {9, INPUT_DESC(grad)}};
|
||||
//将变量var、m、v、beta1_power、lr、beta1、beta2、epsilon、grad处理并存入对应InputDesc结构体的相应变量中，存入ApplyAdaMaxD对应空间内并用input_map指针保存
|
||||
ATTR_MAP(ApplyAdaMaxD) = {{"use_locking", ATTR_DESC(use_locking, AnyTraits<bool>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
// 存入ApplyAdaMaxD对应空间并用attr_map_指针保存
|
||||
// 其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
OUTPUT_MAP(ApplyAdaMaxD) = {{0, OUTPUT_DESC(var)}, {1, OUTPUT_DESC(m)}, {2, OUTPUT_DESC(v)}};
|
||||
//将变量var、m、v处理并存入对应OutputDesc结构体的相应变量中，存入ApplyAdaMaxD对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(ApplyAdaMaxD, kNameApplyAdaMax, ADPT_DESC(ApplyAdaMaxD))
|
||||
//构造指向ApplyAdaMaxD的指针并储存，创建结构体RegAdptDescApplyAdaMaxD
|
||||
|
||||
// ApplyGradientDescent
|
||||
INPUT_MAP(ApplyGradientDescent) = {{1, INPUT_DESC(var)}, {2, INPUT_DESC(alpha)}, {3, INPUT_DESC(delta)}};
|
||||
//将变量var、alpha、delta处理并存入对应InputDesc结构体的相应变量中，存入ApplyGradientDescent对应空间内并用input_map指针保存
|
||||
ATTR_MAP(ApplyGradientDescent) = {{"use_locking", ATTR_DESC(use_locking, AnyTraits<bool>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
// 存入ApplyGradientDescent对应空间并用attr_map_指针保存
|
||||
// 其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
OUTPUT_MAP(ApplyGradientDescent) = {{0, OUTPUT_DESC(var)}};
|
||||
//将变量var处理并存入对应OutputDesc结构体的相应变量中，存入ApplyGradientDescent对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(ApplyGradientDescent, kNameApplyGradientDescent, ADPT_DESC(ApplyGradientDescent))
|
||||
//构造指向ApplyGradientDescent的指针并储存，创建结构体RegAdptDescApplyGradientDescent
|
||||
|
||||
// ApplyPowerSignD
|
||||
INPUT_MAP(ApplyPowerSignD) = {{1, INPUT_DESC(var)}, {2, INPUT_DESC(m)}, {3, INPUT_DESC(lr)},
|
||||
{4, INPUT_DESC(logbase)}, {5, INPUT_DESC(sign_decay)}, {6, INPUT_DESC(beta)},
|
||||
{7, INPUT_DESC(grad)}};
|
||||
//将变量var、m、lr、logbase、sign_decay、beta、grad处理并存入对应InputDesc结构体的相应变量中，存入ApplyPowerSignD对应空间内并用input_map指针保存
|
||||
ATTR_MAP(ApplyPowerSignD) = {{"use_locking", ATTR_DESC(use_locking, AnyTraits<bool>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
// 存入ApplyPowerSignD对应空间并用attr_map_指针保存
|
||||
// 其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
OUTPUT_MAP(ApplyGradientDescent) = {{0, OUTPUT_DESC(var)}};
|
||||
OUTPUT_MAP(ApplyPowerSignD) = {{0, OUTPUT_DESC(var)}, {1, OUTPUT_DESC(m)}};
|
||||
//将变量var、m处理并存入对应OutputDesc结构体的相应变量中，存入ApplyPowerSignDt对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(ApplyPowerSignD, kNameApplyPowerSign, ADPT_DESC(ApplyPowerSignD))
|
||||
//构造指向ApplyPowerSignD的指针并储存，创建结构体RegAdptDescApplyPowerSignD
|
||||
|
||||
// ApplyProximalGradientDescent
|
||||
INPUT_MAP(ApplyProximalGradientDescent) = {
|
||||
{1, INPUT_DESC(var)}, {2, INPUT_DESC(alpha)}, {3, INPUT_DESC(l1)}, {4, INPUT_DESC(l2)}, {5, INPUT_DESC(delta)}};
|
||||
//将变量var、alpha、l1、l2、delta处理并存入对应InputDesc结构体的相应变量中，存入ApplyProximalGradientDescent对应空间内并用input_map指针保存
|
||||
ATTR_MAP(ApplyProximalGradientDescent) = {{"use_locking", ATTR_DESC(use_locking, AnyTraits<bool>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
// 存入ApplyProximalGradientDescent对应空间并用attr_map_指针保存
|
||||
// 其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
OUTPUT_MAP(ApplyProximalGradientDescent) = {{0, OUTPUT_DESC(var)}};
|
||||
//将变量var处理并存入对应OutputDesc结构体的相应变量中，存入ApplyProximalGradientDescent对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(ApplyProximalGradientDescent, kNameApplyProximalGradientDescent, ADPT_DESC(ApplyProximalGradientDescent))
|
||||
//构造指向ApplyProximalGradientDescent的指针并储存，创建结构体RegAdptDescApplyProximalGradientDescent
|
||||
|
||||
// SGD
|
||||
INPUT_MAP(SGD) = {{1, INPUT_DESC(parameters)}, {2, INPUT_DESC(gradient)}, {3, INPUT_DESC(learning_rate)},
|
||||
{4, INPUT_DESC(accum)}, {5, INPUT_DESC(momentum)}, {6, INPUT_DESC(stat)}};
|
||||
//将变量parameters、gradient、learning_rate、accum、momentum、stat处理并存入对应InputDesc结构体的相应变量中，存入SGD对应空间内并用input_map指针保存
|
||||
ATTR_MAP(SGD) = {{"dampening", ATTR_DESC(dampening, AnyTraits<float>())},
|
||||
{"weight_decay", ATTR_DESC(weight_decay, AnyTraits<float>())},
|
||||
{"nesterov", ATTR_DESC(nesterov, AnyTraits<bool>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
// 存入SGD对应空间并用attr_map_指针保存
|
||||
// 其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
OUTPUT_MAP(SGD) = {{0, OUTPUT_DESC(parameters)}};//将变量parameters处理并存入对应OutputDesc结构体的相应变量中，存入SGD对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(SGD, kNameSGD, ADPT_DESC(SGD))//构造指向SGD的指针并储存，创建结构体RegAdptDescSGD
|
||||
|
||||
// SparseApplyAdagradD
|
||||
INPUT_MAP(SparseApplyAdagradD) = {
|
||||
{1, INPUT_DESC(var)}, {2, INPUT_DESC(accum)}, {3, INPUT_DESC(grad)}, {4, INPUT_DESC(indices)}};
|
||||
//将变量var、accum、grad、indices处理并存入对应InputDesc结构体的相应变量中，存入SparseApplyAdagradD对应空间内并用input_map指针保存
|
||||
ATTR_MAP(SparseApplyAdagradD) = {{"lr", ATTR_DESC(lr, AnyTraits<float>())},
|
||||
{"use_locking", ATTR_DESC(use_locking, AnyTraits<bool>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
// 存入SparseApplyAdagradD对应空间并用attr_map_指针保存
|
||||
// 其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
OUTPUT_MAP(SparseApplyAdagradD) = {{0, OUTPUT_DESC(var)}};
|
||||
//将变量var处理并存入对应OutputDesc结构体的相应变量中，存入SparseApplyAdagradD对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(SparseApplyAdagradD, kNameSparseApplyAdagrad, ADPT_DESC(SparseApplyAdagradD))
|
||||
//构造指向SparseApplyAdagradD的指针并储存，创建结构体RegAdptDescSparseApplyAdagradD
|
||||
|
||||
// ApplyProximalAdagradD
|
||||
INPUT_MAP(ApplyProximalAdagradD) = {{1, INPUT_DESC(var)}, {2, INPUT_DESC(accum)}, {3, INPUT_DESC(lr)},
|
||||
{4, INPUT_DESC(l1)}, {5, INPUT_DESC(l2)}, {6, INPUT_DESC(grad)}};
|
||||
//将变量var、accum、lr、l1、l2、grad处理并存入对应InputDesc结构体的相应变量中，存入ApplyProximalAdagradD对应空间内并用input_map指针保存
|
||||
ATTR_MAP(ApplyProximalAdagradD) = {{"use_locking", ATTR_DESC(use_locking, AnyTraits<bool>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
// 存入ApplyProximalAdagradD对应空间并用attr_map_指针保存
|
||||
// 其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
OUTPUT_MAP(ApplyProximalAdagradD) = {{0, OUTPUT_DESC(var)}, {1, OUTPUT_DESC(accum)}};
|
||||
//将变量accum处理并存入对应OutputDesc结构体的相应变量中，存入ApplyProximalAdagradD对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(ApplyProximalAdagradD, kNameApplyProximalAdagrad, ADPT_DESC(ApplyProximalAdagradD))
|
||||
//构造指向ApplyProximalAdagradD的指针并储存，创建结构体RegAdptDescApplyProximalAdagradD
|
||||
|
||||
// SparseApplyProximalAdagradD
|
||||
INPUT_MAP(SparseApplyProximalAdagradD) = {{1, INPUT_DESC(var)}, {2, INPUT_DESC(accum)}, {3, INPUT_DESC(lr)},
|
||||
{4, INPUT_DESC(l1)}, {5, INPUT_DESC(l2)}, {6, INPUT_DESC(grad)},
|
||||
{7, INPUT_DESC(indices)}};
|
||||
//将变量var、accum、lr、l1、l2、grad、indices处理并存入对应InputDesc结构体的相应变量中，存入SparseApplyProximalAdagradD对应空间内并用input_map指针保存
|
||||
ATTR_MAP(SparseApplyProximalAdagradD) = {{"use_locking", ATTR_DESC(use_locking, AnyTraits<bool>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
// 存入SparseApplyProximalAdagradD对应空间并用attr_map_指针保存
|
||||
// 其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
OUTPUT_MAP(SparseApplyProximalAdagradD) = {{0, OUTPUT_DESC(var)}, {1, OUTPUT_DESC(accum)}};
|
||||
//将变量accum处理并存入对应OutputDesc结构体的相应变量中，存入SparseApplyProximalAdagradD对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(SparseApplyProximalAdagradD, kNameSparseApplyProximalAdagradD, ADPT_DESC(SparseApplyProximalAdagradD))
|
||||
//构造指向SparseApplyProximalAdagradD的指针并储存，创建结构体RegAdptDescSparseApplyProximalAdagradD
|
||||
|
||||
// SparseApplyFtrlD
|
||||
INPUT_MAP(SparseApplyFtrlD) = {{1, INPUT_DESC(var)},
|
||||
{2, INPUT_DESC(accum)},
|
||||
{3, INPUT_DESC(linear)},
|
||||
{4, INPUT_DESC(grad)},
|
||||
{5, INPUT_DESC(indices)}};
|
||||
//将变量var、accum、linear、grad、indices处理并存入对应InputDesc结构体的相应变量中，存入SparseApplyFtrlD对应空间内并用input_map指针保存
|
||||
ATTR_MAP(SparseApplyFtrlD) = {{"use_locking", ATTR_DESC(use_locking, AnyTraits<bool>())},
|
||||
{"lr", ATTR_DESC(lr, AnyTraits<float>())},
|
||||
{"l1", ATTR_DESC(l1, AnyTraits<float>())},
|
||||
{"l2", ATTR_DESC(l2, AnyTraits<float>())},
|
||||
{"lr_power", ATTR_DESC(lr_power, AnyTraits<float>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
// 存入SparseApplyFtrlD对应空间并用attr_map_指针保存
|
||||
// 其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
OUTPUT_MAP(SparseApplyFtrlD) = {{0, OUTPUT_DESC(var)}};
|
||||
//将变量var处理并存入对应OutputDesc结构体的相应变量中，存入SparseApplyFtrlD对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(SparseApplyFtrlD, kNameSparseApplyFtrlD, ADPT_DESC(SparseApplyFtrlD))
|
||||
//构造指向SparseApplyFtrlD的指针并储存，创建结构体RegAdptDescSparseApplyFtrlD
|
||||
|
||||
// SparseApplyFtrlV2D
|
||||
INPUT_MAP(SparseApplyFtrlV2D) = {{1, INPUT_DESC(var)},
|
||||
{2, INPUT_DESC(accum)},
|
||||
{3, INPUT_DESC(linear)},
|
||||
{4, INPUT_DESC(grad)},
|
||||
{5, INPUT_DESC(indices)}};
|
||||
//将变量var、accum、linear、grad、indices处理并存入对应InputDesc结构体的相应变量中，存入SparseApplyFtrlV2D对应空间内并用input_map指针保存
|
||||
ATTR_MAP(SparseApplyFtrlV2D) = {{"lr", ATTR_DESC(lr, AnyTraits<float>())}, {"l1", ATTR_DESC(l1, AnyTraits<float>())}};
|
||||
//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
//存入SparseApplyFtrlV2D对应空间并用attr_map_指针保存
|
||||
//其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
OUTPUT_MAP(SparseApplyFtrlV2D) = {{0, OUTPUT_DESC(var)}, {1, OUTPUT_DESC(accum)}, {2, OUTPUT_DESC(linear)}};
|
||||
//将变量var、accum、linear处理并存入对应OutputDesc结构体的相应变量中，存入SparseApplyFtrlV2D对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(SparseApplyFtrlV2D, kNameSparseApplyFtrlV2D, ADPT_DESC(SparseApplyFtrlV2D))
|
||||
//构造指向SparseApplyFtrlV2D的指针并储存，创建结构体RegAdptDescSparseApplyFtrlV2D
|
||||
|
||||
// ApplyFtrl
|
||||
INPUT_MAP(ApplyFtrl) = {{1, INPUT_DESC(var)}, {2, INPUT_DESC(accum)}, {3, INPUT_DESC(linear)},
|
||||
{4, INPUT_DESC(grad)}, {5, INPUT_DESC(lr)}, {6, INPUT_DESC(l1)},
|
||||
{7, INPUT_DESC(l2)}, {8, INPUT_DESC(lr_power)}};
|
||||
//将变量var、accum、linear、grad、lr、l1、l2、lr_power处理并存入对应InputDesc结构体的相应变量中，存入ApplyFtrl对应空间内并用input_map指针保存
|
||||
ATTR_MAP(ApplyFtrl) = {{"use_locking", ATTR_DESC(use_locking, AnyTraits<bool>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
// 存入ApplyFtrl对应空间并用attr_map_指针保存
|
||||
// 其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
OUTPUT_MAP(ApplyFtrl) = {{0, OUTPUT_DESC(var)}};//将变量var处理并存入对应OutputDesc结构体的相应变量中，存入ApplyFtrl对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(ApplyFtrl, kNameApplyFtrl, ADPT_DESC(ApplyFtrl))//构造指向ApplyFtrl的指针并储存，创建结构体RegAdptDescApplyFtrl
|
||||
|
||||
// ApplyRMSPropD
|
||||
INPUT_MAP(ApplyRMSPropD) = {
|
||||
{1, INPUT_DESC(var)}, {2, INPUT_DESC(ms)}, {3, INPUT_DESC(mom)}, {4, INPUT_DESC(lr)}, {5, INPUT_DESC(grad)}};
|
||||
//将变量var、ms、mom、lr、grad处理并存入对应InputDesc结构体的相应变量中，存入ApplyRMSPropD对应空间内并用input_map指针保存
|
||||
INPUT_ATTR_MAP(ApplyRMSPropD) = {{6, ATTR_DESC(rho, AnyTraits<float>())},
|
||||
{7, ATTR_DESC(momentum, AnyTraits<float>())},
|
||||
{8, ATTR_DESC(epsilon, AnyTraits<float>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
// 存入ApplyRMSPropD对应空间并用attr_map_指针保存
|
||||
// 其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
ATTR_MAP(ApplyRMSPropD) = {{"use_locking", ATTR_DESC(use_locking, AnyTraits<bool>())}};
|
||||
OUTPUT_MAP(ApplyRMSPropD) = {{0, OUTPUT_DESC(var)}};//将变量var处理并存入对应OutputDesc结构体的相应变量中，存入ApplyRMSPropD对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(ApplyRMSPropD, kNameApplyRMSProp, ADPT_DESC(ApplyRMSPropD))//构造指向ApplyRMSPropD的指针并储存，创建结构体RegAdptDescApplyRMSPropD
|
||||
|
||||
// ApplyCenteredRMSProp
|
||||
INPUT_MAP(ApplyCenteredRMSProp) = {{1, INPUT_DESC(var)}, {2, INPUT_DESC(mg)}, {3, INPUT_DESC(ms)},
|
||||
{4, INPUT_DESC(mom)}, {5, INPUT_DESC(grad)}, {6, INPUT_DESC(lr)},
|
||||
{7, INPUT_DESC(rho)}, {8, INPUT_DESC(momentum)}, {9, INPUT_DESC(epsilon)}};
|
||||
//将变量var、mg、ms、mom、grad、lr、rho、momentum、epsilon处理并存入对应InputDesc结构体的相应变量中，存入ApplyCenteredRMSProp对应空间内并用input_map指针保存
|
||||
ATTR_MAP(ApplyCenteredRMSProp) = {{"use_locking", ATTR_DESC(use_locking, AnyTraits<bool>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
// 存入ApplyCenteredRMSProp对应空间并用attr_map_指针保存
|
||||
// 其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
OUTPUT_MAP(ApplyCenteredRMSProp) = {{0, OUTPUT_DESC(var)}};
|
||||
//将变量var处理并存入对应OutputDesc结构体的相应变量中，存入ApplyCenteredRMSProp对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(ApplyCenteredRMSProp, kNameApplyCenteredRMSProp, ADPT_DESC(ApplyCenteredRMSProp))
|
||||
//构造指向ApplyCenteredRMSProp的指针并储存，创建结构体RegAdptDescApplyCenteredRMSProp
|
||||
} // namespace mindspore::transform
|
||||
|
|
@ -0,0 +1,97 @@
|
|||
/**
|
||||
* Copyright 2019-2021 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_NN_TRAINING_OPS_DECLARE_H_//判断宏是否被定义，如果宏没有定义，则编译下面代码
|
||||
#define MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_NN_TRAINING_OPS_DECLARE_H_//定义预处理宏_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_NN_TRAINING_OPS_DECLARE_H_
|
||||
|
||||
#include <string>//导入标准库中的字符串类及相关操作
|
||||
#include "utils/hash_map.h"//按照路径寻找以下文件，导入到本文件,以下同理
|
||||
#include "transform/graph_ir/op_declare/op_declare_macro.h"
|
||||
#include "ops/nn_training_ops.h"
|
||||
|
||||
namespace mindspore::transform {//创建名为transform的空间，其空间处于空间mindspore下
|
||||
//对实现ApplyAdam的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(ApplyAdam)
|
||||
DECLARE_OP_USE_OUTPUT(ApplyAdam)
|
||||
//对实现ApplyAdamD的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(ApplyAdamD)
|
||||
DECLARE_OP_USE_OUTPUT(ApplyAdamD)
|
||||
//对实现ApplyAdagradD的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(ApplyAdagradD)
|
||||
DECLARE_OP_USE_OUTPUT(ApplyAdagradD)
|
||||
//对实现ApplyAdagradV2D的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(ApplyAdagradV2D)
|
||||
DECLARE_OP_USE_OUTPUT(ApplyAdagradV2D)
|
||||
//对实现ApplyAddSignD的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(ApplyAddSignD)
|
||||
DECLARE_OP_USE_OUTPUT(ApplyAddSignD)
|
||||
//对实现SparseApplyAdagradV2D的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(SparseApplyAdagradV2D)
|
||||
DECLARE_OP_USE_OUTPUT(SparseApplyAdagradV2D)
|
||||
//对实现DataFormatDimMap的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(DataFormatDimMap)
|
||||
DECLARE_OP_USE_OUTPUT(DataFormatDimMap)
|
||||
//对实现ApplyAdadeltaD的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(ApplyAdadeltaD)
|
||||
DECLARE_OP_USE_OUTPUT(ApplyAdadeltaD)
|
||||
//对实现ApplyAdaMaxD的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(ApplyAdaMaxD)
|
||||
DECLARE_OP_USE_OUTPUT(ApplyAdaMaxD)
|
||||
//对实现ApplyGradientDescent的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(ApplyGradientDescent)
|
||||
DECLARE_OP_USE_OUTPUT(ApplyGradientDescent)
|
||||
//对实现ApplyPowerSignD的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(ApplyPowerSignD)
|
||||
DECLARE_OP_USE_OUTPUT(ApplyPowerSignD)
|
||||
//对实现ApplyProximalGradientDescent的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(ApplyProximalGradientDescent)
|
||||
DECLARE_OP_USE_OUTPUT(ApplyProximalGradientDescent)
|
||||
//对实现SGD的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(SGD)
|
||||
DECLARE_OP_USE_OUTPUT(SGD)
|
||||
//对实现ApplyMomentum的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(ApplyMomentum)
|
||||
DECLARE_OP_USE_OUTPUT(ApplyMomentum)
|
||||
//对实现SparseApplyAdagradD的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(SparseApplyAdagradD)
|
||||
DECLARE_OP_USE_OUTPUT(SparseApplyAdagradD)
|
||||
//对实现ApplyProximalAdagradD的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(ApplyProximalAdagradD)
|
||||
DECLARE_OP_USE_OUTPUT(ApplyProximalAdagradD)
|
||||
//对实现SparseApplyProximalAdagradD的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(SparseApplyProximalAdagradD)
|
||||
DECLARE_OP_USE_OUTPUT(SparseApplyProximalAdagradD)
|
||||
//对实现LarsV2Update的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(LarsV2Update)
|
||||
DECLARE_OP_USE_OUTPUT(LarsV2Update)
|
||||
//对实现ApplyFtrl的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(ApplyFtrl)
|
||||
DECLARE_OP_USE_OUTPUT(ApplyFtrl)
|
||||
//对实现SparseApplyFtrlD的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(SparseApplyFtrlD)
|
||||
DECLARE_OP_USE_OUTPUT(SparseApplyFtrlD)
|
||||
//对实现SparseApplyFtrlV2D的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(SparseApplyFtrlV2D)
|
||||
DECLARE_OP_USE_OUTPUT(SparseApplyFtrlV2D)
|
||||
//对实现ApplyRMSPropD的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(ApplyRMSPropD)
|
||||
DECLARE_OP_USE_INPUT_ATTR(ApplyRMSPropD)
|
||||
DECLARE_OP_USE_OUTPUT(ApplyRMSPropD)
|
||||
//对实现ApplyCenteredRMSProp的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(ApplyCenteredRMSProp)
|
||||
DECLARE_OP_USE_OUTPUT(ApplyCenteredRMSProp)
|
||||
} // namespace mindspore::transform
|
||||
#endif // MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_NN_TRAINING_OPS_DECLARE_H_
|
||||
|
|
@ -0,0 +1,204 @@
|
|||
/**
|
||||
* Copyright 2019-2021 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#include "transform/graph_ir/op_declare/nonlinear_fuc_ops_declare.h"//按照路径寻找以下文件，导入到本文件
|
||||
|
||||
namespace mindspore::transform {//创建名为transform的空间，其空间处于空间mindspore下
|
||||
// Relu
|
||||
INPUT_MAP(Relu) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入Relu对应空间内并用input_map指针保存
|
||||
ATTR_MAP(Relu) = EMPTY_ATTR_MAP;//将空变量存入Relu对应空间并用attr_map_指针保存
|
||||
OUTPUT_MAP(Relu) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入Relu对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(Relu, prim::kPrimRelu->name(), ADPT_DESC(Relu))//构造指向Relu的指针并储存，创建结构体RegAdptDescRelu
|
||||
|
||||
// ReluV2
|
||||
INPUT_MAP(ReluV2) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入ReluV2对应空间内并用input_map指针保存
|
||||
ATTR_MAP(ReluV2) = EMPTY_ATTR_MAP;//将空变量存入ReluV2对应空间并用attr_map_指针保存
|
||||
OUTPUT_MAP(ReluV2) = {{0, OUTPUT_DESC(y)}, {1, OUTPUT_DESC(mask)}};
|
||||
//将变量y、mask处理并存入对应OutputDesc结构体的相应变量中，存入ReluV2对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(ReluV2, kNameReluV2, ADPT_DESC(ReluV2))//构造指向ReluV2的指针并储存，创建结构体RegAdptDescReluV2
|
||||
|
||||
// Elu
|
||||
INPUT_MAP(Elu) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入Elu对应空间内并用input_map指针保存
|
||||
ATTR_MAP(Elu) = {{"alpha", ATTR_DESC(alpha, AnyTraits<float>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
// 存入Elu对应空间并用attr_map_指针保存
|
||||
// 其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
OUTPUT_MAP(Elu) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入Elu对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(Elu, kNameElu, ADPT_DESC(Elu))//构造指向Elu的指针并储存，创建结构体RegAdptDescElu
|
||||
|
||||
// EluGrad
|
||||
INPUT_MAP(EluGrad) = {{1, INPUT_DESC(grads)}, {2, INPUT_DESC(activations)}};
|
||||
//将变量grads、activations处理并存入对应InputDesc结构体的相应变量中，存入EluGrad对应空间内并用input_map指针保存
|
||||
ATTR_MAP(EluGrad) = EMPTY_ATTR_MAP;//将空变量存入EluGrad对应空间并用attr_map_指针保存
|
||||
OUTPUT_MAP(EluGrad) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入EluGrad对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(EluGrad, kNameEluGrad, ADPT_DESC(EluGrad))//构造指向EluGrad的指针并储存，创建结构体RegAdptDescEluGrad
|
||||
|
||||
// PRelu
|
||||
INPUT_MAP(PRelu) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(weight)}};
|
||||
//将变量x、weight处理并存入对应InputDesc结构体的相应变量中，存入PRelu对应空间内并用input_map指针保存
|
||||
ATTR_MAP(PRelu) = EMPTY_ATTR_MAP;//将空变量存入PRelu对应空间并用attr_map_指针保存
|
||||
OUTPUT_MAP(PRelu) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入PRelu对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(PRelu, kNamePrelu, ADPT_DESC(PRelu))//构造指向PRelu的指针并储存，创建结构体RegAdptDescPRelu
|
||||
|
||||
// PReluGrad
|
||||
INPUT_MAP(PReluGrad) = {{1, INPUT_DESC(grads)}, {2, INPUT_DESC(features)}, {3, INPUT_DESC(weights)}};
|
||||
//将变量grads、features、weights处理并存入对应InputDesc结构体的相应变量中，存入PReluGrad对应空间内并用input_map指针保存
|
||||
ATTR_MAP(PReluGrad) = EMPTY_ATTR_MAP;//将空变量存入PReluGrad对应空间并用attr_map_指针保存
|
||||
OUTPUT_MAP(PReluGrad) = {{0, OUTPUT_DESC(dx)}, {1, OUTPUT_DESC(da)}};
|
||||
//将变量dx、da处理并存入对应OutputDesc结构体的相应变量中，存入PReluGrad对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(PReluGrad, kNamePreluGrad, ADPT_DESC(PReluGrad))//构造指向PReluGrad的指针并储存，创建结构体RegAdptDescPReluGrad
|
||||
|
||||
// Selu
|
||||
INPUT_MAP(Selu) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入Selu对应空间内并用input_map指针保存
|
||||
ATTR_MAP(Selu) = EMPTY_ATTR_MAP;//将空变量存入Selu对应空间并用attr_map_指针保存
|
||||
OUTPUT_MAP(Selu) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入Selu对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(Selu, kNameSelu, ADPT_DESC(Selu))//构造指向Selu的指针并储存，创建结构体RegAdptDescSelu
|
||||
|
||||
// Sigmoid
|
||||
INPUT_MAP(Sigmoid) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入Sigmoid对应空间内并用input_map指针保存
|
||||
ATTR_MAP(Sigmoid) = EMPTY_ATTR_MAP;//将空变量存入Sigmoid对应空间并用attr_map_指针保存
|
||||
OUTPUT_MAP(Sigmoid) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入Sigmoid对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(Sigmoid, kNameSigmoid, ADPT_DESC(Sigmoid))//构造指向Sigmoid的指针并储存，创建结构体RegAdptDescSigmoid
|
||||
|
||||
// SigmoidGrad
|
||||
INPUT_MAP(SigmoidGrad) = {{1, INPUT_DESC(y)}, {2, INPUT_DESC(dy)}};
|
||||
//将变量y、dy处理并存入对应InputDesc结构体的相应变量中，存入SigmoidGrad对应空间内并用input_map指针保存
|
||||
ATTR_MAP(SigmoidGrad) = EMPTY_ATTR_MAP;//将空变量存入SigmoidGrad对应空间并用attr_map_指针保存
|
||||
OUTPUT_MAP(SigmoidGrad) = {{0, OUTPUT_DESC(z)}};//将变量z处理并存入对应OutputDesc结构体的相应变量中，存入SigmoidGrad对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(SigmoidGrad, kNameSigmoidGrad, ADPT_DESC(SigmoidGrad))//构造指向SigmoidGrad的指针并储存，创建结构体RegAdptDescSigmoidGrad
|
||||
|
||||
// HardSwish
|
||||
INPUT_MAP(HardSwish) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入HardSwish对应空间内并用input_map指针保存
|
||||
ATTR_MAP(HardSwish) = EMPTY_ATTR_MAP;//将空变量存入HardSwish对应空间并用attr_map_指针保存
|
||||
OUTPUT_MAP(HardSwish) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入HardSwish对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(HardSwish, kNameHSwish, ADPT_DESC(HardSwish))//构造指向HardSwish的指针并储存，创建结构体RegAdptDescHardSwish
|
||||
|
||||
// HardSwishGrad
|
||||
INPUT_MAP(HardSwishGrad) = {{1, INPUT_DESC(grad)}, {2, INPUT_DESC(x)}};
|
||||
//将变量grad、x处理并存入对应InputDesc结构体的相应变量中，存入HardSwishGrad对应空间内并用input_map指针保存
|
||||
ATTR_MAP(HardSwishGrad) = EMPTY_ATTR_MAP;//将空变量存入HardSwishGrad对应空间并用attr_map_指针保存
|
||||
OUTPUT_MAP(HardSwishGrad) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入HardSwishGrad对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(HardSwishGrad, kNameHSwishGrad, ADPT_DESC(HardSwishGrad))//构造指向HardSwishGrad的指针并储存，创建结构体RegAdptDescHardSwishGrad
|
||||
|
||||
// HSigmoid
|
||||
INPUT_MAP(HardSigmoid) = {{1, INPUT_DESC(input_x)}};//将变量input_x处理并存入对应InputDesc结构体的相应变量中，存入HardSigmoid对应空间内并用input_map指针保存
|
||||
ATTR_MAP(HardSigmoid) = {{"alpha", ATTR_DESC(alpha, AnyTraits<float>())},
|
||||
{"beta", ATTR_DESC(beta, AnyTraits<float>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
// 存入HardSigmoid对应空间并用attr_map_指针保存
|
||||
// 其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
OUTPUT_MAP(HardSigmoid) = {{0, OUTPUT_DESC(output_y)}};//将变量output_y处理并存入对应OutputDesc结构体的相应变量中，存入HardSigmoid对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(HardSigmoid, kNameHSigmoid, ADPT_DESC(HardSigmoid))//构造指向HardSigmoid的指针并储存，创建结构体RegAdptDescHardSigmoid
|
||||
|
||||
// Relu6
|
||||
INPUT_MAP(Relu6) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入Relu6对应空间内并用input_map指针保存
|
||||
ATTR_MAP(Relu6) = EMPTY_ATTR_MAP;//将空变量存入Relu6对应空间并用attr_map_指针保存
|
||||
OUTPUT_MAP(Relu6) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入Relu6对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(Relu6, kNameReLU6, ADPT_DESC(Relu6))//构造指向Relu6的指针并储存，创建结构体RegAdptDescRelu6
|
||||
|
||||
// Relu6Grad
|
||||
INPUT_MAP(Relu6Grad) = {{1, INPUT_DESC(gradients)}, {2, INPUT_DESC(features)}};
|
||||
//将变量gradients、features处理并存入对应InputDesc结构体的相应变量中，存入Relu6Grad对应空间内并用input_map指针保存
|
||||
ATTR_MAP(Relu6Grad) = EMPTY_ATTR_MAP;//将空变量存入Relu6Grad对应空间并用attr_map_指针保存
|
||||
OUTPUT_MAP(Relu6Grad) = {{0, OUTPUT_DESC(backprops)}};//将变量backprops处理并存入对应OutputDesc结构体的相应变量中，存入Relu6Grad对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(Relu6Grad, kNameReLU6Grad, ADPT_DESC(Relu6Grad))//构造指向Relu6Grad的指针并储存，创建结构体RegAdptDescRelu6Grad
|
||||
|
||||
// Softsign
|
||||
INPUT_MAP(Softsign) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入Softsign对应空间内并用input_map指针保存
|
||||
ATTR_MAP(Softsign) = EMPTY_ATTR_MAP;//将空变量存入Softsign对应空间并用attr_map_指针保存
|
||||
OUTPUT_MAP(Softsign) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入Softsign对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(Softsign, kNameSoftsign, ADPT_DESC(Softsign))//构造指向Softsign的指针并储存，创建结构体RegAdptDescSoftsign
|
||||
|
||||
// Softplus
|
||||
INPUT_MAP(Softplus) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入Softplus对应空间内并用input_map指针保存
|
||||
ATTR_MAP(Softplus) = EMPTY_ATTR_MAP;//将空变量存入Softplus对应空间并用attr_map_指针保存
|
||||
OUTPUT_MAP(Softplus) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入Softplus对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(Softplus, kNameSoftplus, ADPT_DESC(Softplus))//构造指向Softplus的指针并储存，创建结构体RegAdptDescSoftplus
|
||||
|
||||
// SoftplusGrad
|
||||
INPUT_MAP(SoftplusGrad) = {{1, INPUT_DESC(gradients)}, {2, INPUT_DESC(features)}};
|
||||
//将变量gradients、features处理并存入对应InputDesc结构体的相应变量中，存入SoftplusGrad对应空间内并用input_map指针保存
|
||||
ATTR_MAP(SoftplusGrad) = EMPTY_ATTR_MAP;//将空变量存入SoftplusGrad对应空间并用attr_map_指针保存
|
||||
OUTPUT_MAP(SoftplusGrad) = {{0, OUTPUT_DESC(backprops)}};
|
||||
//将变量backprops处理并存入对应OutputDesc结构体的相应变量中，存入SoftplusGrad对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(SoftplusGrad, kNameSoftplusGrad, ADPT_DESC(SoftplusGrad))//构造指向SoftplusGrad的指针并储存，创建结构体RegAdptDescSoftplusGrad
|
||||
|
||||
// ReluGrad
|
||||
INPUT_MAP(ReluGrad) = {{1, INPUT_DESC(gradients)}, {2, INPUT_DESC(features)}};
|
||||
//将变量gradients、features处理并存入对应InputDesc结构体的相应变量中，存入ReluGrad对应空间内并用input_map指针保存
|
||||
ATTR_MAP(ReluGrad) = EMPTY_ATTR_MAP;//将空变量存入ReluGrad对应空间并用attr_map_指针保存
|
||||
OUTPUT_MAP(ReluGrad) = {{0, OUTPUT_DESC(backprops)}};//将变量backprops处理并存入对应OutputDesc结构体的相应变量中，存入ReluGrad对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(ReluGrad, prim::kPrimReluGrad->name(), ADPT_DESC(ReluGrad))//构造指向ReluGrad的指针并储存，创建结构体RegAdptDescReluGrad
|
||||
|
||||
// ReluGradV2
|
||||
INPUT_MAP(ReluGradV2) = {{1, INPUT_DESC(gradients)}, {2, INPUT_DESC(mask)}};
|
||||
//将变量gradients、mask处理并存入对应InputDesc结构体的相应变量中，存入ReluGradV2对应空间内并用input_map指针保存
|
||||
ATTR_MAP(ReluGradV2) = EMPTY_ATTR_MAP;//将空变量存入ReluGradV2对应空间并用attr_map_指针保存
|
||||
OUTPUT_MAP(ReluGradV2) = {{0, OUTPUT_DESC(backprops)}};//将变量backprops处理并存入对应OutputDesc结构体的相应变量中，存入ReluGradV2对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(ReluGradV2, kNameReluGradV2, ADPT_DESC(ReluGradV2))//构造指向ReluGradV2的指针并储存，创建结构体RegAdptDescReluGradV2
|
||||
|
||||
// Tanh
|
||||
INPUT_MAP(Tanh) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入Tanh对应空间内并用input_map指针保存
|
||||
ATTR_MAP(Tanh) = EMPTY_ATTR_MAP;//将空变量存入ReluGradV2对应空间并用attr_map_指针保存
|
||||
OUTPUT_MAP(Tanh) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入Tanh对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(Tanh, prim::kPrimTanh->name(), ADPT_DESC(Tanh))//构造指向Tanh的指针并储存，创建结构体RegAdptDescTanh
|
||||
|
||||
// TanhGrad
|
||||
INPUT_MAP(TanhGrad) = {{1, INPUT_DESC(y)}, {2, INPUT_DESC(dy)}};
|
||||
//将变量y、dy处理并存入对应InputDesc结构体的相应变量中，存入TanhGrad对应空间内并用input_map指针保存
|
||||
ATTR_MAP(TanhGrad) = EMPTY_ATTR_MAP;//将空变量存入ReluGradV2对应空间并用attr_map_指针保存
|
||||
OUTPUT_MAP(TanhGrad) = {{0, OUTPUT_DESC(z)}};//将变量z处理并存入对应OutputDesc结构体的相应变量中，存入TanhGrad对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(TanhGrad, prim::kPrimTanhGrad->name(), ADPT_DESC(TanhGrad))//构造指向TanhGrad的指针并储存，创建结构体RegAdptDescTanhGrad
|
||||
|
||||
// Mish
|
||||
INPUT_MAP(Mish) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入Mish对应空间内并用input_map指针保存
|
||||
ATTR_MAP(Mish) = EMPTY_ATTR_MAP;//将空变量存入Mish对应空间并用attr_map_指针保存
|
||||
OUTPUT_MAP(Mish) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入Mish对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(Mish, kNameMish, ADPT_DESC(Mish))//构造指向Mish的指针并储存，创建结构体RegAdptDescMish
|
||||
|
||||
// GeLU
|
||||
INPUT_MAP(Gelu) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入Gelu对应空间内并用input_map指针保存
|
||||
ATTR_MAP(Gelu) = EMPTY_ATTR_MAP;//将空变量存入Gelu对应空间并用attr_map_指针保存
|
||||
OUTPUT_MAP(Gelu) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入Gelu对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(Gelu, prim::kPrimGeLU->name(), ADPT_DESC(Gelu))//构造指向Gelu的指针并储存，创建结构体RegAdptDescGelu
|
||||
|
||||
// GeLUGrad
|
||||
INPUT_MAP(GeluGrad) = {{1, INPUT_DESC(dy)}, {2, INPUT_DESC(x)}, {3, INPUT_DESC(y)}};
|
||||
//将变量dy、x、y处理并存入对应InputDesc结构体的相应变量中，存入GeluGrad对应空间内并用input_map指针保存
|
||||
ATTR_MAP(GeluGrad) = EMPTY_ATTR_MAP;//将空变量存入GeluGrad对应空间并用attr_map_指针保存
|
||||
OUTPUT_MAP(GeluGrad) = {{0, OUTPUT_DESC(z)}};//将变量z处理并存入对应OutputDesc结构体的相应变量中，存入GeluGrad对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(GeluGrad, prim::kPrimGeLUGrad->name(), ADPT_DESC(GeluGrad))//构造指向GeluGrad的指针并储存，创建结构体RegAdptDescGeluGrad
|
||||
|
||||
// FastGeLU
|
||||
INPUT_MAP(FastGelu) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入FastGelu对应空间内并用input_map指针保存
|
||||
ATTR_MAP(FastGelu) = EMPTY_ATTR_MAP;//将空变量存入FastGelu对应空间并用attr_map_指针保存
|
||||
OUTPUT_MAP(FastGelu) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入FastGelu对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(FastGelu, prim::kPrimFastGeLU->name(), ADPT_DESC(FastGelu))//构造指向FastGelu的指针并储存，创建结构体RegAdptDescFastGelu
|
||||
|
||||
// FastGeLUGrad
|
||||
INPUT_MAP(FastGeluGrad) = {{1, INPUT_DESC(dy)}, {2, INPUT_DESC(x)}};
|
||||
//将变量dy、x处理并存入对应InputDesc结构体的相应变量中，存入FastGeluGrad对应空间内并用input_map指针保存
|
||||
ATTR_MAP(FastGeluGrad) = EMPTY_ATTR_MAP;//将空变量存入FastGeluGrad对应空间并用attr_map_指针保存
|
||||
OUTPUT_MAP(FastGeluGrad) = {{0, OUTPUT_DESC(z)}};//将变量z处理并存入对应OutputDesc结构体的相应变量中，存入FastGeluGrad对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(FastGeluGrad, prim::kPrimFastGeLUGrad->name(), ADPT_DESC(FastGeluGrad))
|
||||
//构造指向FastGeluGrad的指针并储存，创建结构体RegAdptDescFastGeluGrad
|
||||
|
||||
// LeakyRelu
|
||||
INPUT_MAP(LeakyRelu) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入LeakyRelu对应空间内并用input_map指针保存
|
||||
ATTR_MAP(LeakyRelu) = {{"alpha", ATTR_DESC(negative_slope, AnyTraits<float>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
// 存入LeakyRelu对应空间并用attr_map_指针保存
|
||||
// 其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
OUTPUT_MAP(LeakyRelu) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入LeakyRelu对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(LeakyRelu, prim::kPrimLeakyRelu->name(), ADPT_DESC(LeakyRelu))//构造指向LeakyRelu的指针并储存，创建结构体RegAdptDescLeakyRelu
|
||||
} // namespace mindspore::transform
|
||||
|
|
@ -0,0 +1,108 @@
|
|||
/**
|
||||
* Copyright 2019-2021 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_NONLINEAR_FUC_OPS_DECLARE_H_//判断宏是否被定义，如果宏没有定义，则编译下面代码
|
||||
#define MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_NONLINEAR_FUC_OPS_DECLARE_H_//定义预处理宏_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_NONLINEAR_FUC_OPS_DECLARE_H_
|
||||
|
||||
#include <string>//导入标准库中的字符串类及相关操作
|
||||
#include "utils/hash_map.h"//按照路径寻找以下文件，导入到本文件,以下同理
|
||||
#include "ops/nonlinear_fuc_ops.h"
|
||||
#include "transform/graph_ir/op_declare/op_declare_macro.h"
|
||||
|
||||
namespace mindspore::transform {//创建名为transform的空间，其空间处于空间mindspore下
|
||||
//对实现ReluGrad的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(ReluGrad)
|
||||
DECLARE_OP_USE_OUTPUT(ReluGrad)
|
||||
//对实现ReluGradV2的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(ReluGradV2)
|
||||
DECLARE_OP_USE_OUTPUT(ReluGradV2)
|
||||
//对实现Relu6的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(Relu6)
|
||||
DECLARE_OP_USE_OUTPUT(Relu6)
|
||||
//对实现Relu6Grad的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(Relu6Grad)
|
||||
DECLARE_OP_USE_OUTPUT(Relu6Grad)
|
||||
//对实现Softsign的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(Softsign)
|
||||
DECLARE_OP_USE_OUTPUT(Softsign)
|
||||
//对实现Softplus的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(Softplus)
|
||||
DECLARE_OP_USE_OUTPUT(Softplus)
|
||||
//对实现SoftplusGrad的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(SoftplusGrad)
|
||||
DECLARE_OP_USE_OUTPUT(SoftplusGrad)
|
||||
//对实现Tanh的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(Tanh)
|
||||
DECLARE_OP_USE_OUTPUT(Tanh)
|
||||
//对实现TanhGrad的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(TanhGrad)
|
||||
DECLARE_OP_USE_OUTPUT(TanhGrad)
|
||||
//对实现Mish的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(Mish)
|
||||
DECLARE_OP_USE_OUTPUT(Mish)
|
||||
//对实现Gelu的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(Gelu)
|
||||
DECLARE_OP_USE_OUTPUT(Gelu)
|
||||
//对实现GeluGrad的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(GeluGrad)
|
||||
DECLARE_OP_USE_OUTPUT(GeluGrad)
|
||||
//对实现FastGelu的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(FastGelu)
|
||||
DECLARE_OP_USE_OUTPUT(FastGelu)
|
||||
//对实现FastGeluGrad的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(FastGeluGrad)
|
||||
DECLARE_OP_USE_OUTPUT(FastGeluGrad)
|
||||
//对实现Relu的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(Relu)
|
||||
DECLARE_OP_USE_OUTPUT(Relu)
|
||||
//对实现ReluV2的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(ReluV2)
|
||||
DECLARE_OP_USE_OUTPUT(ReluV2)
|
||||
//对实现PRelu的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(PRelu)
|
||||
DECLARE_OP_USE_OUTPUT(PRelu)
|
||||
//对实现Elu的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(Elu)
|
||||
DECLARE_OP_USE_OUTPUT(Elu)
|
||||
//对实现EluGrad的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(EluGrad)
|
||||
DECLARE_OP_USE_OUTPUT(EluGrad)
|
||||
//对实现PReluGrad的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(PReluGrad)
|
||||
DECLARE_OP_USE_OUTPUT(PReluGrad)
|
||||
//对实现Selu的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(Selu)
|
||||
DECLARE_OP_USE_OUTPUT(Selu)
|
||||
//对实现Sigmoid的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(Sigmoid)
|
||||
DECLARE_OP_USE_OUTPUT(Sigmoid)
|
||||
//对实现HardSwish的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(HardSwish)
|
||||
DECLARE_OP_USE_OUTPUT(HardSwish)
|
||||
//对实现HardSwishGrad的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(HardSwishGrad)
|
||||
DECLARE_OP_USE_OUTPUT(HardSwishGrad)
|
||||
//对实现HardSigmoid的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(HardSigmoid)
|
||||
DECLARE_OP_USE_OUTPUT(HardSigmoid)
|
||||
//对实现SigmoidGrad的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(SigmoidGrad)
|
||||
DECLARE_OP_USE_OUTPUT(SigmoidGrad)
|
||||
//对实现LeakyRelu的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(LeakyRelu)
|
||||
DECLARE_OP_USE_OUTPUT(LeakyRelu)
|
||||
} // namespace mindspore::transform
|
||||
#endif // MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_NONLINEAR_FUC_OPS_DECLARE_H_
|
||||
|
|
@ -0,0 +1,43 @@
|
|||
/**
|
||||
* Copyright 2019 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#include "transform/graph_ir/op_declare/npu_loss_scale_ops_declare.h"//按照路径寻找以下文件，导入到本文件
|
||||
|
||||
namespace mindspore::transform {//创建名为transform的空间，其空间处于空间mindspore下
|
||||
// NPUGetFloatStatus
|
||||
INPUT_MAP(NPUGetFloatStatus) = {{1, INPUT_DESC(addr)}};//将变量addr处理并存入对应InputDesc结构体的相应变量中，存入NPUGetFloatStatus对应空间内并用input_map指针保存
|
||||
OUTPUT_MAP(NPUGetFloatStatus) = {{0, OUTPUT_DESC(data)}};
|
||||
//将变量data处理并存入对应OutputDesc结构体的相应变量中，存入NPUGetFloatStatus对应空间内并用output_map_指针保存
|
||||
ATTR_MAP(NPUGetFloatStatus) = EMPTY_ATTR_MAP;//将空变量存入NPUGetFloatStatus对应空间并用attr_map_指针保存
|
||||
REG_ADPT_DESC(NPUGetFloatStatus, kNameNPUGetFloatStatus, ADPT_DESC(NPUGetFloatStatus))
|
||||
//构造指向NPUGetFloatStatus的指针并储存，创建结构体RegAdptDescNPUGetFloatStatus
|
||||
|
||||
// NPUAllocFloatStatus
|
||||
INPUT_MAP(NPUAllocFloatStatus) = EMPTY_INPUT_MAP;//将空变量存入NPUAllocFloatStatus对应空间内并用input_map指针保存
|
||||
ATTR_MAP(NPUAllocFloatStatus) = EMPTY_ATTR_MAP;//将空变量存入NPUAllocFloatStatus对应空间并用attr_map_指针保存
|
||||
OUTPUT_MAP(NPUAllocFloatStatus) = {{0, OUTPUT_DESC(data)}};
|
||||
//将变量data处理并存入对应OutputDesc结构体的相应变量中，存入NPUAllocFloatStatus对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(NPUAllocFloatStatus, kNameNPUAllocFloatStatus, ADPT_DESC(NPUAllocFloatStatus))
|
||||
//构造指向NPUAllocFloatStatus的指针并储存，创建结构体RegAdptDescNPUAllocFloatStatus
|
||||
|
||||
// NPUClearFloatStatus
|
||||
INPUT_MAP(NPUClearFloatStatus) = {{1, INPUT_DESC(addr)}};//将变量addr处理并存入对应InputDesc结构体的相应变量中，存入NPUClearFloatStatus对应空间内并用input_map指针保存
|
||||
OUTPUT_MAP(NPUClearFloatStatus) = {{0, OUTPUT_DESC(data)}};
|
||||
//将变量data处理并存入对应OutputDesc结构体的相应变量中，存入NPUClearFloatStatus对应空间内并用output_map_指针保存
|
||||
ATTR_MAP(NPUClearFloatStatus) = EMPTY_ATTR_MAP;//将空变量存入NPUClearFloatStatus对应空间并用attr_map_指针保存
|
||||
REG_ADPT_DESC(NPUClearFloatStatus, kNameNPUClearFloatStatus, ADPT_DESC(NPUClearFloatStatus))
|
||||
//构造指向NPUClearFloatStatus的指针并储存，创建结构体RegAdptDescNPUClearFloatStatus
|
||||
} // namespace mindspore::transform
|
||||
|
|
@ -0,0 +1,37 @@
|
|||
/**
|
||||
* Copyright 2019-2021 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_NPU_LOSS_SCALE_OPS_DECLARE_H_//判断宏是否被定义，如果宏没有定义，则编译下面代码
|
||||
#define MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_NPU_LOSS_SCALE_OPS_DECLARE_H_
|
||||
//定义预处理宏_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_NPU_LOSS_SCALE_OPS_DECLARE_H_
|
||||
|
||||
#include <string>//导入标准库中的字符串类及相关操作
|
||||
#include "utils/hash_map.h"//按照路径寻找以下文件，导入到本文件,以下同理
|
||||
#include "transform/graph_ir/op_declare/op_declare_macro.h"
|
||||
#include "ops/npu_loss_scale_ops.h"
|
||||
|
||||
namespace mindspore::transform {//创建名为transform的空间，其空间处于空间mindspore下
|
||||
//对实现NPUGetFloatStatus的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(NPUGetFloatStatus)
|
||||
DECLARE_OP_USE_OUTPUT(NPUGetFloatStatus)
|
||||
//对实现NPUAllocFloatStatus的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(NPUAllocFloatStatus)
|
||||
DECLARE_OP_USE_OUTPUT(NPUAllocFloatStatus)
|
||||
//对实现NPUClearFloatStatus的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(NPUClearFloatStatus)
|
||||
DECLARE_OP_USE_OUTPUT(NPUClearFloatStatus)
|
||||
} // namespace mindspore::transform
|
||||
#endif // MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_NPU_LOSS_SCALE_OPS_DECLARE_H_
|
||||
|
|
@ -0,0 +1,205 @@
|
|||
/**
|
||||
* Copyright 2019-2021 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_ADAPTER_BASE_H_
|
||||
#define MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_ADAPTER_BASE_H_
|
||||
|
||||
#include <string>
|
||||
#include <memory>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
#include <sstream>
|
||||
|
||||
#include "utils/hash_map.h"
|
||||
#include "include/transform/graph_ir/util.h"
|
||||
#include "ir/anf.h"
|
||||
#include "ir/primitive.h"
|
||||
#include "ir/value.h"
|
||||
#include "include/transform/graph_ir/types.h"
|
||||
#include "graph/operator_reg.h"
|
||||
#include "external/ge/ge_api.h"
|
||||
#include "graph/tensor.h"
|
||||
|
||||
namespace ge {
|
||||
class CustomOperator : public Operator {
|
||||
public:
|
||||
CustomOperator(const string &name, const string &type) : Operator(name, type) {}
|
||||
|
||||
~CustomOperator() override{};
|
||||
|
||||
void CustomInputRegister(const string &name) { Operator::InputRegister(name); }
|
||||
|
||||
void CustomOutputRegister(const string &name) { Operator::OutputRegister(name); }
|
||||
|
||||
void CustomInferFuncRegister(const std::function<graphStatus(Operator &)> &func) {
|
||||
Operator::InferFuncRegister(func);
|
||||
}
|
||||
};
|
||||
} // namespace ge
|
||||
|
||||
namespace mindspore {
|
||||
namespace transform {
|
||||
using CusOperatorPtr = std::shared_ptr<ge::CustomOperator>;
|
||||
using CustomOperator = ge::CustomOperator;
|
||||
using AttrFunc = std::function<void(OperatorPtr, ValuePtr)>;
|
||||
using OutputFunc = std::function<OutHandler(OperatorPtr)>;
|
||||
using InputOpFunc = std::function<void(OperatorPtr, OperatorPtr)>;
|
||||
using InputHandleFunc = std::function<void(OperatorPtr, OutHandler)>;
|
||||
using CreateDynInputOpFunc = std::function<void(OperatorPtr, unsigned int)>;
|
||||
using DynInputOpFunc = std::function<void(OperatorPtr, unsigned int, OperatorPtr)>;
|
||||
using DynInputHandleFunc = std::function<void(OperatorPtr, unsigned int, OutHandler)>;
|
||||
using UpdateOutputDescFunc = std::function<void(OperatorPtr, GeTensorDesc)>;
|
||||
using CreateDynOutputOpFunc = std::function<void(OperatorPtr, unsigned int)>;
|
||||
using CreateDynSubGraphFunc = std::function<void(OperatorPtr, unsigned int)>;
|
||||
using DynSubGraphFunc = std::function<void(OperatorPtr, unsigned int, DfGraphPtr)>;
|
||||
|
||||
//定义结构体AttrDesc，包含string型变量name和AttrFunc型变量set_attr
|
||||
struct AttrDesc {
|
||||
std::string name;
|
||||
AttrFunc set_attr;
|
||||
};
|
||||
|
||||
/*定义结构体InputDesc，包含string型变量name;
|
||||
InputOpFunc型变量set_op;
|
||||
InputHandleFunc型变量set_handle;
|
||||
UpdateOutputDescFunc型变量update_input_desc;
|
||||
*/
|
||||
struct InputDesc {
|
||||
std::string name;
|
||||
InputOpFunc set_op;
|
||||
InputHandleFunc set_handle;
|
||||
UpdateOutputDescFunc update_input_desc;
|
||||
};
|
||||
/*定义结构体DynInputDesc，包含string型变量name;
|
||||
CreateDynInputOpFunc型变量create_dyn_input;
|
||||
DynInputOpFunc型变量set_op;
|
||||
DynInputHandleFunc型变量set_handle;
|
||||
*/
|
||||
struct DynInputDesc {
|
||||
std::string name;
|
||||
CreateDynInputOpFunc create_dyn_input;
|
||||
DynInputOpFunc set_op;
|
||||
DynInputHandleFunc set_handle;
|
||||
};
|
||||
/*定义结构体DynSubGraphDesc，包含string型变量name;
|
||||
CreateDynSubGraphFunc型变量create_dyn_subgraph;
|
||||
DynSubGraphFunc型变量set_subgraph;
|
||||
*/
|
||||
struct DynSubGraphDesc {
|
||||
std::string name;
|
||||
CreateDynSubGraphFunc create_dyn_subgraph;
|
||||
DynSubGraphFunc set_subgraph;
|
||||
};
|
||||
/*定义结构体OutputDesc，包含string型变量name;
|
||||
UpdateOutputDescFunc型变量update_out_desc;
|
||||
*/
|
||||
struct OutputDesc {
|
||||
std::string name;
|
||||
UpdateOutputDescFunc update_out_desc;
|
||||
};
|
||||
/*定义结构体DynOutputDesc，包含string型变量name;
|
||||
CreateDynOutputOpFunc型变量create_dyn_output;
|
||||
*/
|
||||
struct DynOutputDesc {
|
||||
std::string name;
|
||||
CreateDynOutputOpFunc create_dyn_output;
|
||||
};
|
||||
//创建BaseOpAdapter类
|
||||
/*
|
||||
创建并定义属于BaseOpAdapter类中的函数，其功能包括：生成OperatorPtr类型变量
|
||||
设置sub图表
|
||||
设置输入对象
|
||||
设置Attr
|
||||
获取输出对象
|
||||
更新输出数据
|
||||
获取输入的map
|
||||
获取输入的attrmap
|
||||
将attr添加到drawgraph
|
||||
从drawgraph获取attr
|
||||
清空attr的vect属性
|
||||
等功能，其具体的实例将在其他属于adapter部分的文件中进一步分析
|
||||
*/
|
||||
class BaseOpAdapter {
|
||||
public:
|
||||
virtual ~BaseOpAdapter() {}
|
||||
virtual OperatorPtr generate(const AnfNodePtr &anf) = 0;
|
||||
virtual OperatorPtr generate(const std::string &type) { return std::make_shared<ge::Operator>(type); }
|
||||
virtual int setSubgraph(const OperatorPtr &op, int index, const std::shared_ptr<std::vector<DfGraph>> &branches) = 0;
|
||||
virtual int setInput(const OperatorPtr &op, int index, const OperatorPtr &input) = 0;
|
||||
virtual int setInput(const OperatorPtr &op, int index, const OutHandler &handle) = 0;
|
||||
virtual int setInput(const OperatorPtr &op, int index,
|
||||
const std::shared_ptr<std::vector<OutHandler>> &handler_vec) = 0;
|
||||
virtual int setAttr(const OperatorPtr &op, const std::string &attrKey, const ValuePtr &attrValue) = 0;
|
||||
virtual int setAttr(const OperatorPtr &op, const PrimitivePtr &prim) = 0;
|
||||
virtual int setAttr(const OperatorPtr &op, const AnfNodePtr &node) = 0;
|
||||
virtual mindspore::HashMap<std::string, ValuePtr> GetExtraAttr() = 0;
|
||||
template <typename T, typename _ = typename std::enable_if<!std::is_base_of<Value, T>::value>::type>
|
||||
int setAttr(const OperatorPtr &op, const std::string &attrKey, const std::shared_ptr<T> &attrValue) {
|
||||
return setAttr(op, attrKey, MakeValue(attrValue));
|
||||
}
|
||||
template <typename T, typename _ = typename std::enable_if<!is_shared_ptr<T>::value>::type>
|
||||
int setAttr(const OperatorPtr &op, const std::string &attrKey, const T &attrValue) {
|
||||
return setAttr(op, attrKey, MakeValue(attrValue));
|
||||
}
|
||||
virtual OutHandler getOutput(const OperatorPtr &op, int index) = 0;
|
||||
virtual void updateOutputDesc(const OperatorPtr &op, const abstract::BaseShapePtr &shp, const TypePtr &type,
|
||||
const AnfNodePtr &node) = 0;
|
||||
virtual const mindspore::HashMap<int, InputDesc> &getInputMap() = 0;
|
||||
virtual const mindspore::HashMap<unsigned int, AttrDesc> &getInputAttrMap() = 0;
|
||||
virtual const mindspore::HashMap<int, DynInputDesc> &getDynInputMap() = 0;
|
||||
virtual const mindspore::HashMap<int, OutputDesc> &getOutputMap() = 0;
|
||||
virtual const mindspore::HashMap<int, DynSubGraphDesc> &getDynSubgraphMap() = 0;
|
||||
void AddAttrToDrawGraph(const std::string &attr_str) { attrs_vec_.push_back(attr_str); }
|
||||
const std::vector<std::string> &GetAttrsFromDrawGraph() const { return attrs_vec_; }
|
||||
void clearAttrVect() { attrs_vec_.clear(); }
|
||||
|
||||
private:
|
||||
std::vector<std::string> attrs_vec_;
|
||||
};
|
||||
|
||||
using OpAdapterPtr = std::shared_ptr<BaseOpAdapter>;
|
||||
|
||||
enum AttrType {
|
||||
ATTR_INT = 0,
|
||||
ATTR_FLOAT,
|
||||
ATTR_DOUBLE,
|
||||
ATTR_STRING,
|
||||
ATTR_TENSOR,
|
||||
ATTR_BOOL,
|
||||
ATTR_LIST_INT,
|
||||
ATTR_LIST_ANY_INT,
|
||||
ATTR_ENUM
|
||||
};
|
||||
|
||||
struct GeEnum {};
|
||||
struct TFType {};
|
||||
struct GEType {};
|
||||
|
||||
// declare Any type
|
||||
template <typename T>
|
||||
struct AnyTraits {
|
||||
using type = T;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct AnyTraits<int> {
|
||||
using type = int64_t;
|
||||
};
|
||||
|
||||
using ExtraAttr = mindspore::HashMap<std::string, ValuePtr>;
|
||||
} // namespace transform
|
||||
} // namespace mindspore
|
||||
#endif // MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_ADAPTER_BASE_H_
|
||||
|
|
@ -0,0 +1,79 @@
|
|||
/**
|
||||
* Copyright 2019 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_ADAPTER_DESC_H_
|
||||
#define MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_ADAPTER_DESC_H_
|
||||
|
||||
#include <memory>
|
||||
#include "transform/graph_ir/op_adapter.h"
|
||||
|
||||
namespace mindspore {
|
||||
namespace transform {
|
||||
//创建OpAdapterDesc类
|
||||
/*
|
||||
定义OpAdapterDesc，根据参数不同对应不同的方法，包括根据输入改对象的train_和infer_属性或令其为空
|
||||
*/
|
||||
class OpAdapterDesc {
|
||||
public:
|
||||
OpAdapterDesc() : train_(nullptr), infer_(nullptr) {}
|
||||
|
||||
OpAdapterDesc(const OpAdapterPtr &train, const OpAdapterPtr &infer) : train_(train), infer_(infer) {}
|
||||
|
||||
explicit OpAdapterDesc(const OpAdapterPtr &common) : train_(common), infer_(common) {}
|
||||
|
||||
OpAdapterDesc(const OpAdapterDesc &desc) {
|
||||
this->train_ = desc.train_;
|
||||
this->infer_ = desc.infer_;
|
||||
}
|
||||
|
||||
OpAdapterDesc(OpAdapterDesc &&desc) {
|
||||
this->train_ = desc.train_;
|
||||
this->infer_ = desc.infer_;
|
||||
desc.train_ = nullptr;
|
||||
desc.infer_ = nullptr;
|
||||
}
|
||||
|
||||
~OpAdapterDesc() = default;
|
||||
|
||||
OpAdapterPtr Get(bool train) const { return train ? train_ : infer_; }
|
||||
//定义名为operator的函数返回值类型为OpAdapterDesc，主要执行train_和infer_变量的修改
|
||||
OpAdapterDesc &operator=(const OpAdapterDesc &desc) {
|
||||
if (this != &desc) {
|
||||
this->train_ = desc.train_;
|
||||
this->infer_ = desc.infer_;
|
||||
}
|
||||
return *this;
|
||||
}
|
||||
//定义名为operator的函数返回值类型为OpAdapterDesc，主要执行train_和infer_变量的修改以及属于desc的train_和infer_变量的清空
|
||||
OpAdapterDesc &operator=(OpAdapterDesc &&desc) {
|
||||
if (this != &desc) {
|
||||
this->train_ = desc.train_;
|
||||
this->infer_ = desc.infer_;
|
||||
desc.train_ = nullptr;
|
||||
desc.infer_ = nullptr;
|
||||
}
|
||||
return *this;
|
||||
}
|
||||
//定义私有化OpAdapterPtr类型变量train_和infer_
|
||||
private:
|
||||
OpAdapterPtr train_;
|
||||
OpAdapterPtr infer_;
|
||||
};
|
||||
|
||||
using OpAdapterDescPtr = std::shared_ptr<OpAdapterDesc>;
|
||||
} // namespace transform
|
||||
} // namespace mindspore
|
||||
#endif // MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_ADAPTER_DESC_H_
|
||||
|
|
@ -0,0 +1,36 @@
|
|||
/**
|
||||
* Copyright 2019-2021 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
#include "include/transform/graph_ir/op_adapter_map.h"
|
||||
#include <memory>
|
||||
#include "graph/operator.h"
|
||||
#include "transform/graph_ir/op_adapter_desc.h"
|
||||
|
||||
namespace mindspore {
|
||||
namespace transform {
|
||||
//定义adpt_map_存储的数据
|
||||
namespace {
|
||||
mindspore::HashMap<std::string, OpAdapterDescPtr> adpt_map_ = {
|
||||
{kNameCustomOp, std::make_shared<OpAdapterDesc>(std::make_shared<OpAdapter<Operator>>())}};
|
||||
} // namespace
|
||||
//分别定义指针cus_input_map_和cus_output_map_
|
||||
template <>
|
||||
mindspore::HashMap<std::string, mindspore::HashMap<int, std::string>> OpAdapter<ge::Operator>::cus_input_map_{};
|
||||
template <>
|
||||
mindspore::HashMap<std::string, mindspore::HashMap<int, std::string>> OpAdapter<ge::Operator>::cus_output_map_{};
|
||||
//定义get的方法体为返回adpt_map_
|
||||
mindspore::HashMap<std::string, OpAdapterDescPtr> &OpAdapterMap::get() { return adpt_map_; }
|
||||
} // namespace transform
|
||||
} // namespace mindspore
|
||||
|
|
@ -0,0 +1,359 @@
|
|||
/**
|
||||
* Copyright 2019 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#include "transform/graph_ir/op_adapter_util.h"
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include <algorithm>
|
||||
|
||||
#include "include/common/utils/utils.h"
|
||||
#include "utils/check_convert_utils.h"
|
||||
#include "transform/graph_ir/op_adapter_base.h"
|
||||
#include "transform/graph_ir/io_format_map.h"
|
||||
|
||||
namespace mindspore {
|
||||
namespace transform {
|
||||
//定义名为ConvertAnyUtil的函数，返回值类型为GeTensor
|
||||
/*将me_tensor转化为ge_tensor
|
||||
检验其是否为空并返回ge_tensor*/
|
||||
GeTensor ConvertAnyUtil(const ValuePtr &value, const AnyTraits<mindspore::tensor::Tensor> &) {
|
||||
// To-DO the format may read from ME tensor//执行计划可能会从Me tensor中读取
|
||||
MS_EXCEPTION_IF_NULL(value);
|
||||
auto me_tensor = value->cast<MeTensorPtr>();
|
||||
auto ge_tensor = TransformUtil::ConvertTensor(me_tensor, kOpFormat_ND);
|
||||
return ge_tensor == nullptr ? GeTensor() : *ge_tensor;
|
||||
}
|
||||
//定义名为ConvertAnyUtil的函数，返回值类型为std::vector<int64_t>
|
||||
/*
|
||||
检验value是否为空，引入list
|
||||
检验name是否为pad，若是，则
|
||||
判断value与isa<ValueSequence>()是否相匹配，若不相配则输出EXCEPTION日志"Value should be ValueTuple, but got[*此处为value->type_name()]"
|
||||
定义vec执行value->cast<ValueSequencePtr>()
|
||||
重设list的size并对其开始的两位赋值
|
||||
*/
|
||||
std::vector<int64_t> ConvertAnyUtil(const ValuePtr &value, const std::string &name,
|
||||
const AnyTraits<std::vector<int64_t>>) {
|
||||
MS_EXCEPTION_IF_NULL(value);
|
||||
std::vector<int64_t> list;
|
||||
if (name == "pad") {
|
||||
if (!value->isa<ValueSequence>()) {
|
||||
MS_LOG(EXCEPTION) << "Value should be ValueTuple, but got" << value->type_name();
|
||||
}
|
||||
auto vec = value->cast<ValueSequencePtr>();
|
||||
list.resize(vec->value().size() + 2);
|
||||
list[0] = 1;
|
||||
list[1] = 1;
|
||||
(void)std::transform(vec->value().begin(), vec->value().end(), list.begin() + 2,
|
||||
[](const ValuePtr &val) { return static_cast<int64_t>(GetValue<int64_t>(val)); });
|
||||
}
|
||||
/*若name不为pad，则定义data获取value，设定size的值并将int转化为list*/
|
||||
else {
|
||||
int64_t data = GetValue<int64_t>(value);
|
||||
int size = 2; // 2 int in list//list中有两个int型
|
||||
list = TransformUtil::ConvertIntToList(data, size);
|
||||
}
|
||||
//返回list
|
||||
return list;
|
||||
}
|
||||
//定义名为ConvertAnyUtil的函数，返回值类型为std::string
|
||||
/*
|
||||
判断value是否为空
|
||||
定义vec储存value->cast<ValueTuplePtr>()所得数据并判断vec是否为空，若是，则输出EXCEPTION日志"not ValueTuplePtr"
|
||||
利用buffer储存value并用","隔开
|
||||
返回buffer.str()
|
||||
*/
|
||||
std::string ConvertAnyUtil(const ValuePtr &value, const AnyTraits<std::vector<int64_t>>, const AnyTraits<std::string>) {
|
||||
MS_EXCEPTION_IF_NULL(value);
|
||||
auto vec = value->cast<ValueTuplePtr>();
|
||||
if (vec == nullptr) {
|
||||
MS_LOG(EXCEPTION) << "not ValueTuplePtr";
|
||||
}
|
||||
std::ostringstream buffer;
|
||||
int i = 0;
|
||||
for (auto &it : vec->value()) {
|
||||
if (i != 0) {
|
||||
buffer << ",";
|
||||
}
|
||||
buffer << GetValue<int64_t>(it);
|
||||
i++;
|
||||
}
|
||||
return buffer.str();
|
||||
}
|
||||
//定义名为ConvertAnyUtil的函数，返回值类型为std::vector<float>
|
||||
/*
|
||||
判断value是否为空
|
||||
定义vec储存value->cast<ValueTuplePtr>()所得数据并判断vec是否为空，若是，则输出EXCEPTION日志"not ValueTuplePtr"
|
||||
定义list，并重设其size属性
|
||||
返回list
|
||||
*/
|
||||
std::vector<float> ConvertAnyUtil(const ValuePtr &value, const AnyTraits<std::vector<float>>, const AnyTraits<float>) {
|
||||
MS_EXCEPTION_IF_NULL(value);
|
||||
auto vec = value->cast<ValueTuplePtr>();
|
||||
if (vec == nullptr) {
|
||||
MS_LOG(EXCEPTION) << "not ValueTuplePtr";
|
||||
}
|
||||
std::vector<float> list;
|
||||
list.resize(vec->value().size());
|
||||
(void)std::transform(vec->value().begin(), vec->value().end(), list.begin(),
|
||||
[](const ValuePtr &val) { return static_cast<float>(GetValue<float>(val)); });
|
||||
return list;
|
||||
}
|
||||
//定义名为ConvertAnyUtil的函数，返回值类型为std::vector<int64_t>
|
||||
/*
|
||||
判断value是否为空
|
||||
定义vec储存value->cast<ValueTuplePtr>()所得数据并判断vec是否为空，若是，则输出EXCEPTION日志"not ValueTuplePtr"
|
||||
定义list，并重设其size属性
|
||||
判断format是否为kOpFormat_NHWC，若是则进一步判断list的size属性是否小于四，若是，则输出对应EXCEPTION日志
|
||||
若不是，则给list中的每个位置赋值
|
||||
返回list
|
||||
*/
|
||||
std::vector<int64_t> ConvertAnyUtil(const ValuePtr &value, const std::string &format,
|
||||
const AnyTraits<std::vector<i0nt64_t>>, const AnyTraits<int64_t>) {
|
||||
MS_EXCEPTION_IF_NULL(value);
|
||||
auto vec = value->cast<ValueTuplePtr>();
|
||||
if (vec == nullptr) {
|
||||
MS_LOG(EXCEPTION) << "not ValueTuplePtr";
|
||||
}
|
||||
std::vector<int64_t> list;
|
||||
list.resize(vec->value().size());
|
||||
(void)std::transform(vec->value().begin(), vec->value().end(), list.begin(),
|
||||
[](const ValuePtr &val) { return static_cast<int64_t>(GetValue<int64_t>(val)); });
|
||||
if (format == kOpFormat_NHWC) {
|
||||
if (list.size() < 4) {
|
||||
MS_LOG(EXCEPTION) << "The size of list is less than 4";
|
||||
} else {
|
||||
int64_t temp = list[1];
|
||||
list[1] = list[2];
|
||||
list[2] = list[3];
|
||||
list[3] = temp;
|
||||
}
|
||||
}
|
||||
return list;
|
||||
}
|
||||
|
||||
GeDataType ConvertAnyUtil(const ValuePtr &value, const AnyTraits<GEType>) {
|
||||
MS_EXCEPTION_IF_NULL(value);
|
||||
if (!value->isa<Type>()) {
|
||||
MS_LOG(EXCEPTION) << "error convert Value to TypePtr for value: " << value->ToString()
|
||||
<< ", type: " << value->type_name() << ", value should be a Typeptr";
|
||||
}
|
||||
auto type = value->cast<TypePtr>();
|
||||
MS_EXCEPTION_IF_NULL(type);
|
||||
TypeId me_type = type->type_id();
|
||||
if (kObjectTypeTensorType == me_type) {
|
||||
me_type = dyn_cast<TensorType>(type)->element()->type_id();
|
||||
}
|
||||
return TransformUtil::ConvertDataType(me_type);
|
||||
}
|
||||
|
||||
GeTensor VectorToTensorUtil(const ValuePtr &value) {
|
||||
// convert tuple or list to ge tensor, only supported one dim for now
|
||||
//将tuple或者list转化为ge_tensor，当前只支持一个dim
|
||||
MS_EXCEPTION_IF_NULL(value);
|
||||
auto vec = value->isa<ValueTuple>() ? value->cast<ValueTuplePtr>()->value() : value->cast<ValueListPtr>()->value();
|
||||
if (vec.empty()) {
|
||||
MS_LOG(WARNING) << "Convert a none tuple to an empty ge tensor";
|
||||
return GeTensor(GeTensorDesc(ge::Shape({0})));
|
||||
}
|
||||
MS_EXCEPTION_IF_NULL(vec[0]);
|
||||
if (vec[0]->isa<Int32Imm>()) {
|
||||
MS_LOG(INFO) << "convert value to tensor with data type = Int32";
|
||||
auto data = ConvertAnyUtil(value, AnyTraits<int32_t>(), AnyTraits<std::vector<int32_t>>());
|
||||
auto desc = TransformUtil::GetGeTensorDesc({static_cast<int>(vec.size())}, kNumberTypeInt32, kOpFormat_NCHW);
|
||||
if (desc == nullptr) {
|
||||
MS_LOG(EXCEPTION) << "Update conversion descriptor failed!";
|
||||
}
|
||||
return GeTensor(*desc, reinterpret_cast<uint8_t *>(data.data()), data.size() * sizeof(int32_t));
|
||||
} else if (vec[0]->isa<Int64Imm>()) {
|
||||
MS_LOG(INFO) << "convert value to tensor with data type = Int64";
|
||||
auto data = ConvertAnyUtil(value, AnyTraits<int64_t>(), AnyTraits<std::vector<int64_t>>());
|
||||
auto desc = TransformUtil::GetGeTensorDesc({static_cast<int>(vec.size())}, kNumberTypeInt64, kOpFormat_NCHW);
|
||||
if (desc == nullptr) {
|
||||
MS_LOG(EXCEPTION) << "Update conversion descriptor failed!";
|
||||
}
|
||||
return GeTensor(*desc, reinterpret_cast<uint8_t *>(data.data()), data.size() * sizeof(int64_t));
|
||||
} else if (vec[0]->isa<FP32Imm>()) {
|
||||
MS_LOG(INFO) << "convert value to tensor with data type = Float32";
|
||||
auto data = ConvertAnyUtil(value, AnyTraits<float>(), AnyTraits<std::vector<float>>());
|
||||
auto desc = TransformUtil::GetGeTensorDesc({static_cast<int>(vec.size())}, kNumberTypeFloat32, kOpFormat_NCHW);
|
||||
if (desc == nullptr) {
|
||||
MS_LOG(EXCEPTION) << "Update conversion descriptor failed!";
|
||||
}
|
||||
return GeTensor(*desc, reinterpret_cast<uint8_t *>(data.data()), data.size() * sizeof(float));
|
||||
} else if (vec[0]->isa<BoolImm>()) {
|
||||
MS_LOG(INFO) << "convert value to tensor with data type = Bool";
|
||||
// We use uint8_t to save bool type data
|
||||
//我们用uin8_t来保存bool类型的数据
|
||||
auto data = ConvertAnyUtil(value, AnyTraits<bool>(), AnyTraits<std::vector<uint8_t>>());
|
||||
auto desc = TransformUtil::GetGeTensorDesc({static_cast<int>(vec.size())}, kNumberTypeBool, kOpFormat_NCHW);
|
||||
if (desc == nullptr) {
|
||||
MS_LOG(EXCEPTION) << "Update conversion descriptor failed!";
|
||||
}
|
||||
return GeTensor(*desc, static_cast<uint8_t *>(data.data()), data.size() * sizeof(uint8_t));
|
||||
} else {
|
||||
MS_LOG(EXCEPTION) << "Unsupported data type of tuple or list elements: " << vec[0]->type_name();
|
||||
}
|
||||
}
|
||||
|
||||
GeTensor ConvertAnyUtil(const ValuePtr &value, const AnyTraits<AnyValue>) {
|
||||
MS_EXCEPTION_IF_NULL(value);
|
||||
if (value->isa<MeTensor>()) {
|
||||
// convert me tensor to ge tensor
|
||||
//将me_tensor转化为ge_tensor
|
||||
return ConvertAnyUtil(value, AnyTraits<MeTensor>());
|
||||
} else if (value->isa<ValueList>() || value->isa<ValueTuple>()) {
|
||||
return VectorToTensorUtil(value);
|
||||
} else if (value->isa<Int32Imm>()) {
|
||||
// convert scalar Int to GeTensor
|
||||
//将scalar Int转化为GeTensor
|
||||
MS_LOG(INFO) << "convert scalar to tensor with data type = Int32";
|
||||
GeTensorDesc desc(GeShape(), ge::FORMAT_NCHW, ge::DT_INT32);
|
||||
auto v = GetValue<int32_t>(value);
|
||||
desc.SetRealDimCnt(0);
|
||||
return GeTensor(desc, reinterpret_cast<uint8_t *>(&v), sizeof(int32_t));
|
||||
} else if (value->isa<Int64Imm>()) {
|
||||
// convert scalar Int64 to GeTensor
|
||||
//将scalar转化为GeTensor
|
||||
MS_LOG(INFO) << "convert scalar to tensor with data type = Int64";
|
||||
GeTensorDesc desc(GeShape(), ge::FORMAT_NCHW, ge::DT_INT64);
|
||||
auto v = GetValue<int64_t>(value);
|
||||
desc.SetRealDimCnt(0);
|
||||
return GeTensor(desc, reinterpret_cast<uint8_t *>(&v), sizeof(int64_t));
|
||||
} else if (value->isa<FP32Imm>()) {
|
||||
// convert scalar FP32 to GeTensor
|
||||
//将scalar FP32转化为GeTensor
|
||||
MS_LOG(INFO) << "convert scalar to tensor with data type = FP32";
|
||||
GeTensorDesc desc(GeShape(), ge::FORMAT_NCHW, ge::DT_FLOAT);
|
||||
auto v = GetValue<float>(value);
|
||||
desc.SetRealDimCnt(0);
|
||||
return GeTensor(desc, reinterpret_cast<uint8_t *>(&v), sizeof(float));
|
||||
} else if (value->isa<BoolImm>()) {
|
||||
// convert scalar FP32 to GeTensor
|
||||
//将scalar FP32转化为GeTensor
|
||||
MS_LOG(INFO) << "convert scalar to tensor with data type = Bool";
|
||||
GeTensorDesc desc(GeShape(), ge::FORMAT_NCHW, ge::DT_BOOL);
|
||||
auto v = GetValue<bool>(value);
|
||||
desc.SetRealDimCnt(0);
|
||||
return GeTensor(desc, reinterpret_cast<uint8_t *>(&v), sizeof(bool));
|
||||
} else if (value->isa<StringImm>()) {
|
||||
// convert String to GeTensor
|
||||
//将String转化为GeTensor
|
||||
MS_LOG(INFO) << "convert string to tensor with data type = String";
|
||||
std::string v = GetValue<std::string>(value);
|
||||
std::vector<int64_t> ge_shape;
|
||||
GeShape shape(ge_shape);
|
||||
GeTensorDesc desc(shape, ge::FORMAT_NCHW, ge::DT_STRING);
|
||||
GeTensor str_tensor(desc);
|
||||
(void)str_tensor.SetData(v);
|
||||
return str_tensor;
|
||||
} else {
|
||||
MS_LOG(WARNING) << "Unsupported value type: " << value->type_name()
|
||||
<< " to convert to tensor. Value: " << value->ToString();
|
||||
}
|
||||
return GeTensor();
|
||||
}
|
||||
|
||||
bool IsCustomPrim(const PrimitivePtr &prim) {
|
||||
if (prim == nullptr) {
|
||||
return false;
|
||||
}
|
||||
|
||||
ValuePtr flag = prim->GetAttr("_custom_op_flag");
|
||||
if (flag == nullptr) {
|
||||
return false;
|
||||
}
|
||||
|
||||
bool is_custom_op = GetValue<bool>(flag);
|
||||
if (!is_custom_op && prim->GetAttr("_custom_op_impl_config_path") != nullptr) {
|
||||
MS_LOG(EXCEPTION) << "The custom op flag is false, but the op information config path is not null, non-custom op "
|
||||
"can not assign the op information config path.";
|
||||
}
|
||||
|
||||
return is_custom_op;
|
||||
}
|
||||
|
||||
bool IsCustomCNode(const AnfNodePtr &anf) {
|
||||
if (anf == nullptr) {
|
||||
return false;
|
||||
}
|
||||
auto node = anf->cast<CNodePtr>();
|
||||
if (node == nullptr) {
|
||||
return false;
|
||||
}
|
||||
if (node->inputs().empty()) {
|
||||
MS_LOG(EXCEPTION) << "Length of node inputs is empty";
|
||||
}
|
||||
MS_EXCEPTION_IF_NULL(node->inputs()[0]);
|
||||
if (!node->inputs()[0]->isa<ValueNode>()) {
|
||||
return false;
|
||||
}
|
||||
auto cus_prim = GetValueNode<PrimitivePtr>(node->inputs()[0]);
|
||||
if (cus_prim == nullptr) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return IsCustomPrim(cus_prim);
|
||||
}
|
||||
|
||||
std::string GetOpIOFormat(const AnfNodePtr &anf) {
|
||||
std::string ret;
|
||||
if (anf == nullptr) {
|
||||
MS_LOG(ERROR) << "The anf is nullptr";
|
||||
return ret;
|
||||
}
|
||||
auto node = anf->cast<CNodePtr>();
|
||||
if (node == nullptr) {
|
||||
MS_LOG(ERROR) << "The anf is not a cnode.";
|
||||
return ret;
|
||||
}
|
||||
if (node->inputs().empty()) {
|
||||
MS_LOG(EXCEPTION) << "Length of node inputs is empty.";
|
||||
}
|
||||
MS_EXCEPTION_IF_NULL(node->inputs()[0]);
|
||||
if (!node->inputs()[0]->isa<ValueNode>()) {
|
||||
MS_LOG(ERROR) << "The anf is not a value node.";
|
||||
return ret;
|
||||
}
|
||||
auto prim = GetValueNode<PrimitivePtr>(node->inputs()[0]);
|
||||
if (prim == nullptr) {
|
||||
MS_LOG(ERROR) << "The anf is not a Primitive.";
|
||||
return ret;
|
||||
}
|
||||
if (prim->HasAttr("io_format")) {
|
||||
return GetValue<std::string>(prim->GetAttr("io_format"));
|
||||
}
|
||||
auto io_format_map = IOFormatMap::get();
|
||||
auto iter = io_format_map.find(prim->name());
|
||||
if (iter == io_format_map.end()) {
|
||||
return "NCHW";
|
||||
}
|
||||
if (iter->second == "format") {
|
||||
ValuePtr format = prim->GetAttr("format");
|
||||
MS_EXCEPTION_IF_NULL(format);
|
||||
if (format->isa<Int64Imm>()) {
|
||||
bool converted = CheckAndConvertUtils::ConvertAttrValueToString(prim->name(), "format", &format);
|
||||
if (converted) {
|
||||
return GetValue<std::string>(format);
|
||||
}
|
||||
} else {
|
||||
return GetValue<std::string>(format);
|
||||
}
|
||||
}
|
||||
return iter->second;
|
||||
}
|
||||
} // namespace transform
|
||||
} // namespace mindspore
|
||||
|
|
@ -0,0 +1,77 @@
|
|||
/**
|
||||
* Copyright 2019 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_ADAPTER_UTIL_H_
|
||||
#define MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_ADAPTER_UTIL_H_
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "transform/graph_ir/op_adapter_base.h"
|
||||
|
||||
namespace mindspore {
|
||||
namespace transform {
|
||||
template <typename P, typename Q>
|
||||
static Q ConvertAnyUtil(const ValuePtr &value, const AnyTraits<P> &, const AnyTraits<Q> &) {
|
||||
return static_cast<Q>(GetValue<P>(value));
|
||||
}
|
||||
|
||||
GeTensor ConvertAnyUtil(const ValuePtr &value, const AnyTraits<mindspore::tensor::Tensor> &traits);
|
||||
|
||||
std::vector<int64_t> ConvertAnyUtil(const ValuePtr &value, const std::string &name,
|
||||
const AnyTraits<std::vector<int64_t>>);
|
||||
|
||||
std::string ConvertAnyUtil(const ValuePtr &value, const AnyTraits<std::vector<int64_t>>, const AnyTraits<std::string>);
|
||||
|
||||
std::vector<float> ConvertAnyUtil(const ValuePtr &value, const AnyTraits<std::vector<float>>, const AnyTraits<float>);
|
||||
|
||||
std::vector<int64_t> ConvertAnyUtil(const ValuePtr &value, const std::string &format,
|
||||
const AnyTraits<std::vector<int64_t>>, const AnyTraits<int64_t>);
|
||||
|
||||
GeDataType ConvertAnyUtil(const ValuePtr &value, const AnyTraits<GEType>);
|
||||
|
||||
template <typename P, typename Q>
|
||||
//定义名为ConvertAnyUtil的函数，返回值类型为std::vector<Q>
|
||||
/*
|
||||
判断value是否为空
|
||||
若value既非isa<ValueTuple>()也非isa<ValueList>()则输出EXCEPTION日志
|
||||
判断value是否为isa<ValueTuple>()，若是则执行value->cast<ValueTuplePtr>()->value()并将结果储存在vec中
|
||||
定义变量data并将对it执类型转换后添加到data的尾部
|
||||
返回data
|
||||
*/
|
||||
std::vector<Q> ConvertAnyUtil(const ValuePtr &value, AnyTraits<P>, const AnyTraits<std::vector<Q>>) {
|
||||
MS_EXCEPTION_IF_NULL(value);
|
||||
if (!value->isa<ValueTuple>() && !value->isa<ValueList>()) {
|
||||
MS_LOG(EXCEPTION) << "error convert Value to vector for value: " << value->ToString()
|
||||
<< ", type: " << value->type_name() << ", value should be a tuple or list";
|
||||
}
|
||||
auto vec = value->isa<ValueTuple>() ? value->cast<ValueTuplePtr>()->value() : value->cast<ValueListPtr>()->value();
|
||||
std::vector<Q> data;
|
||||
for (auto &it : vec) {
|
||||
data.push_back(ConvertAnyUtil(it, AnyTraits<P>(), AnyTraits<Q>()));
|
||||
}
|
||||
return data;
|
||||
}
|
||||
//转换value的数据类型
|
||||
GeTensor ConvertAnyUtil(const ValuePtr &value, const AnyTraits<AnyValue>);
|
||||
//判断prim是否为CustomPrim
|
||||
//判断node是否为CustomCNode
|
||||
bool IsCustomPrim(const PrimitivePtr &prim);
|
||||
bool IsCustomCNode(const AnfNodePtr &node);
|
||||
std::string GetOpIOFormat(const AnfNodePtr &node);
|
||||
} // namespace transform
|
||||
} // namespace mindspore
|
||||
#endif // MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_ADAPTER_UTIL_H_
|
||||
|
|
@ -0,0 +1,86 @@
|
|||
/**
|
||||
* Copyright 2019-2021 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#include "transform/graph_ir/op_declare/pad_ops_declare.h"//按照路径寻找以下文件，导入到本文件
|
||||
#include <vector>//提供vector数组构建函数模版等
|
||||
|
||||
namespace mindspore::transform {//创建名为transform的空间，其空间处于空间mindspore下
|
||||
// PadD
|
||||
INPUT_MAP(PadD) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入NPUGetFloatStatus对应空间内并用input_map指针保存
|
||||
ATTR_MAP(PadD) = {{"paddings", ATTR_DESC(paddings, AnyTraits<std::vector<std::vector<int64_t>>>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
// 存入PadD对应空间并用attr_map_指针保存
|
||||
// 其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
// std:vector<>的作用为构建<>内类型的容量可变的数组
|
||||
OUTPUT_MAP(PadD) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入PadD对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(PadD, kNamePadD, ADPT_DESC(PadD))//构造指向PadD的指针并储存，创建结构体RegAdptDescPadD
|
||||
|
||||
// Pad
|
||||
INPUT_MAP(Pad) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(paddings)}};
|
||||
//将变量x、paddings处理并存入对应InputDesc结构体的相应变量中，存入Pad对应空间内并用input_map指针保存
|
||||
ATTR_MAP(Pad) = EMPTY_ATTR_MAP;//将空变量存入Pad对应空间并用attr_map_指针保存
|
||||
OUTPUT_MAP(Pad) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入Pad对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(Pad, kNamePadV1, ADPT_DESC(Pad))//构造指向Pad的指针并储存，创建结构体RegAdptDescPad
|
||||
|
||||
// BroadcastToD
|
||||
INPUT_MAP(BroadcastToD) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入BroadcastToD对应空间内并用input_map指针保存
|
||||
ATTR_MAP(BroadcastToD) = {{"shape", ATTR_DESC(shape, AnyTraits<int64_t>(), AnyTraits<std::vector<int64_t>>())}};
|
||||
//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
//存入BroadcastToD对应空间并用attr_map_指针保存
|
||||
//其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
//std:vector<>的作用为构建<>内类型的容量可变的数组
|
||||
OUTPUT_MAP(BroadcastToD) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入BroadcastToD对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(BroadcastToD, kNameBroadcastTo, ADPT_DESC(BroadcastToD))//构造指向BroadcastToD的指针并储存，创建结构体RegAdptDescBroadcastToD
|
||||
|
||||
// Diag
|
||||
INPUT_MAP(Diag) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入Diag对应空间内并用input_map指针保存
|
||||
ATTR_MAP(Diag) = EMPTY_ATTR_MAP;//将空变量存入Diag对应空间并用attr_map_指针保存
|
||||
OUTPUT_MAP(Diag) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入Diag对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(Diag, kNameDiag, ADPT_DESC(Diag))//构造指向Diag的指针并储存，创建结构体RegAdptDescDiag
|
||||
|
||||
// FillD
|
||||
INPUT_MAP(FillD) = {{1, INPUT_DESC(value)}};//将变量value处理并存入对应InputDesc结构体的相应变量中，存入FillD对应空间内并用input_map指针保存
|
||||
ATTR_MAP(FillD) = {{"dims", ATTR_DESC(dims, AnyTraits<std::vector<int64_t>>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
// 存入FillD对应空间并用attr_map_指针保存
|
||||
// 其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
// std:vector<>的作用为构建<>内类型的容量可变的数组
|
||||
OUTPUT_MAP(FillD) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入FillD对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(FillD, kNameFillD, ADPT_DESC(FillD))//构造指向FillD的指针并储存，创建结构体RegAdptDescFillD
|
||||
|
||||
// Fill
|
||||
INPUT_MAP(Fill) = {{1, INPUT_DESC(dims)}, {2, INPUT_DESC(value)}};
|
||||
//将变量dims、value处理并存入对应InputDesc结构体的相应变量中，存入Fill对应空间内并用input_map指针保存
|
||||
ATTR_MAP(Fill) = EMPTY_ATTR_MAP;//将空变量存入Fill对应空间并用attr_map_指针保存
|
||||
OUTPUT_MAP(Fill) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入Fill对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(Fill, kNameFillV1, ADPT_DESC(Fill))//构造指向Fill的指针并储存，创建结构体RegAdptDescFill
|
||||
|
||||
// PadV3
|
||||
INPUT_MAP(PadV3) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(paddings)}, {3, INPUT_DESC(constant_values)}};
|
||||
//将变量x、paddings、constant_values处理并存入对应InputDesc结构体的相应变量中，存入PadV3对应空间内并用input_map指针保存
|
||||
ATTR_MAP(PadV3) = {{"mode", ATTR_DESC(mode, AnyTraits<std::string>())},
|
||||
{"pad_contiguous", ATTR_DESC(paddings_contiguous, AnyTraits<bool>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
// 存入PadV3对应空间并用attr_map_指针保存
|
||||
// 其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
// std:vector<>的作用为构建<>内类型的容量可变的数组
|
||||
OUTPUT_MAP(PadV3) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入PadV3对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(PadV3, kNamePadV3, ADPT_DESC(PadV3))//构造指向PadV3的指针并储存，创建结构体RegAdptDescPadV3
|
||||
|
||||
// PadV2
|
||||
INPUT_MAP(PadV2) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(paddings)}, {3, INPUT_DESC(constant_values)}};
|
||||
//将变量x、paddings、constant_values处理并存入对应InputDesc结构体的相应变量中，存入PadV2对应空间内并用input_map指针保存
|
||||
ATTR_MAP(PadV2) = EMPTY_ATTR_MAP;//将空变量存入PadV2对应空间并用attr_map_指针保存
|
||||
OUTPUT_MAP(PadV2) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入PadV2对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(PadV2, kNamePadV2, ADPT_DESC(PadV2))//构造指向PadV2的指针并储存，创建结构体RegAdptDescPadV2
|
||||
} // namespace mindspore::transform
|
||||
|
|
@ -0,0 +1,51 @@
|
|||
/**
|
||||
* Copyright 2019-2021 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_PAD_OPS_DECLARE_H_//判断宏是否被定义，如果宏没有定义，则编译下面代码
|
||||
#define MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_PAD_OPS_DECLARE_H_//定义预处理宏_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_PAD_OPS_DECLARE_H_
|
||||
|
||||
#include <string>//导入标准库中的字符串类及相关操作
|
||||
#include "utils/hash_map.h"//按照路径寻找以下文件，导入到本文件,以下同理
|
||||
#include "transform/graph_ir/op_declare/op_declare_macro.h"
|
||||
#include "ops/pad_ops.h"
|
||||
|
||||
namespace mindspore::transform {//创建名为transform的空间，其空间处于空间mindspore下
|
||||
//对实现PadD的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(PadD)
|
||||
DECLARE_OP_USE_OUTPUT(PadD)
|
||||
//对实现Pad的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(Pad)
|
||||
DECLARE_OP_USE_OUTPUT(Pad)
|
||||
//对实现BroadcastToD的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(BroadcastToD)
|
||||
DECLARE_OP_USE_OUTPUT(BroadcastToD)
|
||||
//对实现Diag的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(Diag)
|
||||
DECLARE_OP_USE_OUTPUT(Diag)
|
||||
//对实现FillD的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(FillD)
|
||||
DECLARE_OP_USE_OUTPUT(FillD)
|
||||
//对实现Fill的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(Fill)
|
||||
DECLARE_OP_USE_OUTPUT(Fill)
|
||||
//对实现PadV3的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(PadV3)
|
||||
DECLARE_OP_USE_OUTPUT(PadV3)
|
||||
//对实现PadV2的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(PadV2)
|
||||
DECLARE_OP_USE_OUTPUT(PadV2)
|
||||
} // namespace mindspore::transform
|
||||
#endif // MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_PAD_OPS_DECLARE_H_
|
||||
|
|
@ -0,0 +1,43 @@
|
|||
/**
|
||||
* Copyright 2019 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#include "transform/graph_ir/op_declare/quantize_ops_declare.h"//按照路径寻找以下文件，导入到本文件
|
||||
|
||||
namespace mindspore::transform {//创建名为transform的空间，其空间处于空间mindspore下
|
||||
// AscendQuant
|
||||
INPUT_MAP(AscendQuant) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入AscendQuant对应空间内并用input_map指针保存
|
||||
ATTR_MAP(AscendQuant) = {{"scale", ATTR_DESC(scale, AnyTraits<float>())},
|
||||
{"offset", ATTR_DESC(offset, AnyTraits<float>())},
|
||||
{"sqrt_mode", ATTR_DESC(sqrt_mode, AnyTraits<bool>())},
|
||||
{"round_mode", ATTR_DESC(round_mode, AnyTraits<std::string>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
// 存入AscendQuant对应空间并用attr_map_指针保存
|
||||
// 其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
// std:vector<>的作用为构建<>内类型的容量可变的数组
|
||||
OUTPUT_MAP(AscendQuant) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入AscendQuant对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(AscendQuant, kNameAscendQuant, ADPT_DESC(AscendQuant))//构造指向AscendQuant的指针并储存，创建结构体RegAdptDescAscendQuant
|
||||
|
||||
// AscendDequant
|
||||
INPUT_MAP(AscendDequant) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(deq_scale)}};
|
||||
//将变量x、deq_scale处理并存入对应InputDesc结构体的相应变量中，存入AscendDequant对应空间内并用input_map指针保存
|
||||
ATTR_MAP(AscendDequant) = {{"sqrt_mode", ATTR_DESC(sqrt_mode, AnyTraits<bool>())},
|
||||
{"relu_flag", ATTR_DESC(relu_flag, AnyTraits<bool>())},
|
||||
{"dtype", ATTR_DESC(dtype, AnyTraits<GEType>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
// 存入AscendDequant对应空间并用attr_map_指针保存
|
||||
// 其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
// std:vector<>的作用为构建<>内类型的容量可变的数组
|
||||
OUTPUT_MAP(AscendDequant) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入AscendDequant对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(AscendDequant, kNameAscendDequant, ADPT_DESC(AscendDequant))//构造指向AscendDequant的指针并储存，创建结构体RegAdptDescAscendDequant
|
||||
} // namespace mindspore::transform
|
||||
|
|
@ -0,0 +1,33 @@
|
|||
/**
|
||||
* Copyright 2019-2021 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_QUANTIZE_OPS_DECLARE_H_//判断宏是否被定义，如果宏没有定义，则编译下面代码
|
||||
#define MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_QUANTIZE_OPS_DECLARE_H_//定义预处理宏_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_QUANTIZE_OPS_DECLARE_H_
|
||||
|
||||
#include <string>//导入标准库中的字符串类及相关操作
|
||||
#include "utils/hash_map.h"//按照路径寻找以下文件，导入到本文件,以下同理
|
||||
#include "transform/graph_ir/op_declare/op_declare_macro.h"
|
||||
#include "ops/quantize_ops.h"
|
||||
|
||||
namespace mindspore::transform {//创建名为transform的空间，其空间处于空间mindspore下
|
||||
//对实现AscendQuant的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(AscendQuant)
|
||||
DECLARE_OP_USE_OUTPUT(AscendQuant)
|
||||
//对实现AscendDequant的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(AscendDequant)
|
||||
DECLARE_OP_USE_OUTPUT(AscendDequant)
|
||||
} // namespace mindspore::transform
|
||||
#endif // MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_QUANTIZE_OPS_DECLARE_H_
|
||||
|
|
@ -0,0 +1,59 @@
|
|||
/**
|
||||
* Copyright 2019-2021 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#include "transform/graph_ir/op_declare/random_ops_declare.h"//按照路径寻找以下文件，导入到本文件
|
||||
|
||||
namespace mindspore::transform {//创建名为transform的空间，其空间处于空间mindspore下
|
||||
// DropOutGenMask
|
||||
INPUT_MAP(DropOutGenMask) = {{1, INPUT_DESC(shape)}, {2, INPUT_DESC(prob)}};
|
||||
//将变量shape、prob处理并存入对应InputDesc结构体的相应变量中，存入DropOutGenMask对应空间内并用input_map指针保存
|
||||
ATTR_MAP(DropOutGenMask) = {{"Seed0", ATTR_DESC(seed, AnyTraits<int64_t>())},
|
||||
{"Seed1", ATTR_DESC(seed2, AnyTraits<int64_t>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
// 存入DropOutGenMask对应空间并用attr_map_指针保存
|
||||
// 其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
OUTPUT_MAP(DropOutGenMask) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入DropOutGenMask对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(DropOutGenMask, prim::kPrimDropoutGenMask->name(), ADPT_DESC(DropOutGenMask))
|
||||
//构造指向DropOutGenMask的指针并储存，创建结构体RegAdptDescDropOutGenMask
|
||||
|
||||
// LinSpace
|
||||
INPUT_MAP(LinSpace) = {{1, INPUT_DESC(start)}, {2, INPUT_DESC(stop)}, {3, INPUT_DESC(num)}};
|
||||
//将变量start、stop、num处理并存入对应InputDesc结构体的相应变量中，存入LinSpace对应空间内并用input_map指针保存
|
||||
ATTR_MAP(LinSpace) = EMPTY_ATTR_MAP;//将空变量存入LinSpace对应空间并用attr_map_指针保存
|
||||
OUTPUT_MAP(LinSpace) = {{0, OUTPUT_DESC(output)}};//将变量output处理并存入对应OutputDesc结构体的相应变量中，存入LinSpace对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(LinSpace, kNameLinSpace, ADPT_DESC(LinSpace))//构造指向LinSpace的指针并储存，创建结构体RegAdptDescLinSpace
|
||||
|
||||
// RandomChoiceWithMask
|
||||
INPUT_MAP(RandomChoiceWithMask) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入RandomChoiceWithMask对应空间内并用input_map指针保存
|
||||
ATTR_MAP(RandomChoiceWithMask) = {{"count", ATTR_DESC(count, AnyTraits<int64_t>())},
|
||||
{"seed", ATTR_DESC(seed, AnyTraits<int64_t>())},
|
||||
{"seed2", ATTR_DESC(seed2, AnyTraits<int64_t>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
// 存入RandomChoiceWithMask对应空间并用attr_map_指针保存
|
||||
// 其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
OUTPUT_MAP(RandomChoiceWithMask) = {{0, OUTPUT_DESC(y)}, {1, OUTPUT_DESC(mask)}};
|
||||
//将变量y、mask处理并存入对应OutputDesc结构体的相应变量中，存入RandomChoiceWithMask对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(RandomChoiceWithMask, kNameRandomChoiceWithMask, ADPT_DESC(RandomChoiceWithMask))
|
||||
//构造指向RandomChoiceWithMask的指针并储存，创建结构体RegAdptDescRandomChoiceWithMask
|
||||
|
||||
// TruncatedNormal
|
||||
INPUT_MAP(TruncatedNormal) = {{1, INPUT_DESC(shape)}};//将变量shape处理并存入对应InputDesc结构体的相应变量中，存入TruncatedNormal对应空间内并用input_map指针保存
|
||||
ATTR_MAP(TruncatedNormal) = {{"seed", ATTR_DESC(seed, AnyTraits<int64_t>())},
|
||||
{"seed2", ATTR_DESC(seed2, AnyTraits<int64_t>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
// 存入TruncatedNormal对应空间并用attr_map_指针保存
|
||||
// 其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
OUTPUT_MAP(TruncatedNormal) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入TruncatedNormal对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(TruncatedNormal, kNameTruncatedNormal, ADPT_DESC(TruncatedNormal))
|
||||
//构造指向TruncatedNormal的指针并储存，创建结构体RegAdptDescTruncatedNormal
|
||||
} // namespace mindspore::transform
|
||||
|
|
@ -0,0 +1,39 @@
|
|||
/**
|
||||
* Copyright 2019-2021 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_RANDOM_OPS_DECLARE_H_//判断宏是否被定义，如果宏没有定义，则编译下面代码
|
||||
#define MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_RANDOM_OPS_DECLARE_H_//定义预处理宏_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_QUANTIZE_OPS_DECLARE_H_
|
||||
|
||||
#include <string>//导入标准库中的字符串类及相关操作
|
||||
#include "utils/hash_map.h"//按照路径寻找以下文件，导入到本文件,以下同理
|
||||
#include "transform/graph_ir/op_declare/op_declare_macro.h"
|
||||
#include "ops/random_ops.h"
|
||||
|
||||
namespace mindspore::transform {//创建名为transform的空间，其空间处于空间mindspore下
|
||||
//对实现DropOutGenMask的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(DropOutGenMask)
|
||||
DECLARE_OP_USE_OUTPUT(DropOutGenMask)
|
||||
//对实现LinSpace的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(LinSpace)
|
||||
DECLARE_OP_USE_OUTPUT(LinSpace)
|
||||
//对实现RandomChoiceWithMask的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(RandomChoiceWithMask)
|
||||
DECLARE_OP_USE_OUTPUT(RandomChoiceWithMask)
|
||||
//对实现TruncatedNormal的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(TruncatedNormal)
|
||||
DECLARE_OP_USE_OUTPUT(TruncatedNormal)
|
||||
} // namespace mindspore::transform
|
||||
#endif // MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_RANDOM_OPS_DECLARE_H_
|
||||
|
|
@ -0,0 +1,144 @@
|
|||
/**
|
||||
* Copyright 2019-2021 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#include "transform/graph_ir/op_declare/reduce_ops_declare.h"//按照路径寻找以下文件，导入到本文件
|
||||
#include <vector>//提供vector数组构建函数模版等
|
||||
|
||||
namespace mindspore::transform {//创建名为transform的空间，其空间处于空间mindspore下
|
||||
// BNTrainingReduce
|
||||
INPUT_MAP(BNTrainingReduce) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入BNTrainingReduce对应空间内并用input_map指针保存
|
||||
ATTR_MAP(BNTrainingReduce) = EMPTY_ATTR_MAP;//将空变量存入BNTrainingReduce对应空间并用attr_map_指针保存
|
||||
OUTPUT_MAP(BNTrainingReduce) = {{0, OUTPUT_DESC(sum)}, {1, OUTPUT_DESC(square_sum)}};
|
||||
//将变量sum、square_sum处理并存入对应OutputDesc结构体的相应变量中，存入BNTrainingReduce对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(BNTrainingReduce, kNameBNTrainingReduce, ADPT_DESC(BNTrainingReduce))
|
||||
//构造指向BNTrainingReduce的指针并储存，创建结构体RegAdptDescBNTrainingReduce
|
||||
|
||||
// BNTrainingReduceGrad
|
||||
INPUT_MAP(BNTrainingReduceGrad) = {{1, INPUT_DESC(grads)}, {2, INPUT_DESC(x)}, {3, INPUT_DESC(diff_scale)},
|
||||
{4, INPUT_DESC(diff_offset)}, {5, INPUT_DESC(scale)}, {6, INPUT_DESC(batch_mean)},
|
||||
{7, INPUT_DESC(batch_variance)}};
|
||||
//将变量grads、x、diff_scale、diff_offset、scale、batch_mean、batch_variance处理并存入对应InputDesc结构体的相应变量中
|
||||
//存入BNTrainingReduceGrad对应空间内并用input_map指针保存
|
||||
ATTR_MAP(BNTrainingReduceGrad) = {{"epsilon", ATTR_DESC(epsilon, AnyTraits<float>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
// 存入BNTrainingReduceGrad对应空间并用attr_map_指针保存
|
||||
// 其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
OUTPUT_MAP(BNTrainingReduceGrad) = {{0, OUTPUT_DESC(y)}};
|
||||
//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入BNTrainingReduceGrad对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(BNTrainingReduceGrad, kNameBNTrainingReduceGrad, ADPT_DESC(BNTrainingReduceGrad))
|
||||
//构造指向BNTrainingReduceGrad的指针并储存，创建结构体RegAdptDescBNTrainingReduceGrad
|
||||
|
||||
// BNTrainingUpdate
|
||||
INPUT_MAP(BNTrainingUpdate) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(sum)}, {3, INPUT_DESC(square_sum)},
|
||||
{4, INPUT_DESC(scale)}, {5, INPUT_DESC(offset)}, {6, INPUT_DESC(mean)},
|
||||
{7, INPUT_DESC(variance)}};
|
||||
//将变量x、sum、square_sum、scale、offset、mean、variance处理并存入对应InputDesc结构体的相应变量中
|
||||
//存入BNTrainingUpdate对应空间内并用input_map指针保存
|
||||
ATTR_MAP(BNTrainingUpdate) = {{"factor", ATTR_DESC(factor, AnyTraits<float>())},
|
||||
{"epsilon", ATTR_DESC(epsilon, AnyTraits<float>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
// 存入BNTrainingUpdate对应空间并用attr_map_指针保存
|
||||
// 其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
OUTPUT_MAP(BNTrainingUpdate) = {{0, OUTPUT_DESC(y)},
|
||||
{1, OUTPUT_DESC(mean)},
|
||||
{2, OUTPUT_DESC(variance)},
|
||||
{3, OUTPUT_DESC(batch_mean)},
|
||||
{4, OUTPUT_DESC(batch_variance)}};
|
||||
//将变量y、mean、variance、batch_mean、batch_variance处理并存入对应OutputDesc结构体的相应变量中，存入BNTrainingUpdate对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(BNTrainingUpdate, kNameBNTrainingUpdate, ADPT_DESC(BNTrainingUpdate))
|
||||
//构造指向BNTrainingUpdate的指针并储存，创建结构体RegAdptDescBNTrainingUpdate
|
||||
|
||||
// BNTrainingUpdateGrad
|
||||
INPUT_MAP(BNTrainingUpdateGrad) = {
|
||||
{1, INPUT_DESC(grads)}, {2, INPUT_DESC(x)}, {3, INPUT_DESC(batch_mean)}, {4, INPUT_DESC(batch_variance)}};
|
||||
//将变量grads、x、batch_mean、batch_variance处理并存入对应InputDesc结构体的相应变量中
|
||||
//存入BNTrainingUpdateGrad对应空间内并用input_map指针保存
|
||||
ATTR_MAP(BNTrainingUpdateGrad) = {{"epsilon", ATTR_DESC(epsilon, AnyTraits<float>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
// 存入BNTrainingUpdateGrad对应空间并用attr_map_指针保存
|
||||
// 其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
OUTPUT_MAP(BNTrainingUpdateGrad) = {{0, OUTPUT_DESC(diff_scale)}, {1, OUTPUT_DESC(diff_offset)}};
|
||||
//将变量diff_scale、diff_offset处理并存入对应OutputDesc结构体的相应变量中，存入BNTrainingUpdateGrad对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(BNTrainingUpdateGrad, kNameBNTrainingUpdateGrad, ADPT_DESC(BNTrainingUpdateGrad))
|
||||
//构造指向BNTrainingUpdateGrad的指针并储存，创建结构体RegAdptDescBNTrainingUpdateGrad
|
||||
|
||||
// ReduceAnyD
|
||||
INPUT_MAP(ReduceAnyD) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入ReduceAnyD对应空间内并用input_map指针保存
|
||||
INPUT_ATTR_MAP(ReduceAnyD) = {
|
||||
{2, ATTR_DESC(axes, AnyTraits<std::vector<int64_t>>(), AnyTraits<std::vector<int64_t>>())}};//对实现ReduceAnyD的对象转换后转移出相应存储空间
|
||||
ATTR_MAP(ReduceAnyD) = {{"keep_dims", ATTR_DESC(keep_dims, AnyTraits<bool>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
// 存入ReduceAnyD对应空间并用attr_map_指针保存
|
||||
// 其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
OUTPUT_MAP(ReduceAnyD) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入ReduceAnyD对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(ReduceAnyD, kNameReduceAnyD, ADPT_DESC(ReduceAnyD))//构造指向ReduceAnyD的指针并储存，创建结构体RegAdptDescReduceAnyD
|
||||
|
||||
// ReduceSumD
|
||||
INPUT_MAP(ReduceSumD) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入ReduceSumD对应空间内并用input_map指针保存
|
||||
INPUT_ATTR_MAP(ReduceSumD) = {
|
||||
{2, ATTR_DESC(axes, AnyTraits<std::vector<int64_t>>(), AnyTraits<std::vector<int64_t>>())}};//对实现ReduceSumD的对象转换后转移出相应存储空间
|
||||
ATTR_MAP(ReduceSumD) = {{"keep_dims", ATTR_DESC(keep_dims, AnyTraits<bool>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
// 存入ReduceSumD对应空间并用attr_map_指针保存
|
||||
// 其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
OUTPUT_MAP(ReduceSumD) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入ReduceSumD对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(ReduceSumD, prim::kPrimReduceSum->name(), ADPT_DESC(ReduceSumD))//构造指向ReduceSumD的指针并储存，创建结构体RegAdptDescReduceSumD
|
||||
|
||||
// ReduceProdD
|
||||
INPUT_MAP(ReduceProdD) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入ReduceProdD对应空间内并用input_map指针保存
|
||||
INPUT_ATTR_MAP(ReduceProdD) = {
|
||||
{2, ATTR_DESC(axes, AnyTraits<std::vector<int64_t>>(), AnyTraits<std::vector<int64_t>>())}};//对实现ReduceProdD的对象转换后转移出相应存储空间
|
||||
ATTR_MAP(ReduceProdD) = {{"keep_dims", ATTR_DESC(keep_dims, AnyTraits<bool>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
// 存入ReduceProdD对应空间并用attr_map_指针保存
|
||||
// 其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
OUTPUT_MAP(ReduceProdD) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入ReduceProdD对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(ReduceProdD, kNameReduceProd, ADPT_DESC(ReduceProdD))//构造指向ReduceProdD的指针并储存，创建结构体RegAdptDescReduceProdD
|
||||
|
||||
// ReduceAllD
|
||||
INPUT_MAP(ReduceAllD) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入ReduceAllD对应空间内并用input_map指针保存
|
||||
INPUT_ATTR_MAP(ReduceAllD) = {
|
||||
{2, ATTR_DESC(axes, AnyTraits<std::vector<int64_t>>(), AnyTraits<std::vector<int64_t>>())}};//对实现ReduceAllD的对象转换后转移出相应存储空间
|
||||
ATTR_MAP(ReduceAllD) = {{"keep_dims", ATTR_DESC(keep_dims, AnyTraits<bool>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
// 存入ReduceAllD对应空间并用attr_map_指针保存
|
||||
// 其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
OUTPUT_MAP(ReduceAllD) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入ReduceAllD对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(ReduceAllD, prim::kPrimReduceAll->name(), ADPT_DESC(ReduceAllD))//构造指向ReduceAllD的指针并储存，创建结构体RegAdptDescReduceAllD
|
||||
|
||||
// ReduceMeanD
|
||||
INPUT_MAP(ReduceMeanD) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入ReduceMeanD对应空间内并用input_map指针保存
|
||||
INPUT_ATTR_MAP(ReduceMeanD) = {
|
||||
{2, ATTR_DESC(axes, AnyTraits<std::vector<int64_t>>(), AnyTraits<std::vector<int64_t>>())}};//对实现ReduceMeanD的对象转换后转移出相应存储空间
|
||||
ATTR_MAP(ReduceMeanD) = {{"keep_dims", ATTR_DESC(keep_dims, AnyTraits<bool>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
// 存入ReduceMeanD对应空间并用attr_map_指针保存
|
||||
// 其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
OUTPUT_MAP(ReduceMeanD) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入ReduceMeanD对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(ReduceMeanD, prim::kPrimReduceMean->name(), ADPT_DESC(ReduceMeanD))//构造指向ReduceMeanD的指针并储存，创建结构体RegAdptDescReduceMeanD
|
||||
|
||||
// ReduceMinD
|
||||
INPUT_MAP(ReduceMinD) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入ReduceMinD对应空间内并用input_map指针保存
|
||||
INPUT_ATTR_MAP(ReduceMinD) = {
|
||||
{2, ATTR_DESC(axes, AnyTraits<std::vector<int64_t>>(), AnyTraits<std::vector<int64_t>>())}};//对实现ReduceMinD的对象转换后转移出相应存储空间
|
||||
ATTR_MAP(ReduceMinD) = {{"keep_dims", ATTR_DESC(keep_dims, AnyTraits<bool>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
// 存入ReduceMinD对应空间并用attr_map_指针保存
|
||||
// 其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
OUTPUT_MAP(ReduceMinD) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入ReduceMinD对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(ReduceMinD, prim::kPrimReduceMin->name(), ADPT_DESC(ReduceMinD))//构造指向ReduceMinD的指针并储存，创建结构体RegAdptDescReduceMinD
|
||||
|
||||
// ReduceMaxD
|
||||
INPUT_MAP(ReduceMaxD) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入ReduceMaxD对应空间内并用input_map指针保存
|
||||
INPUT_ATTR_MAP(ReduceMaxD) = {
|
||||
{2, ATTR_DESC(axes, AnyTraits<std::vector<int64_t>>(), AnyTraits<std::vector<int64_t>>())}};//对实现ReduceMaxD的对象转换后转移出相应存储空间
|
||||
ATTR_MAP(ReduceMaxD) = {{"keep_dims", ATTR_DESC(keep_dims, AnyTraits<bool>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
// 存入ReduceMaxD对应空间并用attr_map_指针保存
|
||||
// 其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
OUTPUT_MAP(ReduceMaxD) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入ReduceMaxD对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(ReduceMaxD, prim::kPrimReduceMax->name(), ADPT_DESC(ReduceMaxD))//构造指向ReduceMaxD的指针并储存，创建结构体RegAdptDescReduceMaxD
|
||||
} // namespace mindspore::transform
|
||||
|
|
@ -0,0 +1,69 @@
|
|||
/**
|
||||
* Copyright 2019-2021 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_REDUCE_OPS_DECLARE_H_//判断宏是否被定义，如果宏没有定义，则编译下面代码
|
||||
#define MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_REDUCE_OPS_DECLARE_H_//定义预处理宏_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_REDUCE_OPS_DECLARE_H_
|
||||
|
||||
#include <string>//导入标准库中的字符串类及相关操作
|
||||
#include "utils/hash_map.h"//按照路径寻找以下文件，导入到本文件,以下同理
|
||||
#include "transform/graph_ir/op_declare/op_declare_macro.h"
|
||||
#include "ops/reduce_ops.h"
|
||||
|
||||
namespace mindspore::transform {//创建名为transform的空间，其空间处于空间mindspore下
|
||||
//对实现ReduceMean的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(ReduceMean)
|
||||
//对实现ReduceMinD的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(ReduceMinD)
|
||||
DECLARE_OP_USE_INPUT_ATTR(ReduceMinD)
|
||||
DECLARE_OP_USE_OUTPUT(ReduceMinD)
|
||||
//对实现ReduceMaxD的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(ReduceMaxD)
|
||||
DECLARE_OP_USE_INPUT_ATTR(ReduceMaxD)
|
||||
DECLARE_OP_USE_OUTPUT(ReduceMaxD)
|
||||
//对实现ReduceAllD的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(ReduceAllD)
|
||||
DECLARE_OP_USE_INPUT_ATTR(ReduceAllD)
|
||||
DECLARE_OP_USE_OUTPUT(ReduceAllD)
|
||||
//对实现BNTrainingReduce的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(BNTrainingReduce)
|
||||
DECLARE_OP_USE_OUTPUT(BNTrainingReduce)
|
||||
//对实现BNTrainingReduceGrad的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(BNTrainingReduceGrad)
|
||||
DECLARE_OP_USE_OUTPUT(BNTrainingReduceGrad)
|
||||
//对实现BNTrainingUpdate的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(BNTrainingUpdate)
|
||||
DECLARE_OP_USE_OUTPUT(BNTrainingUpdate)
|
||||
//对实现BNTrainingUpdateGrad的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(BNTrainingUpdateGrad)
|
||||
DECLARE_OP_USE_OUTPUT(BNTrainingUpdateGrad)
|
||||
//对实现ReduceSumD的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(ReduceSumD)
|
||||
DECLARE_OP_USE_INPUT_ATTR(ReduceSumD)
|
||||
DECLARE_OP_USE_OUTPUT(ReduceSumD)
|
||||
//对实现ReduceAnyD的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(ReduceAnyD)
|
||||
DECLARE_OP_USE_INPUT_ATTR(ReduceAnyD)
|
||||
DECLARE_OP_USE_OUTPUT(ReduceAnyD)
|
||||
//对实现ReduceMeanD的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(ReduceMeanD)
|
||||
DECLARE_OP_USE_INPUT_ATTR(ReduceMeanD)
|
||||
DECLARE_OP_USE_OUTPUT(ReduceMeanD)
|
||||
//对实现ReduceProdD的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(ReduceProdD)
|
||||
DECLARE_OP_USE_INPUT_ATTR(ReduceProdD)
|
||||
DECLARE_OP_USE_OUTPUT(ReduceProdD)
|
||||
} // namespace mindspore::transform
|
||||
#endif // MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_REDUCE_OPS_DECLARE_H_
|
||||
|
|
@ -0,0 +1,54 @@
|
|||
/**
|
||||
* Copyright 2019-2021 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_RNN_DECLARE_H_//判断宏是否被定义，如果宏没有定义，则编译下面代码
|
||||
#define MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_RNN_DECLARE_H_//定义预处理宏_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_RNN_DECLARE_H_
|
||||
|
||||
#include <string>//导入标准库中的字符串类及相关操作
|
||||
#include "utils/hash_map.h"//按照路径寻找以下文件，导入到本文件,以下同理
|
||||
#include "ops/rnn.h"
|
||||
#include "transform/graph_ir/op_declare/op_declare_macro.h"
|
||||
|
||||
namespace mindspore::transform {//创建名为transform的空间，其空间处于空间mindspore下
|
||||
//对实现BasicLSTMCell的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(BasicLSTMCell)
|
||||
DECLARE_OP_USE_OUTPUT(BasicLSTMCell)
|
||||
//对实现BasicLSTMCellInputGrad的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(BasicLSTMCellInputGrad)
|
||||
DECLARE_OP_USE_OUTPUT(BasicLSTMCellInputGrad)
|
||||
//对实现BasicLSTMCellWeightGrad的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(BasicLSTMCellWeightGrad)
|
||||
DECLARE_OP_USE_OUTPUT(BasicLSTMCellWeightGrad)
|
||||
//对实现BasicLSTMCellCStateGrad的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(BasicLSTMCellCStateGrad)
|
||||
DECLARE_OP_USE_OUTPUT(BasicLSTMCellCStateGrad)
|
||||
//对实现LSTMInputGrad的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(LSTMInputGrad)
|
||||
DECLARE_OP_USE_OUTPUT(LSTMInputGrad)
|
||||
//对实现DynamicRNN的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(DynamicRNN)
|
||||
DECLARE_OP_USE_OUTPUT(DynamicRNN)
|
||||
//对实现DynamicRNNGrad的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(DynamicRNNGrad)
|
||||
DECLARE_OP_USE_OUTPUT(DynamicRNNGrad)
|
||||
//对实现DynamicGRUV2的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(DynamicGRUV2)
|
||||
DECLARE_OP_USE_OUTPUT(DynamicGRUV2)
|
||||
//对实现DynamicGRUV2Grad的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(DynamicGRUV2Grad)
|
||||
DECLARE_OP_USE_OUTPUT(DynamicGRUV2Grad)
|
||||
} // namespace mindspore::transform
|
||||
#endif // MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_RNN_DECLARE_H_
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
/**
|
||||
* Copyright 2019 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#include "transform/graph_ir/op_declare/rpn_ops_declare.h"//按照路径寻找以下文件，导入到本文件
|
||||
|
||||
namespace mindspore::transform {//创建名为transform的空间，其空间处于空间mindspore下
|
||||
// NMSWithMask
|
||||
INPUT_MAP(NMSWithMask) = {{1, INPUT_DESC(box_scores)}};//将变量box_scores处理并存入对应InputDesc结构体的相应变量中，存入NMSWithMask对应空间内并用input_map指针保存
|
||||
ATTR_MAP(NMSWithMask) = {{"iou_threshold", ATTR_DESC(iou_threshold, AnyTraits<float>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
// 存入NMSWithMask对应空间并用attr_map_指针保存
|
||||
// 其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
OUTPUT_MAP(NMSWithMask) = {
|
||||
{0, OUTPUT_DESC(selected_boxes)}, {1, OUTPUT_DESC(selected_idx)}, {2, OUTPUT_DESC(selected_mask)}};
|
||||
//将变量selected_boxes、selected_idx、selected_mask处理并存入对应OutputDesc结构体的相应变量中，存入NMSWithMask对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(NMSWithMask, kNameNMSWithMask, ADPT_DESC(NMSWithMask))//构造指向NMSWithMask的指针并储存，创建结构体RegAdptDescNMSWithMask
|
||||
} // namespace mindspore::transform
|
||||
|
|
@ -0,0 +1,30 @@
|
|||
/**
|
||||
* Copyright 2019-2021 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_RPN_OPS_DECLARE_H_//判断宏是否被定义，如果宏没有定义，则编译下面代码
|
||||
#define MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_RPN_OPS_DECLARE_H_//定义预处理宏_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_RPN_OPS_DECLARE_H_
|
||||
|
||||
#include <string>//导入标准库中的字符串类及相关操作
|
||||
#include "utils/hash_map.h"//按照路径寻找以下文件，导入到本文件,以下同理
|
||||
#include "transform/graph_ir/op_declare/op_declare_macro.h"
|
||||
#include "ops/rpn_ops.h"
|
||||
|
||||
namespace mindspore::transform {//创建名为transform的空间，其空间处于空间mindspore下
|
||||
//对实现NMSWithMask的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(NMSWithMask)
|
||||
DECLARE_OP_USE_OUTPUT(NMSWithMask)
|
||||
} // namespace mindspore::transform
|
||||
#endif // MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_RPN_OPS_DECLARE_H_
|
||||
|
|
@ -0,0 +1,289 @@
|
|||
/**
|
||||
* Copyright 2019-2021 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#include <vector>//提供vector数组构建函数模版等
|
||||
#include "transform/graph_ir/op_declare/selection_ops_declare.h"//按照路径寻找以下文件，导入到本文件
|
||||
|
||||
namespace mindspore::transform {//创建名为transform的空间，其空间处于空间mindspore下
|
||||
// CumsumD
|
||||
INPUT_MAP(CumsumD) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入CumsumD对应空间内并用input_map指针保存
|
||||
INPUT_ATTR_MAP(CumsumD) = {{2, ATTR_DESC(axis, AnyTraits<int64_t>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
// 存入CumsumD对应空间并用attr_map_指针保存
|
||||
// 其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
ATTR_MAP(CumsumD) = {{"exclusive", ATTR_DESC(exclusive, AnyTraits<bool>())},
|
||||
{"reverse", ATTR_DESC(reverse, AnyTraits<bool>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
// 存入CumsumD对应空间并用attr_map_指针保存
|
||||
// 其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
OUTPUT_MAP(CumsumD) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入CumsumD对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(CumsumD, kNameCumSum, ADPT_DESC(CumsumD))//构造指向CumsumD的指针并储存，创建结构体RegAdptDescCumsumD
|
||||
|
||||
// GatherV2
|
||||
INPUT_MAP(GatherV2) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(indices)}, {3, INPUT_DESC(axis)}};
|
||||
//将变量x、indices、axis处理并存入对应InputDesc结构体的相应变量中，存入GatherV2对应空间内并用input_map指针保存
|
||||
ATTR_MAP(GatherV2) = EMPTY_ATTR_MAP;//将空变量存入GatherV2对应空间并用attr_map_指针保存
|
||||
OUTPUT_MAP(GatherV2) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入GatherV2对应空间内并用output_map_指针保存
|
||||
|
||||
// CumprodD
|
||||
INPUT_MAP(CumprodD) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入CumprodD对应空间内并用input_map指针保存
|
||||
INPUT_ATTR_MAP(CumprodD) = {{2, ATTR_DESC(axis, AnyTraits<int64_t>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
// 存入CumprodD对应空间并用attr_map_指针保存
|
||||
// 其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
ATTR_MAP(CumprodD) = {{"exclusive", ATTR_DESC(exclusive, AnyTraits<bool>())},
|
||||
{"reverse", ATTR_DESC(reverse, AnyTraits<bool>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
// 存入CumprodD对应空间并用attr_map_指针保存
|
||||
// 其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
OUTPUT_MAP(CumprodD) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入CumprodD对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(CumprodD, kNameCumProd, ADPT_DESC(CumprodD))//构造指向CumprodD的指针并储存，创建结构体RegAdptDescCumprodD
|
||||
|
||||
//SliceD
|
||||
INPUT_MAP(SliceD) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入SliceD对应空间内并用input_map指针保存
|
||||
INPUT_ATTR_MAP(SliceD) = {{2, ATTR_DESC(offsets, AnyTraits<int64_t>(), AnyTraits<std::vector<int64_t>>())},
|
||||
{3, ATTR_DESC(size, AnyTraits<int64_t>(), AnyTraits<std::vector<int64_t>>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
// 存入SliceD对应空间并用attr_map_指针保存
|
||||
// 其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
// std:vector<>的作用为构建<>内类型的容量可变的数组
|
||||
ATTR_MAP(SliceD) = EMPTY_ATTR_MAP;//将空变量存入SliceD对应空间并用attr_map_指针保存
|
||||
OUTPUT_MAP(SliceD) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入SliceD对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(SliceD, kNameSlice, ADPT_DESC(SliceD))//构造指向SliceD的指针并储存，创建结构体RegAdptDescSliceD
|
||||
|
||||
// TopK
|
||||
INPUT_MAP(TopK) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(k)}};
|
||||
//将变量x、k处理并存入对应InputDesc结构体的相应变量中，存入TopK对应空间内并用input_map指针保存
|
||||
ATTR_MAP(TopK) = {{"sorted", ATTR_DESC(sorted, AnyTraits<bool>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
// 存入TopK对应空间并用attr_map_指针保存
|
||||
// 其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
OUTPUT_MAP(TopK) = {{0, OUTPUT_DESC(values)}, {1, OUTPUT_DESC(indices)}};
|
||||
//将变量values、indices处理并存入对应OutputDesc结构体的相应变量中，存入TopK对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(TopK, kNameTopK, ADPT_DESC(TopK))//构造指向TopK的指针并储存，创建结构体RegAdptDescTopK
|
||||
|
||||
// InTopK
|
||||
INPUT_MAP(InTopKD) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}};//将变量x1、x2处理并存入对应InputDesc结构体的相应变量中，存入InTopKD对应空间内并用input_map指针保存
|
||||
ATTR_MAP(InTopKD) = {{"k", ATTR_DESC(k, AnyTraits<int64_t>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
// 存入InTopKD对应空间并用attr_map_指针保存
|
||||
// 其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
OUTPUT_MAP(InTopKD) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入InTopKD对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(InTopKD, kNameInTopKD, ADPT_DESC(InTopKD))//构造指向InTopKD的指针并储存，创建结构体RegAdptDescInTopKD
|
||||
|
||||
// TileD
|
||||
INPUT_MAP(TileD) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入TileD对应空间内并用input_map指针保存
|
||||
INPUT_ATTR_MAP(TileD) = {{2, ATTR_DESC(multiples, AnyTraits<int64_t>(), AnyTraits<std::vector<int64_t>>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
// 存入SliceD对应空间并用attr_map_指针保存
|
||||
// 其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
// std:vector<>的作用为构建<>内类型的容量可变的数组
|
||||
ATTR_MAP(TileD) = EMPTY_ATTR_MAP;//将空变量存入TileD对应空间并用attr_map_指针保存
|
||||
OUTPUT_MAP(TileD) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入TileD对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(TileD, kNameTile, ADPT_DESC(TileD))//构造指向TileD的指针并储存，创建结构体RegAdptDescTileD
|
||||
|
||||
// OneHot
|
||||
INPUT_MAP(OneHot) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(depth)}, {3, INPUT_DESC(on_value)}, {4, INPUT_DESC(off_value)}};
|
||||
//将变量x、depth、on_value、off_value处理并存入对应InputDesc结构体的相应变量中，存入OneHot对应空间内并用input_map指针保存
|
||||
ATTR_MAP(OneHot) = {{"axis", ATTR_DESC(axis, AnyTraits<int64_t>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
// 存入OneHot对应空间并用attr_map_指针保存
|
||||
// 其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
OUTPUT_MAP(OneHot) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入OneHot对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(OneHot, prim::kPrimOneHot->name(), ADPT_DESC(OneHot))//构造指向OneHot的指针并储存，创建结构体RegAdptDescOneHot
|
||||
|
||||
// GatherV2D
|
||||
INPUT_MAP(GatherV2D) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(indices)}};
|
||||
//将变量x、indices处理并存入对应InputDesc结构体的相应变量中，存入GatherV2D对应空间内并用input_map指针保存
|
||||
INPUT_ATTR_MAP(GatherV2D) = {{3, ATTR_DESC(axis, AnyTraits<int64_t>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
// 存入GatherV2D对应空间并用attr_map_指针保存
|
||||
// 其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
ATTR_MAP(GatherV2D) = EMPTY_ATTR_MAP;//将空变量存入GatherV2D对应空间并用attr_map_指针保存
|
||||
OUTPUT_MAP(GatherV2D) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入GatherV2D对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(GatherV2D, prim::kPrimGather->name(), ADPT_DESC(GatherV2D))//构造指向GatherV2D的指针并储存，创建结构体RegAdptDescGatherV2D
|
||||
REG_ADPT_DESC(Gather, kNameGather, ADPT_DESC(GatherV2D))//构造指向Gather的指针并储存，创建结构体RegAdptDescGather
|
||||
|
||||
// ScatterNdD
|
||||
INPUT_MAP(ScatterNdD) = {{1, INPUT_DESC(indices)}, {2, INPUT_DESC(x)}};
|
||||
//将变量indices、x处理并存入对应InputDesc结构体的相应变量中，存入ScatterNdD对应空间内并用input_map指针保存
|
||||
INPUT_ATTR_MAP(ScatterNdD) = {
|
||||
{3, ATTR_DESC(shape, AnyTraits<std::vector<int64_t>>(), AnyTraits<std::vector<int64_t>>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
// 存入ScatterNdD对应空间并用attr_map_指针保存
|
||||
// 其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
// std:vector<>的作用为构建<>内类型的容量可变的数组
|
||||
ATTR_MAP(ScatterNdD) = EMPTY_ATTR_MAP;//将空变量存入ScatterNdD对应空间并用attr_map_指针保存
|
||||
OUTPUT_MAP(ScatterNdD) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入ScatterNdD对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(ScatterNdD, kNameScatterNdD, ADPT_DESC(ScatterNdD))//构造指向ScatterNdD的指针并储存，创建结构体RegAdptDescScatterNdD
|
||||
|
||||
// ScatterNonAliasingAdd
|
||||
INPUT_MAP(ScatterNonAliasingAdd) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(indices)}, {3, INPUT_DESC(updates)}};
|
||||
//将变量x、indices、updates处理并存入对应InputDesc结构体的相应变量中，存入ScatterNonAliasingAdd对应空间内并用input_map指针保存
|
||||
ATTR_MAP(ScatterNonAliasingAdd) = EMPTY_ATTR_MAP;//将空变量存入ScatterNonAliasingAdd对应空间并用attr_map_指针保存
|
||||
OUTPUT_MAP(ScatterNonAliasingAdd) = {{0, OUTPUT_DESC(y)}};
|
||||
//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入ScatterNonAliasingAdd对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(ScatterNonAliasingAdd, kNameScatterNonAliasingAdd, ADPT_DESC(ScatterNonAliasingAdd))
|
||||
//构造指向ScatterNonAliasingAdd的指针并储存，创建结构体RegAdptDescScatterNonAliasingAdd
|
||||
|
||||
// GatherNd
|
||||
INPUT_MAP(GatherNd) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(indices)}};
|
||||
//将变量x、indices处理并存入对应InputDesc结构体的相应变量中，存入GatherNd对应空间内并用input_map指针保存
|
||||
ATTR_MAP(GatherNd) = EMPTY_ATTR_MAP;//将空变量存入GatherNd对应空间并用attr_map_指针保存
|
||||
OUTPUT_MAP(GatherNd) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入GatherNd对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(GatherNd, kNameGatherNd, ADPT_DESC(GatherNd))//构造指向GatherNd的指针并储存，创建结构体RegAdptDescGatherNd
|
||||
|
||||
// GatherD
|
||||
INPUT_MAP(GatherD) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(dim)}, {3, INPUT_DESC(index)}};
|
||||
//将变量x、dim、index处理并存入对应InputDesc结构体的相应变量中，存入GatherD对应空间内并用input_map指针保存
|
||||
ATTR_MAP(GatherD) = EMPTY_ATTR_MAP;//将空变量存入GatherD对应空间并用attr_map_指针保存
|
||||
OUTPUT_MAP(GatherD) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入GatherD对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(GatherD, kNameGatherD, ADPT_DESC(GatherD))//构造指向GatherD的指针并储存，创建结构体RegAdptDescGatherD
|
||||
|
||||
// Range
|
||||
INPUT_MAP(RangeD) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入RangeD对应空间内并用input_map指针保存
|
||||
ATTR_MAP(RangeD) = {{"start", ATTR_DESC(start, AnyTraits<float>())},
|
||||
{"limit", ATTR_DESC(limit, AnyTraits<float>())},
|
||||
{"delta", ATTR_DESC(delta, AnyTraits<float>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
// 存入RangeD对应空间并用attr_map_指针保存
|
||||
// 其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
OUTPUT_MAP(RangeD) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入RangeD对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(RangeD, kNameRange, ADPT_DESC(RangeD))//构造指向RangeD的指针并储存，创建结构体RegAdptDescRangeD
|
||||
|
||||
// InplaceAddD
|
||||
INPUT_MAP(InplaceAddD) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(v)}};
|
||||
//将变量x、v处理并存入对应InputDesc结构体的相应变量中，存入InplaceAddD对应空间内并用input_map指针保存
|
||||
ATTR_MAP(InplaceAddD) = {{"indices", ATTR_DESC(indices, AnyTraits<std::vector<int64_t>>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
// 存入InplaceAddD对应空间并用attr_map_指针保存
|
||||
// 其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
// std:vector<>的作用为构建<>内类型的容量可变的数组
|
||||
OUTPUT_MAP(InplaceAddD) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入InplaceAddD对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(InplaceAddD, kNameInplaceAddD, ADPT_DESC(InplaceAddD))//构造指向InplaceAddD的指针并储存，创建结构体RegAdptDescInplaceAddD
|
||||
|
||||
// InplaceSubD
|
||||
INPUT_MAP(InplaceSubD) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(v)}};
|
||||
//将变量x、v处理并存入对应InputDesc结构体的相应变量中，存入InplaceSubD对应空间内并用input_map指针保存
|
||||
ATTR_MAP(InplaceSubD) = {{"indices", ATTR_DESC(indices, AnyTraits<std::vector<int64_t>>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
// 存入InplaceSubD对应空间并用attr_map_指针保存
|
||||
// 其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
// std:vector<>的作用为构建<>内类型的容量可变的数组
|
||||
OUTPUT_MAP(InplaceSubD) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入InplaceSubD对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(InplaceSubD, kNameInplaceSubD, ADPT_DESC(InplaceSubD))//构造指向InplaceSubD的指针并储存，创建结构体RegAdptDescInplaceSubD
|
||||
|
||||
// InplaceUpdateD
|
||||
INPUT_MAP(InplaceUpdateD) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(v)}};
|
||||
//将变量x、v处理并存入对应InputDesc结构体的相应变量中，存入InplaceUpdateD对应空间内并用input_map指针保存
|
||||
ATTR_MAP(InplaceUpdateD) = {{"indices", ATTR_DESC(indices, AnyTraits<std::vector<int64_t>>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
// 存入InplaceUpdateD对应空间并用attr_map_指针保存
|
||||
// 其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
// std:vector<>的作用为构建<>内类型的容量可变的数组
|
||||
OUTPUT_MAP(InplaceUpdateD) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入InplaceUpdateD对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(InplaceUpdateD, kNameInplaceUpdateD, ADPT_DESC(InplaceUpdateD))//构造指向InplaceUpdateD的指针并储存，创建结构体RegAdptDescInplaceUpdateD
|
||||
|
||||
// Select
|
||||
INPUT_MAP(Select) = {{1, INPUT_DESC(condition)}, {2, INPUT_DESC(x1)}, {3, INPUT_DESC(x2)}};
|
||||
//将变量condition、x1、x2处理并存入对应InputDesc结构体的相应变量中，存入Select对应空间内并用input_map指针保存
|
||||
ATTR_MAP(Select) = EMPTY_ATTR_MAP;//将空变量存入Select对应空间并用attr_map_指针保存
|
||||
OUTPUT_MAP(Select) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入InplaceUpdateD对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(Select, prim::kPrimSelect->name(), ADPT_DESC(Select))//构造指向Select的指针并储存，创建结构体RegAdptDescSelect
|
||||
|
||||
// StridedSliceGrad
|
||||
INPUT_MAP(StridedSliceGrad) = {
|
||||
{1, INPUT_DESC(dy)}, {2, INPUT_DESC(shape)}, {3, INPUT_DESC(begin)}, {4, INPUT_DESC(end)}, {5, INPUT_DESC(strides)}};
|
||||
//将变量dy、shape、begin、end、strides处理并存入对应InputDesc结构体的相应变量中，存入StridedSliceGrad对应空间内并用input_map指针保存
|
||||
ATTR_MAP(StridedSliceGrad) = {{"begin_mask", ATTR_DESC(begin_mask, AnyTraits<int64_t>())},
|
||||
{"end_mask", ATTR_DESC(end_mask, AnyTraits<int64_t>())},
|
||||
{"ellipsis_mask", ATTR_DESC(ellipsis_mask, AnyTraits<int64_t>())},
|
||||
{"new_axis_mask", ATTR_DESC(new_axis_mask, AnyTraits<int64_t>())},
|
||||
{"shrink_axis_mask", ATTR_DESC(shrink_axis_mask, AnyTraits<int64_t>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
// 存入StridedSliceGrad对应空间并用attr_map_指针保存
|
||||
// 其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
OUTPUT_MAP(StridedSliceGrad) = {{0, OUTPUT_DESC(output)}};
|
||||
//将变量output处理并存入对应OutputDesc结构体的相应变量中，存入StridedSliceGrad对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(StridedSliceGrad, kNameStridedSliceGrad, ADPT_DESC(StridedSliceGrad))
|
||||
//构造指向StridedSliceGrad的指针并储存，创建结构体RegAdptDescStridedSliceGrad
|
||||
|
||||
// StridedSlice
|
||||
INPUT_MAP(StridedSlice) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(begin)}, {3, INPUT_DESC(end)}, {4, INPUT_DESC(strides)}};
|
||||
//将变量x、begin、end、strides处理并存入对应InputDesc结构体的相应变量中，存入StridedSlice对应空间内并用input_map指针保存
|
||||
ATTR_MAP(StridedSlice) = {{"begin_mask", ATTR_DESC(begin_mask, AnyTraits<int64_t>())},
|
||||
{"end_mask", ATTR_DESC(end_mask, AnyTraits<int64_t>())},
|
||||
{"ellipsis_mask", ATTR_DESC(ellipsis_mask, AnyTraits<int64_t>())},
|
||||
{"new_axis_mask", ATTR_DESC(new_axis_mask, AnyTraits<int64_t>())},
|
||||
{"shrink_axis_mask", ATTR_DESC(shrink_axis_mask, AnyTraits<int64_t>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
// 存入StridedSlice对应空间并用attr_map_指针保存
|
||||
// 其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
OUTPUT_MAP(StridedSlice) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入StridedSlice对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(StridedSlice, kNameStridedSlice, ADPT_DESC(StridedSlice))
|
||||
//构造指向StridedSlice的指针并储存，创建结构体RegAdptDescStridedSlice
|
||||
|
||||
// StridedSliceV2
|
||||
INPUT_MAP(StridedSliceV2) = {
|
||||
{1, INPUT_DESC(x)}, {2, INPUT_DESC(begin)}, {3, INPUT_DESC(end)}, {4, INPUT_DESC(axes)}, {5, INPUT_DESC(strides)}};
|
||||
//将变量x、begin、end、axes、strides处理并存入对应InputDesc结构体的相应变量中，存入StridedSlice对应空间内并用input_map指针保存
|
||||
ATTR_MAP(StridedSliceV2) = {{"begin_mask", ATTR_DESC(begin_mask, AnyTraits<int64_t>())},
|
||||
{"end_mask", ATTR_DESC(end_mask, AnyTraits<int64_t>())},
|
||||
{"ellipsis_mask", ATTR_DESC(ellipsis_mask, AnyTraits<int64_t>())},
|
||||
{"new_axis_mask", ATTR_DESC(new_axis_mask, AnyTraits<int64_t>())},
|
||||
{"shrink_axis_mask", ATTR_DESC(shrink_axis_mask, AnyTraits<int64_t>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
// 存入StridedSliceV2对应空间并用attr_map_指针保存
|
||||
// 其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
OUTPUT_MAP(StridedSliceV2) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入StridedSliceV2对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(StridedSliceV2, kNameStridedSliceV2, ADPT_DESC(StridedSliceV2))
|
||||
//构造指向StridedSliceV2的指针并储存，创建结构体RegAdptDescStridedSliceV2
|
||||
|
||||
// UnsortedSegmentSum
|
||||
INPUT_MAP(UnsortedSegmentSumD) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(segment_ids)}};
|
||||
//将变量x、segment_ids处理并存入对应InputDesc结构体的相应变量中，存入UnsortedSegmentSumD对应空间内并用input_map指针保存
|
||||
INPUT_ATTR_MAP(UnsortedSegmentSumD) = {{3, ATTR_DESC(num_segments, AnyTraits<int64_t>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
// 存入UnsortedSegmentSumD对应空间并用attr_map_指针保存
|
||||
// 其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
ATTR_MAP(UnsortedSegmentSumD) = EMPTY_ATTR_MAP;//将空变量存入UnsortedSegmentSumD对应空间并用attr_map_指针保存
|
||||
OUTPUT_MAP(UnsortedSegmentSumD) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入UnsortedSegmentSumD对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(UnsortedSegmentSumD, prim::kPrimUnsortedSegmentSum->name(), ADPT_DESC(UnsortedSegmentSumD))
|
||||
//构造指向UnsortedSegmentSumD的指针并储存，创建结构体RegAdptDescUnsortedSegmentSumD
|
||||
|
||||
// UnsortedSegmentProdD
|
||||
INPUT_MAP(UnsortedSegmentProdD) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(segment_ids)}};
|
||||
//将变量x、segment_ids处理并存入对应InputDesc结构体的相应变量中，存入UnsortedSegmentProdD对应空间内并用input_map指针保存
|
||||
INPUT_ATTR_MAP(UnsortedSegmentProdD) = {{3, ATTR_DESC(num_segments, AnyTraits<int64_t>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
// 存入UnsortedSegmentProdD对应空间并用attr_map_指针保存
|
||||
// 其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
ATTR_MAP(UnsortedSegmentProdD) = EMPTY_ATTR_MAP;//将空变量存入UnsortedSegmentProdD对应空间并用attr_map_指针保存
|
||||
OUTPUT_MAP(UnsortedSegmentProdD) = {{0, OUTPUT_DESC(y)}};
|
||||
//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入UnsortedSegmentProdD对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(UnsortedSegmentProdD, kNameUnsortedSegmentProdD, ADPT_DESC(UnsortedSegmentProdD))
|
||||
//构造指向UnsortedSegmentProdD的指针并储存，创建结构体RegAdptDescUnsortedSegmentProdD
|
||||
|
||||
// UnsortedSegmentMaxD
|
||||
INPUT_MAP(UnsortedSegmentMaxD) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(segment_ids)}};
|
||||
//将变量x、segment_ids处理并存入对应InputDesc结构体的相应变量中，存入UnsortedSegmentMaxD对应空间内并用input_map指针保存
|
||||
INPUT_ATTR_MAP(UnsortedSegmentMaxD) = {{3, ATTR_DESC(num_segments, AnyTraits<int64_t>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
// 存入UnsortedSegmentMaxD对应空间并用attr_map_指针保存
|
||||
// 其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
ATTR_MAP(UnsortedSegmentMaxD) = EMPTY_ATTR_MAP;//将空变量存入UnsortedSegmentMaxD对应空间并用attr_map_指针保存
|
||||
OUTPUT_MAP(UnsortedSegmentMaxD) = {{0, OUTPUT_DESC(y)}};
|
||||
//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入UnsortedSegmentMaxD对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(UnsortedSegmentMaxD, kNameUnsortedSegmentMaxD, ADPT_DESC(UnsortedSegmentMaxD))
|
||||
//构造指向UnsortedSegmentMaxD的指针并储存，创建结构体RegAdptDescUnsortedSegmentMaxD
|
||||
|
||||
// UnsortedSegmentMin
|
||||
INPUT_MAP(UnsortedSegmentMin) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(segment_ids)}, {3, INPUT_DESC(num_segments)}};
|
||||
//将变量x、segment_ids、num_segments处理并存入对应InputDesc结构体的相应变量中，存入UnsortedSegmentMin对应空间内并用input_map指针保存
|
||||
ATTR_MAP(UnsortedSegmentMin) = EMPTY_ATTR_MAP;//将空变量存入UnsortedSegmentMin对应空间并用attr_map_指针保存
|
||||
OUTPUT_MAP(UnsortedSegmentMin) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入UnsortedSegmentMin对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(UnsortedSegmentMin, prim::kPrimUnsortedSegmentMin->name(), ADPT_DESC(UnsortedSegmentMin))
|
||||
//构造指向UnsortedSegmentMin的指针并储存，创建结构体RegAdptDescUnsortedSegmentMin
|
||||
|
||||
// ReverseV2
|
||||
INPUT_MAP(ReverseV2D) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入ReverseV2D对应空间内并用input_map指针保存
|
||||
ATTR_MAP(ReverseV2D) = {{"axis", ATTR_DESC(axis, AnyTraits<int64_t>(), AnyTraits<std::vector<int64_t>>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
// 存入ReverseV2D对应空间并用attr_map_指针保存
|
||||
// 其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
// std:vector<>的作用为构建<>内类型的容量可变的数组
|
||||
OUTPUT_MAP(ReverseV2D) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入ReverseV2D对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(ReverseV2D, kNameReverseV2, ADPT_DESC(ReverseV2D))
|
||||
//构造指向ReverseV2D的指针并储存，创建结构体RegAdptDescReverseV2D
|
||||
} // namespace mindspore::transform
|
||||
|
|
@ -0,0 +1,114 @@
|
|||
/**
|
||||
* Copyright 2019-2021 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_SELECTION_OPS_DECLARE_H_//判断宏是否被定义，如果宏没有定义，则编译下面代码
|
||||
#define MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_SELECTION_OPS_DECLARE_H_//定义预处理宏_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_SELECTION_DECLARE_H_
|
||||
|
||||
#include <string>//导入标准库中的字符串类及相关操作
|
||||
#include "utils/hash_map.h"//按照路径寻找以下文件，导入到本文件,以下同理
|
||||
#include "transform/graph_ir/op_declare/op_declare_macro.h"
|
||||
#include "ops/selection_ops.h"
|
||||
|
||||
namespace mindspore::transform {//创建名为transform的空间，其空间处于空间mindspore下
|
||||
//对实现SliceD的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(SliceD)
|
||||
DECLARE_OP_USE_INPUT_ATTR(SliceD)
|
||||
DECLARE_OP_USE_OUTPUT(SliceD)
|
||||
//对实现ScatterNdD的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(ScatterNdD)
|
||||
DECLARE_OP_USE_INPUT_ATTR(ScatterNdD)
|
||||
DECLARE_OP_USE_OUTPUT(ScatterNdD)
|
||||
//对实现ScatterNonAliasingAdd的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(ScatterNonAliasingAdd)
|
||||
DECLARE_OP_USE_OUTPUT(ScatterNonAliasingAdd)
|
||||
//对实现GatherNd的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(GatherNd)
|
||||
DECLARE_OP_USE_OUTPUT(GatherNd)
|
||||
//对实现GatherD的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(GatherD)
|
||||
DECLARE_OP_USE_OUTPUT(GatherD)
|
||||
//对实现TopK的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(TopK)
|
||||
DECLARE_OP_USE_OUTPUT(TopK)
|
||||
//对实现InTopKD的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(InTopKD)
|
||||
DECLARE_OP_USE_OUTPUT(InTopKD)
|
||||
//对实现Select的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(Select)
|
||||
DECLARE_OP_USE_OUTPUT(Select)
|
||||
//对实现StridedSliceGrad的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(StridedSliceGrad)
|
||||
DECLARE_OP_USE_OUTPUT(StridedSliceGrad)
|
||||
//对实现StridedSlice的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(StridedSlice)
|
||||
DECLARE_OP_USE_OUTPUT(StridedSlice)
|
||||
//对实现StridedSliceV2的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(StridedSliceV2)
|
||||
DECLARE_OP_USE_OUTPUT(StridedSliceV2)
|
||||
//对实现UnsortedSegmentSumD的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(UnsortedSegmentSumD)
|
||||
DECLARE_OP_USE_INPUT_ATTR(UnsortedSegmentSumD)
|
||||
DECLARE_OP_USE_OUTPUT(UnsortedSegmentSumD)
|
||||
//对实现UnsortedSegmentProdD的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(UnsortedSegmentProdD)
|
||||
DECLARE_OP_USE_INPUT_ATTR(UnsortedSegmentProdD)
|
||||
DECLARE_OP_USE_OUTPUT(UnsortedSegmentProdD)
|
||||
//对实现UnsortedSegmentMaxD的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(UnsortedSegmentMaxD)
|
||||
DECLARE_OP_USE_INPUT_ATTR(UnsortedSegmentMaxD)
|
||||
DECLARE_OP_USE_OUTPUT(UnsortedSegmentMaxD)
|
||||
//对实现UnsortedSegmentMin的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(UnsortedSegmentMin)
|
||||
DECLARE_OP_USE_OUTPUT(UnsortedSegmentMin)
|
||||
//对实现CumprodD的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(CumprodD)
|
||||
DECLARE_OP_USE_INPUT_ATTR(CumprodD)
|
||||
DECLARE_OP_USE_OUTPUT(CumprodD)
|
||||
//对实现TileD的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(TileD)
|
||||
DECLARE_OP_USE_INPUT_ATTR(TileD)
|
||||
DECLARE_OP_USE_OUTPUT(TileD)
|
||||
//对实现OneHot的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(OneHot)
|
||||
DECLARE_OP_USE_OUTPUT(OneHot)
|
||||
//对实现GatherV2D的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(GatherV2D)
|
||||
DECLARE_OP_USE_INPUT_ATTR(GatherV2D)
|
||||
DECLARE_OP_USE_OUTPUT(GatherV2D)
|
||||
//对实现RangeD的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(RangeD)
|
||||
DECLARE_OP_USE_OUTPUT(RangeD)
|
||||
//对实现InplaceAddD的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(InplaceAddD)
|
||||
DECLARE_OP_USE_OUTPUT(InplaceAddD)
|
||||
//对实现InplaceSubD的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(InplaceSubD)
|
||||
DECLARE_OP_USE_OUTPUT(InplaceSubD)
|
||||
//对实现InplaceUpdateD的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(InplaceUpdateD)
|
||||
DECLARE_OP_USE_OUTPUT(InplaceUpdateD)
|
||||
//对实现CumsumD的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(CumsumD)
|
||||
DECLARE_OP_USE_INPUT_ATTR(CumsumD)
|
||||
DECLARE_OP_USE_OUTPUT(CumsumD)
|
||||
//对实现GatherV2的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(GatherV2)
|
||||
DECLARE_OP_USE_OUTPUT(GatherV2)
|
||||
//对实现ReverseV2D的对象转换后转移出相应存储空间
|
||||
DECLARE_OP_ADAPTER(ReverseV2D)
|
||||
DECLARE_OP_USE_OUTPUT(ReverseV2D)
|
||||
} // namespace mindspore::transform
|
||||
#endif // MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_SELECTION_OPS_DECLARE_H_
|
||||
|
|
@ -0,0 +1,79 @@
|
|||
/**
|
||||
* Copyright 2019-2021 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#include "transform/graph_ir/op_declare/split_combination_ops_declare.h"//按照路径寻找以下文件，导入到本文件
|
||||
#include <vector>//提供vector数组构建函数模版等
|
||||
|
||||
namespace mindspore::transform {//创建名为transform的空间，其空间处于空间mindspore下
|
||||
// SplitD
|
||||
INPUT_MAP(SplitD) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入SplitD对应空间内并用input_map指针保存
|
||||
ATTR_MAP(SplitD) = {{"axis", ATTR_DESC(split_dim, AnyTraits<int64_t>())},
|
||||
{"output_num", ATTR_DESC(num_split, AnyTraits<int64_t>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
// 存入SplitD对应空间并用attr_map_指针保存
|
||||
// 其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
DYN_OUTPUT_MAP(SplitD) = {{0, DYN_OUTPUT_DESC(y)}};//将变量x处理并存入对应DynOutputDesc结构体的相应变量中，存入SplitD对应空间内并用input_map指针保存
|
||||
REG_ADPT_DESC(SplitD, kNameSplitD, ADPT_DESC(SplitD))//构造指向SplitD的指针并储存，创建结构体RegAdptDescSplitD
|
||||
|
||||
// Pack
|
||||
INPUT_MAP(Pack) = EMPTY_INPUT_MAP;//将空变量存入Pack对应空间并用input_map指针保存
|
||||
DYN_INPUT_MAP(Pack) = {{1, DYN_INPUT_DESC(x)}};//将变量x处理并存入对应DynOutputDesc结构体的相应变量中，存入Pack对应空间内并用dyn_input_map_指针保存
|
||||
ATTR_MAP(Pack) = {{"num", ATTR_DESC(N, AnyTraits<int64_t>())}, {"axis", ATTR_DESC(axis, AnyTraits<int64_t>())}};
|
||||
//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
//存入Pack对应空间并用attr_map_指针保存
|
||||
//其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
OUTPUT_MAP(Pack) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入Pack对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(Pack, prim::kStack, ADPT_DESC(Pack))//构造指向Pack的指针并储存，创建结构体RegAdptDescPack
|
||||
|
||||
// ParallelConcat
|
||||
INPUT_MAP(ParallelConcat) = EMPTY_INPUT_MAP;//将空变量存入ParallelConcat对应空间并用input_map指针保存
|
||||
DYN_INPUT_MAP(ParallelConcat) = {{1, DYN_INPUT_DESC(values)}};
|
||||
//将变量values处理并存入对应DynOutputDesc结构体的相应变量中，存入ParallelConcat对应空间内并用dyn_input_map_指针保存
|
||||
ATTR_MAP(ParallelConcat) = {
|
||||
{"shape", ATTR_DESC(shape, AnyTraits<std::vector<int64_t>>())},
|
||||
{"N", ATTR_DESC(N, AnyTraits<int64_t>())},
|
||||
};//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
//存入ParallelConcat对应空间并用attr_map_指针保存
|
||||
//其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
//std:vector<>的作用为构建<>内类型的容量可变的数组
|
||||
OUTPUT_MAP(ParallelConcat) = {{0, OUTPUT_DESC(output_data)}};
|
||||
//将变量output_data处理并存入对应OutputDesc结构体的相应变量中，存入ParallelConcat对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(ParallelConcat, kNameParallelConcat, ADPT_DESC(ParallelConcat))
|
||||
//构造指向ParallelConcat的指针并储存，创建结构体RegAdptDescParallelConcat
|
||||
|
||||
// ConcatD
|
||||
INPUT_MAP(ConcatD) = EMPTY_INPUT_MAP;//将空变量存入ConcatD对应空间并用input_map指针保存
|
||||
DYN_INPUT_MAP(ConcatD) = {{1, DYN_INPUT_DESC(x)}};//将变量x处理并存入对应DynOutputDesc结构体的相应变量中，存入ConcatD对应空间内并用dyn_input_map_指针保存
|
||||
ATTR_MAP(ConcatD) = {
|
||||
{"axis", ATTR_DESC(concat_dim, AnyTraits<int64_t>())},
|
||||
{"inputNums", ATTR_DESC(N, AnyTraits<int64_t>())},
|
||||
};//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
//存入ConcatD对应空间并用attr_map_指针保存
|
||||
//其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
OUTPUT_MAP(ConcatD) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入ConcatD)对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(ConcatD, prim::kPrimConcat->name(), ADPT_DESC(ConcatD))//构造指向ConcatD的指针并储存，创建结构体RegAdptDescConcatD
|
||||
|
||||
// ConcatV2D Inference for tf
|
||||
INPUT_MAP(ConcatV2D) = EMPTY_INPUT_MAP;//将空变量存入ConcatV2D对应空间并用input_map指针保存
|
||||
DYN_INPUT_MAP(ConcatV2D) = {{1, DYN_INPUT_DESC(x)}};//将变量x处理并存入对应DynOutputDesc结构体的相应变量中，存入ConcatV2D对应空间内并用dyn_input_map_指针保存
|
||||
ATTR_MAP(ConcatV2D) = {
|
||||
{"axis", ATTR_DESC(concat_dim, AnyTraits<int64_t>())},
|
||||
{"N", ATTR_DESC(N, AnyTraits<int64_t>())},
|
||||
};//对相应变量处理并存入对应AttrDesc结构体的相应变量中
|
||||
//存入ConcaV2tD对应空间并用attr_map_指针保存
|
||||
//其中AnyTraits<>的作用为将<>内类型进行构建
|
||||
OUTPUT_MAP(ConcatV2D) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入ConcatV2D对应空间内并用output_map_指针保存
|
||||
REG_ADPT_DESC(ConcatV2D, kNameConcatV2D, ADPT_DESC(ConcatV2D))//构造指向ConcatV2D的指针并储存，创建结构体RegAdptDescConcatV2D
|
||||
} // namespace mindspore::transform
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue