forked from huawei/mindspore2022
Compare commits
No commits in common. "master" and "master" have entirely different histories.
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
|
|
@ -15,132 +15,69 @@
|
|||
*/
|
||||
#include <string>
|
||||
|
||||
// Include the header file for removing duplicate value nodes in the JIT pipeline
|
||||
#include "pipeline/jit/remove_value_node_dup.h"
|
||||
|
||||
// Include the header files for various IR components
|
||||
#include "ir/anf.h"
|
||||
#include "ir/func_graph.h"
|
||||
#include "ir/tensor.h"
|
||||
#include "ir/manager.h"
|
||||
|
||||
// Include the header file for common subexpression elimination (CSE) utility functions
|
||||
#include "include/common/utils/cse.h"
|
||||
|
||||
// Include the header file for logging adapter utility functions
|
||||
#include "utils/log_adapter.h"
|
||||
|
||||
// Include the header file for hashing utility functions
|
||||
#include "utils/hashing.h"
|
||||
|
||||
// Include the header file for common conversion utility functions
|
||||
#include "include/common/utils/convert_utils.h"
|
||||
|
||||
// Start of the "mindspore" namespace
|
||||
namespace mindspore {
|
||||
|
||||
// Start of the "pipeline" namespace
|
||||
namespace pipeline {
|
||||
|
||||
// Definition of the function "TryToDoReplace" which takes in a pointer to a "FuncGraphManager" object,
|
||||
// a constant reference to an "AnfNodePtr" object, a pointer to a "HashCache" object, and a pointer to a "HashValue" object
|
||||
void TryToDoReplace(FuncGraphManager *const manager, const AnfNodePtr &node, HashCache *const hash_cache,
|
||||
HashValue *const hash_value) {
|
||||
|
||||
// Check if the "manager" pointer is null, and throw an exception if it is
|
||||
MS_EXCEPTION_IF_NULL(manager);
|
||||
|
||||
// Check if the "hash_cache" pointer is null, and throw an exception if it is
|
||||
MS_EXCEPTION_IF_NULL(hash_cache);
|
||||
|
||||
// ... (rest of the code)
|
||||
}
|
||||
} // End of the "pipeline" namespace
|
||||
} // End of the "mindspore" namespace
|
||||
|
||||
// Check if the node is a value node of type FuncGraph
|
||||
if (IsValueNode<FuncGraph>(node)) {
|
||||
// If it is, return and do nothing
|
||||
return;
|
||||
}
|
||||
|
||||
// Get the value of the node
|
||||
const auto &to_check_value = GetValueNode(node);
|
||||
|
||||
// Check if the value is null
|
||||
MS_EXCEPTION_IF_NULL(to_check_value);
|
||||
|
||||
// Calculate hash value.
|
||||
|
||||
// Declare a variable to store the hash value
|
||||
size_t h;
|
||||
|
||||
// Find the iterator for the given node in the hash_value map
|
||||
auto hash_iter = hash_value->find(node);
|
||||
|
||||
// If the iterator is at the end of the map, it means the node is not present in the map
|
||||
if (hash_iter == hash_value->end()) {
|
||||
|
||||
// Calculate the hash value by combining the hash values of to_check_value and opt::AbsOf(node)
|
||||
h = hash_combine(to_check_value->hash(), (opt::AbsOf(node)->hash()));
|
||||
|
||||
// Add the calculated hash value to the hash_value map for the given node
|
||||
(*hash_value)[node] = h;
|
||||
|
||||
} else {
|
||||
|
||||
// If the iterator is not at the end, it means the node is already present in the map
|
||||
// Retrieve the hash value from the iterator
|
||||
h = hash_iter->second;
|
||||
}
|
||||
|
||||
// Find the iterator for the given key 'h' in the hash_cache
|
||||
auto bucket_iter = hash_cache->find(h);
|
||||
|
||||
// If the iterator is pointing to the end of the hash_cache, it means the key 'h' is not present
|
||||
if (bucket_iter == hash_cache->end()) {
|
||||
// Since it's the first time we are encountering this key, add a new bucket with 'node' as its value
|
||||
// Meet for the first time, add bucket.
|
||||
(*hash_cache)[h] = {node};
|
||||
return;
|
||||
}
|
||||
|
||||
auto &bucket = bucket_iter->second;
|
||||
// Check if we need to replace the node with a value node that has already been encountered and cached.
|
||||
|
||||
// Iterate over each value in the bucket
|
||||
// Check if need to replace node with value node already met.
|
||||
for (const auto &v : bucket) {
|
||||
// If the value is the same as the node, it means we have already encountered and cached it.
|
||||
// Already met and cached.
|
||||
if (v == node) {
|
||||
return; // Return without doing anything
|
||||
return;
|
||||
}
|
||||
|
||||
// Get the existing value node
|
||||
const auto &existed_value = GetValueNode(v);
|
||||
MS_EXCEPTION_IF_NULL(existed_value);
|
||||
|
||||
// Define a lambda function to check if the existing value is equal to the value we want to check
|
||||
auto equal = [&]() -> bool {
|
||||
// If both the existing value and the value to check are tensors, compare their values
|
||||
if (existed_value->isa<tensor::Tensor>() && to_check_value->isa<tensor::Tensor>()) {
|
||||
return existed_value->cast<tensor::TensorPtr>()->ValueEqual(*(to_check_value->cast<tensor::TensorPtr>()));
|
||||
}
|
||||
// Otherwise, compare the values directly
|
||||
return *existed_value == *to_check_value;
|
||||
};
|
||||
|
||||
// If the values are equal, replace the node with the existing value node
|
||||
if (equal()) {
|
||||
(void)manager->Replace(node, v);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// This code is likely part of a larger codebase and is inside the "pipeline" namespace and "mindspore" namespace.
|
||||
|
||||
// This code is adding a node to a bucket.
|
||||
|
||||
// The "emplace_back" function is used to add a new element at the end of the container (bucket) by constructing it in-place.
|
||||
|
||||
// The "node" is being passed as an argument to the "emplace_back" function, which means a new element will be created using the constructor of the element type.
|
||||
|
||||
// The closing curly braces indicate the end of the "pipeline" namespace and the "mindspore" namespace.
|
||||
// Meet for the first time, append node to bucket.
|
||||
bucket.emplace_back(node);
|
||||
}
|
||||
} // namespace pipeline
|
||||
} // namespace mindspore
|
||||
|
|
|
|||
|
|
@ -16,600 +16,389 @@
|
|||
* limitations under the License.
|
||||
*/
|
||||
|
||||
// Include the header files for the necessary components of the program
|
||||
|
||||
#include "pipeline/jit/resource.h" // Resource related functions and classes
|
||||
#include "pipeline/jit/static_analysis/static_analysis.h" // Static analysis related functions and classes
|
||||
#include "pipeline/jit/debug/trace.h" // Debugging and tracing related functions and classes
|
||||
#include "ir/dtype.h" // Data type related functions and classes
|
||||
#include "pipeline/jit/parse/data_converter.h" // Data conversion related functions and classes
|
||||
#include "frontend/operator/ops.h" // Operator related functions and classes
|
||||
#include "frontend/optimizer/ad/dfunctor.h" // Automatic differentiation related functions and classes
|
||||
#include "include/common/utils/parallel_context.h" // Parallel computing related functions and classes
|
||||
|
||||
// The code is defining a namespace called "mindspore"
|
||||
#include "pipeline/jit/resource.h"
|
||||
#include "pipeline/jit/static_analysis/static_analysis.h"
|
||||
#include "pipeline/jit/debug/trace.h"
|
||||
#include "ir/dtype.h"
|
||||
#include "pipeline/jit/parse/data_converter.h"
|
||||
#include "frontend/operator/ops.h"
|
||||
#include "frontend/optimizer/ad/dfunctor.h"
|
||||
#include "include/common/utils/parallel_context.h"
|
||||
|
||||
namespace mindspore {
|
||||
// namespace to support opmap definition
|
||||
namespace pipeline {
|
||||
|
||||
// Inside the "mindspore" namespace, there is another namespace called "pipeline"
|
||||
namespace pipeline {
|
||||
|
||||
// This namespace is used to support opmap definition
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// Define and initialize a static variable `method_map` of type `BuiltInTypeMap`
|
||||
BuiltInTypeMap &GetMethodMap() {
|
||||
static BuiltInTypeMap method_map = {
|
||||
// Mapping for kObjectTypeString
|
||||
{
|
||||
{kObjectTypeString,
|
||||
{
|
||||
{"__bool__", std::string("str_bool")}, // C.str_bool
|
||||
{"format", std::string("_format")}
|
||||
}
|
||||
}
|
||||
},
|
||||
// Mapping for kMetaTypeNone
|
||||
{
|
||||
{kMetaTypeNone,
|
||||
{
|
||||
{"__bool__", std::string("none_bool")} // C.none_bool
|
||||
}
|
||||
}
|
||||
},
|
||||
// Mapping for kObjectTypeFunction
|
||||
{
|
||||
{kObjectTypeFunction,
|
||||
{
|
||||
{"__bool__", std::string("func_bool")}, // C.str_bool
|
||||
{"__is_csr_func__", prim::kPrimIsCSRFunc}
|
||||
}
|
||||
}
|
||||
},
|
||||
// Mapping for kNumberTypeBool
|
||||
{
|
||||
{kNumberTypeBool,
|
||||
{
|
||||
{"__and__", prim::kPrimBoolAnd}, // P.bool_and
|
||||
{"__or__", prim::kPrimBoolOr}, // P.bool_or
|
||||
{"__eq__", prim::kPrimBoolEq}, // P.bool_eq
|
||||
{"__ne__", std::string("bool_ne")}, // C.bool_ne
|
||||
{"__bool__", prim::kPrimIdentity} // P.identity
|
||||
}
|
||||
}
|
||||
},
|
||||
// Mapping for kNumberTypeInt
|
||||
{
|
||||
{kNumberTypeInt,
|
||||
{
|
||||
{"__add__", prim::kPrimScalarAdd}, // P.scalar_add
|
||||
{"__sub__", prim::kPrimScalarSub}, // P.scalar_sub
|
||||
{"__mul__", prim::kPrimScalarMul}, // P.scalar_mul
|
||||
{"__floordiv__", std::string("int_floordiv")}, // C.int_floordiv
|
||||
{"__truediv__", std::string("int_truediv")}, // C.int_truediv
|
||||
{"__mod__", prim::kPrimScalarMod}, // P.scalar_mod
|
||||
{"__pow__", prim::kPrimScalarPow}, // P.scalar_pow
|
||||
{"__floor__", prim::kPrimIdentity}, // P.identity
|
||||
{"__trunc__", prim::kPrimIdentity}, // P.identity
|
||||
{"__pos__", prim::kPrimScalarUadd}, // P.scalar_uadd
|
||||
{"__neg__", prim::kPrimScalarUsub}, // P.scalar_usub
|
||||
{"__eq__", prim::kPrimScalarEq}, // P.scalar_eq
|
||||
{"__ne__", prim::kPrimScalarNe}, // P.scalar_ne
|
||||
{"__lt__", prim::kPrimScalarLt}, // P.scalar_lt
|
||||
{"__gt__", prim::kPrimScalarGt}, // P.scalar_gt
|
||||
{"__le__", prim::kPrimScalarLe}, // P.scalar_le
|
||||
{"__ge__", prim::kPrimScalarGe}, // P.scalar_ge
|
||||
{"__bool__", std::string("int_bool")}, // C.int_bool
|
||||
{"__ms_to_array__", prim::kPrimScalarToArray} // P.scalar_to_array
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// Return the reference to the `method_map`
|
||||
static BuiltInTypeMap method_map = {{kObjectTypeString,
|
||||
{{"__bool__", std::string("str_bool")}, // C.str_bool
|
||||
{"format", std::string("_format")}}},
|
||||
{kMetaTypeNone,
|
||||
{
|
||||
{"__bool__", std::string("none_bool")} // C.none_bool
|
||||
}},
|
||||
{kObjectTypeFunction,
|
||||
{{"__bool__", std::string("func_bool")}, // C.str_bool
|
||||
{"__is_csr_func__", prim::kPrimIsCSRFunc}}},
|
||||
{kNumberTypeBool,
|
||||
{
|
||||
{"__and__", prim::kPrimBoolAnd}, // P.bool_and
|
||||
{"__or__", prim::kPrimBoolOr}, // P.bool_or
|
||||
{"__eq__", prim::kPrimBoolEq}, // P.bool_eq
|
||||
{"__ne__", std::string("bool_ne")}, // C.bool_ne
|
||||
{"__bool__", prim::kPrimIdentity} // P.identity
|
||||
}},
|
||||
{kNumberTypeInt,
|
||||
{
|
||||
{"__add__", prim::kPrimScalarAdd}, // P.scalar_add
|
||||
{"__sub__", prim::kPrimScalarSub}, // P.scalar_sub
|
||||
{"__mul__", prim::kPrimScalarMul}, // P.scalar_mul
|
||||
{"__floordiv__", std::string("int_floordiv")}, // C.int_floordiv
|
||||
{"__truediv__", std::string("int_truediv")}, // C.int_truediv
|
||||
{"__mod__", prim::kPrimScalarMod}, // P.scalar_mod
|
||||
{"__pow__", prim::kPrimScalarPow}, // P.scalar_pow
|
||||
{"__floor__", prim::kPrimIdentity}, // P.identity
|
||||
{"__trunc__", prim::kPrimIdentity}, // P.identity
|
||||
{"__pos__", prim::kPrimScalarUadd}, // P.scalar_uadd
|
||||
{"__neg__", prim::kPrimScalarUsub}, // P.scalar_usub
|
||||
{"__eq__", prim::kPrimScalarEq}, // P.scalar_eq
|
||||
{"__ne__", prim::kPrimScalarNe}, // P.scalar_ne
|
||||
{"__lt__", prim::kPrimScalarLt}, // P.scalar_lt
|
||||
{"__gt__", prim::kPrimScalarGt}, // P.scalar_gt
|
||||
{"__le__", prim::kPrimScalarLe}, // P.scalar_le
|
||||
{"__ge__", prim::kPrimScalarGe}, // P.scalar_ge
|
||||
{"__bool__", std::string("int_bool")}, // C.int_bool
|
||||
{"__ms_to_array__", prim::kPrimScalarToArray}, // P.scalar_to_array
|
||||
}},
|
||||
{kNumberTypeUInt,
|
||||
{
|
||||
{"__add__", prim::kPrimScalarAdd}, // P.scalar_add,
|
||||
{"__sub__", prim::kPrimScalarSub}, // P.scalar_sub,
|
||||
{"__mul__", prim::kPrimScalarMul}, // P.scalar_mul,
|
||||
{"__floordiv__", prim::kPrimScalarDiv}, // P.scalar_div,
|
||||
{"__truediv__", std::string("int_truediv")}, // C.int_truediv
|
||||
{"__mod__", prim::kPrimScalarMod}, // P.scalar_mod,
|
||||
{"__pow__", prim::kPrimScalarPow}, // P.scalar_pow,
|
||||
{"__floor__", prim::kPrimIdentity}, // P.identity,
|
||||
{"__trunc__", prim::kPrimIdentity}, // P.identity,
|
||||
{"__pos__", prim::kPrimScalarUadd}, // P.scalar_uadd,
|
||||
{"__neg__", prim::kPrimScalarUsub}, // P.scalar_usub,
|
||||
{"__eq__", prim::kPrimScalarEq}, // P.scalar_eq,
|
||||
{"__ne__", prim::kPrimScalarNe}, // P.scalar_ne,
|
||||
{"__lt__", prim::kPrimScalarLt}, // P.scalar_lt,
|
||||
{"__gt__", prim::kPrimScalarGt}, // P.scalar_gt,
|
||||
{"__le__", prim::kPrimScalarLe}, // P.scalar_le,
|
||||
{"__ge__", prim::kPrimScalarGe}, // P.scalar_ge,
|
||||
{"__bool__", std::string("int_bool")}, // C.int_bool
|
||||
{"__ms_to_array__", prim::kPrimScalarToArray}, // P.scalar_to_array,
|
||||
}},
|
||||
{kNumberTypeFloat,
|
||||
{
|
||||
{"__add__", prim::kPrimScalarAdd}, // P.scalar_add,
|
||||
{"__sub__", prim::kPrimScalarSub}, // P.scalar_sub,
|
||||
{"__mul__", prim::kPrimScalarMul}, // P.scalar_mul,
|
||||
{"__floordiv__", std::string("float_floordiv")}, // C.float_floordiv
|
||||
{"__truediv__", prim::kPrimScalarDiv}, // P.scalar_div,
|
||||
{"__mod__", prim::kPrimScalarMod}, // P.scalar_mod,
|
||||
{"__pow__", prim::kPrimScalarPow}, // P.scalar_pow,
|
||||
{"__floor__", prim::kPrimScalarFloor}, // P.scalar_floor,
|
||||
{"__trunc__", prim::kPrimScalarTrunc}, // P.scalar_trunc,
|
||||
{"__pos__", prim::kPrimScalarUadd}, // P.scalar_uadd,
|
||||
{"__neg__", prim::kPrimScalarUsub}, // P.scalar_usub,
|
||||
{"__eq__", prim::kPrimScalarEq}, // P.scalar_eq,
|
||||
{"__ne__", prim::kPrimScalarNe}, // P.scalar_ne,
|
||||
{"__lt__", prim::kPrimScalarLt}, // P.scalar_lt,
|
||||
{"__gt__", prim::kPrimScalarGt}, // P.scalar_gt,
|
||||
{"__le__", prim::kPrimScalarLe}, // P.scalar_le,
|
||||
{"__ge__", prim::kPrimScalarGe}, // P.scalar_ge,
|
||||
{"__bool__", std::string("float_bool")}, // C.float_bool
|
||||
{"__ms_to_array__", prim::kPrimScalarToArray}, // P.scalar_to_array,
|
||||
}},
|
||||
{kObjectTypeTuple,
|
||||
{
|
||||
{"__len__", prim::kPrimTupleLen}, // P.tuple_len,
|
||||
{"__getitem__", prim::kPrimTupleGetItem}, // P.tuple_getitem,
|
||||
{"__setitem__", prim::kPrimTupleSetItem}, // P.tuple_setitem,
|
||||
{"__ms_iter__", prim::kPrimIdentity}, // P.identity,
|
||||
{"__ms_next__", std::string("tuple_next")}, // C.tuple_next,
|
||||
{"__ms_hasnext__", std::string("tuple_hasnext")}, // C.tuple_hasnext
|
||||
{"__bool__", std::string("tuple_bool")} // C.tuple_bool
|
||||
}},
|
||||
{kObjectTypeList,
|
||||
{
|
||||
{"__len__", prim::kPrimListLen}, // P.list_len,
|
||||
{"__getitem__", prim::kPrimListGetItem}, // P.list_getitem,
|
||||
{"__setitem__", prim::kPrimListSetItem}, // P.list_setitem,
|
||||
{"__ms_iter__", prim::kPrimIdentity}, // P.identity
|
||||
{"__ms_next__", std::string("list_next")}, // C.list_next
|
||||
{"append", std::string("list_append")}, // C.list_next
|
||||
{"__bool__", std::string("list_bool")}, // C.list_bool
|
||||
{"__ms_hasnext__", std::string("list_hasnext")},
|
||||
{"insert", std::string("list_insert")},
|
||||
}},
|
||||
{kObjectTypeDictionary,
|
||||
{
|
||||
{"__len__", prim::kPrimDictLen}, // P.dict_len
|
||||
{"__getitem__", prim::kPrimDictGetItem}, // P.dict_getitem
|
||||
{"__setitem__", prim::kPrimDictSetItem}, // P.dict_setitem,
|
||||
{"keys", prim::kPrimDictGetKeys}, // P.dict_getkeys,
|
||||
{"values", prim::kPrimDictGetValues}, // P.dict_getvalues,
|
||||
{"items", prim::kPrimDictItems}, // P.dict_items
|
||||
{"__bool__", std::string("dict_bool")} // C.dict_bool
|
||||
}},
|
||||
{kObjectTypeTensorType,
|
||||
{
|
||||
{"all", std::string("all_")}, // C.reduce_all
|
||||
{"any", std::string("any_")}, // C.reduce_any
|
||||
{"__add__", std::string("add")}, // C.add
|
||||
{"__sub__", std::string("sub")}, // C.sub
|
||||
{"__mul__", std::string("mul")}, // C.mul
|
||||
{"abs", std::string("abs_")}, // C.abs_
|
||||
{"mean", std::string("mean")}, // C.mean
|
||||
{"__truediv__", std::string("truediv")}, // C.truediv
|
||||
{"__floordiv__", std::string("floordiv")}, // C.floordiv
|
||||
{"__mod__", std::string("mod")}, // C.mod
|
||||
{"__pow__", std::string("pow_")}, // C.pow
|
||||
{"__floor__", std::string("array_floor")}, // C.array_floor
|
||||
{"__trunc__", std::string("array_trunc")}, // C.array_trunc
|
||||
{"__pos__", std::string("array_uadd")}, // C.array_uadd
|
||||
{"__neg__", std::string("array_usub")}, // C.array_usub
|
||||
{"__eq__", std::string("eq")}, // C.eq
|
||||
{"__ne__", std::string("ne")}, // C.ne
|
||||
{"__lt__", std::string("lt")}, // C.lt
|
||||
{"__gt__", std::string("gt")}, // C.gt
|
||||
{"__le__", std::string("le")}, // C.le
|
||||
{"__ge__", std::string("ge")}, // C.ge
|
||||
{"expand_as", std::string("expand_tensor_as")}, // C.expand_as
|
||||
{"view", std::string("view")}, // C.view
|
||||
{"__len__", prim::kPrimArrayLen}, // P.array_len,
|
||||
{"__getitem__", prim::kPrimArrayGetItem}, // P.array_getitem,
|
||||
{"__setitem__", prim::kPrimArraySetItem}, // P.array_setitem,
|
||||
{"__ms_iter__", std::string("array_iter")}, // C.array_iter
|
||||
{"__ms_to_array__", prim::kPrimIdentity}, // P.identity,
|
||||
{"item", std::string("item")}, // P.item,
|
||||
{"itemset", std::string("itemset")}, // P.itemset,
|
||||
{"transpose", std::string("transpose")}, // P.transpose
|
||||
{"flatten", std::string("flatten")}, // P.reshape(,-1)
|
||||
{"reshape", std::string("reshape")}, // P.reshape()
|
||||
{"ravel", std::string("ravel")}, // P.reshape(,(-1,))
|
||||
{"swapaxes", std::string("swapaxes")}, // P.transpose()
|
||||
{"narrow", std::string("narrow")}, // narrow()
|
||||
{"masked_fill", std::string("masked_fill")}, // masked_fill()
|
||||
{"expand_dims", std::string("expand_dims")}, // P.expand_dims()
|
||||
{"squeeze", std::string("squeeze")}, // P.squeeze()
|
||||
{"astype", std::string("astype")}, // P.cast()
|
||||
{"cumsum", std::string("cumsum")}, // P.cumsum()
|
||||
{"copy", std::string("copy")}, // copy()
|
||||
{"max", std::string("max")}, // P.reduce_max()
|
||||
{"min", std::string("min")}, // P.reduce_min()
|
||||
{"fill", std::string("fill")}, // P.fill()
|
||||
{"ptp", std::string("ptp")}, // P.reduce_max() - P.reduce_min()
|
||||
{"clip", std::string("clip")}, // P.maximum(P.minimum)
|
||||
{"__bool__", std::string("tensor_bool")}, // C.tensor_bool
|
||||
{"argmax", std::string("argmax")}, // P.Argmax()
|
||||
{"argmin", std::string("argmin")}, // P.Argmax()
|
||||
{"resize", std::string("resize")}, // P.Reshape()
|
||||
{"choose", std::string("choose")}, // P.Select()
|
||||
{"diagonal", std::string("diagonal")}, // P.Eye()
|
||||
{"searchsorted", std::string("searchsorted")}, // P.Select()
|
||||
{"take", std::string("take")}, // P.GatherNd()
|
||||
{"trace", std::string("trace")}, // P.Eye()
|
||||
{"var", std::string("var")}, // P.ReduceSum
|
||||
{"std", std::string("std")}, // P.ReduceSum
|
||||
{"sum", std::string("sum")}, // P.ReduceSum
|
||||
{"repeat", std::string("repeat")}, // C.repeat_elements
|
||||
}},
|
||||
{kObjectTypeRowTensorType,
|
||||
{
|
||||
{"__add__", prim::kPrimRowTensorAdd}, // P.row_tensor_add
|
||||
}},
|
||||
{kObjectTypeCSRTensorType,
|
||||
{
|
||||
{"astype", std::string("csr_astype")}, // C.csr_astype
|
||||
{"abs", std::string("csr_abs")}, // C.csr_abs
|
||||
{"sum", std::string("csr_sum")}, // C.csr_sum
|
||||
{"mv", std::string("csr_mv")}, // C.csr_mv
|
||||
{"to_tuple", std::string("csr_to_tuple")}, // C.csr_to_tuple
|
||||
{"to_coo", std::string("csr_to_coo")}, // C.csr_to_coo
|
||||
{"to_dense", std::string("csr_to_dense")}, // C.csr_to_dense
|
||||
}},
|
||||
{kObjectTypeCOOTensorType,
|
||||
{
|
||||
{"astype", std::string("coo_astype")}, // C.coo_astype
|
||||
{"abs", std::string("coo_abs")}, // C.coo_abs
|
||||
{"to_tuple", std::string("coo_to_tuple")}, // C.coo_to_tuple
|
||||
{"to_csr", std::string("coo_to_csr")}, // C.coo_to_csr
|
||||
{"to_dense", std::string("coo_to_dense")}, // C.coo_to_dense
|
||||
}},
|
||||
{kObjectTypeJTagged, {}},
|
||||
{kObjectTypeSymbolicKeyType, {}},
|
||||
{kObjectTypeEnvType, {}}};
|
||||
return method_map;
|
||||
}
|
||||
// A map that maps number types to a map of operator names and their corresponding primitive operations
|
||||
// For example, for number type UInt, the map contains operator names and their corresponding primitive operations
|
||||
// The primitive operations are represented by enum values from the prim namespace
|
||||
{
|
||||
kNumberTypeUInt,
|
||||
{
|
||||
{"__add__", prim::kPrimScalarAdd}, // P.scalar_add,
|
||||
{"__sub__", prim::kPrimScalarSub}, // P.scalar_sub,
|
||||
{"__mul__", prim::kPrimScalarMul}, // P.scalar_mul,
|
||||
{"__floordiv__", prim::kPrimScalarDiv}, // P.scalar_div,
|
||||
{"__truediv__", std::string("int_truediv")}, // C.int_truediv
|
||||
{"__mod__", prim::kPrimScalarMod}, // P.scalar_mod,
|
||||
{"__pow__", prim::kPrimScalarPow}, // P.scalar_pow,
|
||||
{"__floor__", prim::kPrimIdentity}, // P.identity,
|
||||
{"__trunc__", prim::kPrimIdentity}, // P.identity,
|
||||
{"__pos__", prim::kPrimScalarUadd}, // P.scalar_uadd,
|
||||
{"__neg__", prim::kPrimScalarUsub}, // P.scalar_usub,
|
||||
{"__eq__", prim::kPrimScalarEq}, // P.scalar_eq,
|
||||
{"__ne__", prim::kPrimScalarNe}, // P.scalar_ne,
|
||||
{"__lt__", prim::kPrimScalarLt}, // P.scalar_lt,
|
||||
{"__gt__", prim::kPrimScalarGt}, // P.scalar_gt,
|
||||
{"__le__", prim::kPrimScalarLe}, // P.scalar_le,
|
||||
{"__ge__", prim::kPrimScalarGe}, // P.scalar_ge,
|
||||
{"__bool__", std::string("int_bool")}, // C.int_bool
|
||||
{"__ms_to_array__", prim::kPrimScalarToArray}, // P.scalar_to_array,
|
||||
}
|
||||
},
|
||||
{
|
||||
kNumberTypeFloat,
|
||||
{
|
||||
{"__add__", prim::kPrimScalarAdd}, // P.scalar_add,
|
||||
{"__sub__", prim::kPrimScalarSub}, // P.scalar_sub,
|
||||
{"__mul__", prim::kPrimScalarMul}, // P.scalar_mul,
|
||||
{"__floordiv__", std::string("float_floordiv")}, // C.float_floordiv
|
||||
{"__truediv__", prim::kPrimScalarDiv}, // P.scalar_div,
|
||||
{"__mod__", prim::kPrimScalarMod}, // P.scalar_mod,
|
||||
{"__pow__", prim::kPrimScalarPow}, // P.scalar_pow,
|
||||
{"__floor__", prim::kPrimScalarFloor}, // P.scalar_floor,
|
||||
{"__trunc__", prim::kPrimScalarTrunc}, // P.scalar_trunc,
|
||||
{"__pos__", prim::kPrimScalarUadd}, // P.scalar_uadd,
|
||||
{"__neg__", prim::kPrimScalarUsub}, // P.scalar_usub,
|
||||
{"__eq__", prim::kPrimScalarEq}, // P.scalar_eq,
|
||||
{"__ne__", prim::kPrimScalarNe}, // P.scalar_ne,
|
||||
{"__lt__", prim::kPrimScalarLt}, // P.scalar_lt,
|
||||
{"__gt__", prim::kPrimScalarGt}, // P.scalar_gt,
|
||||
// ... (continues with more operator names and their corresponding primitive operations)
|
||||
}
|
||||
}
|
||||
{
|
||||
// Mapping for object type kObjectTypeScalar
|
||||
kObjectTypeScalar: {
|
||||
{"__add__", prim::kPrimScalarAdd}, // P.scalar_add
|
||||
{"__sub__", prim::kPrimScalarSub}, // P.scalar_sub
|
||||
{"__mul__", prim::kPrimScalarMul}, // P.scalar_mul
|
||||
{"__div__", prim::kPrimScalarDiv}, // P.scalar_div
|
||||
{"__mod__", prim::kPrimScalarMod}, // P.scalar_mod
|
||||
{"__pow__", prim::kPrimScalarPow}, // P.scalar_pow
|
||||
{"__eq__", prim::kPrimScalarEq}, // P.scalar_eq
|
||||
{"__ne__", prim::kPrimScalarNe}, // P.scalar_ne
|
||||
{"__lt__", prim::kPrimScalarLt}, // P.scalar_lt
|
||||
{"__gt__", prim::kPrimScalarGt}, // P.scalar_gt
|
||||
{"__le__", prim::kPrimScalarLe}, // P.scalar_le
|
||||
{"__ge__", prim::kPrimScalarGe}, // P.scalar_ge
|
||||
{"__bool__", std::string("float_bool")}, // C.float_bool
|
||||
{"__ms_to_array__", prim::kPrimScalarToArray} // P.scalar_to_array
|
||||
},
|
||||
// Mapping for object type kObjectTypeTuple
|
||||
kObjectTypeTuple: {
|
||||
{"__len__", prim::kPrimTupleLen}, // P.tuple_len
|
||||
{"__getitem__", prim::kPrimTupleGetItem}, // P.tuple_getitem
|
||||
{"__setitem__", prim::kPrimTupleSetItem}, // P.tuple_setitem
|
||||
{"__ms_iter__", prim::kPrimIdentity}, // P.identity
|
||||
{"__ms_next__", std::string("tuple_next")}, // C.tuple_next
|
||||
{"__ms_hasnext__", std::string("tuple_hasnext")}, // C.tuple_hasnext
|
||||
{"__bool__", std::string("tuple_bool")} // C.tuple_bool
|
||||
},
|
||||
// Mapping for object type kObjectTypeList
|
||||
kObjectTypeList: {
|
||||
{"__len__", prim::kPrimListLen}, // P.list_len
|
||||
{"__getitem__", prim::kPrimListGetItem}, // P.list_getitem
|
||||
{"__setitem__", prim::kPrimListSetItem}, // P.list_setitem
|
||||
{"__ms_iter__", prim::kPrimIdentity}, // P.identity
|
||||
{"__ms_next__", std::string("list_next")}, // C.list_next
|
||||
{"append", std::string("list_append")}, // C.list_append
|
||||
{"__bool__", std::string("list_bool")}, // C.list_bool
|
||||
{"__ms_hasnext__", std::string("list_hasnext")}, // C.list_hasnext
|
||||
{"insert", std::string("list_insert")} // C.list_insert
|
||||
},
|
||||
// Mapping for object type kObjectTypeDictionary
|
||||
kObjectTypeDictionary: {
|
||||
{"__len__", prim::kPrimDictLen}, // P.dict_len
|
||||
{"__getitem__", prim::kPrimDictGetItem}, // P.dict_getitem
|
||||
{"__setitem__", prim::kPrimDictSetItem}, // P.dict_setitem
|
||||
{"keys", prim::kPrimDictGetKeys}, // P.dict_getkeys
|
||||
{"values", prim::kPrimDictGetValues}, // P.dict_getvalues
|
||||
{"items", prim::kPrimDictItems}, // P.dict_items
|
||||
{"__bool__", std::string("dict_bool")} // C.dict_bool
|
||||
},
|
||||
// Mapping for object type kObjectTypeTensorType
|
||||
kObjectTypeTensorType: {
|
||||
{"all", std::string("all_")} // C.reduce_all
|
||||
}
|
||||
}
|
||||
// Mapping of Python functions to their corresponding C++ functions or operations
|
||||
|
||||
{"any", std::string("any_")}, // C.reduce_any
|
||||
{"__add__", std::string("add")}, // C.add
|
||||
{"__sub__", std::string("sub")}, // C.sub
|
||||
{"__mul__", std::string("mul")}, // C.mul
|
||||
{"abs", std::string("abs_")}, // C.abs_
|
||||
{"mean", std::string("mean")}, // C.mean
|
||||
{"__truediv__", std::string("truediv")}, // C.truediv
|
||||
{"__floordiv__", std::string("floordiv")}, // C.floordiv
|
||||
{"__mod__", std::string("mod")}, // C.mod
|
||||
{"__pow__", std::string("pow_")}, // C.pow
|
||||
{"__floor__", std::string("array_floor")}, // C.array_floor
|
||||
{"__trunc__", std::string("array_trunc")}, // C.array_trunc
|
||||
{"__pos__", std::string("array_uadd")}, // C.array_uadd
|
||||
{"__neg__", std::string("array_usub")}, // C.array_usub
|
||||
{"__eq__", std::string("eq")}, // C.eq
|
||||
{"__ne__", std::string("ne")}, // C.ne
|
||||
{"__lt__", std::string("lt")}, // C.lt
|
||||
{"__gt__", std::string("gt")}, // C.gt
|
||||
{"__le__", std::string("le")}, // C.le
|
||||
{"__ge__", std::string("ge")}, // C.ge
|
||||
{"expand_as", std::string("expand_tensor_as")}, // C.expand_as
|
||||
{"view", std::string("view")}, // C.view
|
||||
{"__len__", prim::kPrimArrayLen}, // P.array_len
|
||||
{"__getitem__", prim::kPrimArrayGetItem}, // P.array_getitem
|
||||
{"__setitem__", prim::kPrimArraySetItem}, // P.array_setitem
|
||||
{"__ms_iter__", std::string("array_iter")}, // C.array_iter
|
||||
{"__ms_to_array__", prim::kPrimIdentity}, // P.identity
|
||||
{"item", std::string("item")}, // P.item
|
||||
{"itemset", std::string("itemset")}, // P.itemset
|
||||
{"transpose", std::string("transpose")}, // P.transpose
|
||||
{"flatten", std::string("flatten")}, // P.reshape(,-1)
|
||||
{"reshape", std::string("reshape")}, // P.reshape()
|
||||
{"ravel", std::string("ravel")}, // P.reshape(,(-1,))
|
||||
{"swapaxes", std::string("swapaxes")}, // P.transpose()
|
||||
{"narrow", std::string("narrow")}, // narrow()
|
||||
{"masked_fill", std::string("masked_fill")}, // masked_fill()
|
||||
{"expand_dims", std::string("expand_dims")}, // P.expand_dims()
|
||||
{"squeeze", std::string("squeeze")}, // P.squeeze()
|
||||
{"astype", std::string("astype")}, // P.cast()
|
||||
{"cumsum", std::string("cumsum")} // P.cumsum()
|
||||
{
|
||||
kObjectTypeTensorType,
|
||||
{
|
||||
{"copy", std::string("copy")}, // copy()
|
||||
{"max", std::string("max")}, // P.reduce_max()
|
||||
{"min", std::string("min")}, // P.reduce_min()
|
||||
{"fill", std::string("fill")}, // P.fill()
|
||||
{"ptp", std::string("ptp")}, // P.reduce_max() - P.reduce_min()
|
||||
{"clip", std::string("clip")}, // P.maximum(P.minimum)
|
||||
{"__bool__", std::string("tensor_bool")}, // C.tensor_bool
|
||||
{"argmax", std::string("argmax")}, // P.Argmax()
|
||||
{"argmin", std::string("argmin")}, // P.Argmax()
|
||||
{"resize", std::string("resize")}, // P.Reshape()
|
||||
{"choose", std::string("choose")}, // P.Select()
|
||||
{"diagonal", std::string("diagonal")}, // P.Eye()
|
||||
{"searchsorted", std::string("searchsorted")}, // P.Select()
|
||||
{"take", std::string("take")}, // P.GatherNd()
|
||||
{"trace", std::string("trace")}, // P.Eye()
|
||||
{"var", std::string("var")}, // P.ReduceSum
|
||||
{"std", std::string("std")}, // P.ReduceSum
|
||||
{"sum", std::string("sum")}, // P.ReduceSum
|
||||
{"repeat", std::string("repeat")}, // C.repeat_elements
|
||||
}
|
||||
},
|
||||
{
|
||||
kObjectTypeRowTensorType,
|
||||
{
|
||||
{"__add__", prim::kPrimRowTensorAdd}, // P.row_tensor_add
|
||||
}
|
||||
},
|
||||
{
|
||||
kObjectTypeCSRTensorType,
|
||||
{
|
||||
{"astype", std::string("csr_astype")}, // C.csr_astype
|
||||
{"abs", std::string("csr_abs")}, // C.csr_abs
|
||||
{"sum", std::string("csr_sum")}, // C.csr_sum
|
||||
{"mv", std::string("csr_mv")}, // C.csr_mv
|
||||
{"to_tuple", std::string("csr_to_tuple")}, // C.csr_to_tuple
|
||||
{"to_coo", std::string("csr_to_coo")}, // C.csr_to_coo
|
||||
{"to_dense", std::string("csr_to_dense")}, // C.csr_to_dense
|
||||
}
|
||||
},
|
||||
{
|
||||
kObjectTypeCOOTensorType,
|
||||
{
|
||||
{"astype", std::string("coo_astype")}, // C.coo_astype
|
||||
{"abs", std::string("coo_abs")}, // C.coo_abs
|
||||
{"to_tuple", std::string("coo_to_tuple")}, // C.coo_to_tuple
|
||||
{"to_csr", std::string("coo_to_csr")}, // C.coo_to_csr
|
||||
// ...
|
||||
}
|
||||
}
|
||||
// Create a method map using an initializer list
|
||||
// The method map is a dictionary-like data structure that maps keys to values
|
||||
// Each key-value pair is enclosed in curly braces {}
|
||||
|
||||
// The method map contains four key-value pairs
|
||||
// The first key is of type kObjectTypeString and its value is "to_dense"
|
||||
// The value associated with this key is a std::string object with the value "coo_to_dense"
|
||||
// This key-value pair represents the mapping from "to_dense" to "coo_to_dense"
|
||||
|
||||
// The second key is of type kObjectTypeJTagged and its value is an empty dictionary
|
||||
// This key-value pair represents an empty mapping for kObjectTypeJTagged
|
||||
|
||||
// The third key is of type kObjectTypeSymbolicKeyType and its value is an empty dictionary
|
||||
// This key-value pair represents an empty mapping for kObjectTypeSymbolicKeyType
|
||||
|
||||
// The fourth key is of type kObjectTypeEnvType and its value is an empty dictionary
|
||||
// This key-value pair represents an empty mapping for kObjectTypeEnvType
|
||||
|
||||
// The method map is then returned from the function
|
||||
// The function signature indicates that the return type is the same as the type of the method_map variable, which is not specified in the provided code snippet
|
||||
|
||||
// Define a function named GetAttrMap that returns a reference to a BuiltInTypeMap object
|
||||
BuiltInTypeMap &GetAttrMap() {
|
||||
|
||||
// Define a static BuiltInTypeMap object named attr_map
|
||||
static BuiltInTypeMap attr_map = {
|
||||
|
||||
// For the key kObjectTypeTensorType, assign a map of attribute names and their corresponding primitive operations
|
||||
{kObjectTypeTensorType,
|
||||
{
|
||||
{"shape", prim::kPrimShape}, // Attribute name "shape" corresponds to primitive operation prim::kPrimShape
|
||||
{"dtype", prim::kPrimDType}, // Attribute name "dtype" corresponds to primitive operation prim::kPrimDType
|
||||
{"size", std::string("size_")}, // Attribute name "size" corresponds to a string "size_"
|
||||
{"ndim", std::string("ndim_")}, // Attribute name "ndim" corresponds to a string "ndim_"
|
||||
{"T", std::string("T_")}, // Attribute name "T" corresponds to a string "T_"
|
||||
{"itemsize", std::string("itemsize_")}, // Attribute name "itemsize" corresponds to a string "itemsize_"
|
||||
{"nbytes", std::string("nbytes_")}, // Attribute name "nbytes" corresponds to a string "nbytes_"
|
||||
{"strides", std::string("strides_")}, // Attribute name "strides" corresponds to a string "strides_"
|
||||
{"shape", prim::kPrimShape}, // C.shape_
|
||||
{"dtype", prim::kPrimDType}, // C.dtype_
|
||||
{"size", std::string("size_")}, // C.size_
|
||||
{"ndim", std::string("ndim_")}, // C.ndim_
|
||||
{"T", std::string("T_")}, // C.T_
|
||||
{"itemsize", std::string("itemsize_")}, // C.itemsize_
|
||||
{"nbytes", std::string("nbytes_")}, // C.nbytes_
|
||||
{"strides", std::string("strides_")}, // C.strides_
|
||||
}},
|
||||
|
||||
// For the key kObjectTypeRowTensorType, assign a map of attribute names and their corresponding primitive operations
|
||||
{kObjectTypeRowTensorType,
|
||||
{
|
||||
{"values", prim::kPrimRowTensorGetValues}, // Attribute name "values" corresponds to primitive operation prim::kPrimRowTensorGetValues
|
||||
{"indices", prim::kPrimRowTensorGetIndices}, // Attribute name "indices" corresponds to primitive operation prim::kPrimRowTensorGetIndices
|
||||
{"dense_shape", prim::kPrimRowTensorGetDenseShape}, // Attribute name "dense_shape" corresponds to primitive operation prim::kPrimRowTensorGetDenseShape
|
||||
{"values", prim::kPrimRowTensorGetValues}, // F.row_tensor_get_values
|
||||
{"indices", prim::kPrimRowTensorGetIndices}, // F.row_tensor_get_indices
|
||||
{"dense_shape", prim::kPrimRowTensorGetDenseShape}, // F.row_tensor_get_dense_shape
|
||||
}},
|
||||
|
||||
// For the key kObjectTypeCOOTensorType, assign a map of attribute names and their corresponding primitive operations
|
||||
{kObjectTypeCOOTensorType,
|
||||
{
|
||||
{"values", prim::kPrimCOOTensorGetValues}, // Attribute name "values" corresponds to primitive operation prim::kPrimCOOTensorGetValues
|
||||
{"indices", prim::kPrimCOOTensorGetIndices}, // Attribute name "indices" corresponds to primitive operation prim::kPrimCOOTensorGetIndices
|
||||
{"shape", prim::kPrimCOOTensorGetDenseShape}, // Attribute name "shape" corresponds to primitive operation prim::kPrimCOOTensorGetDenseShape
|
||||
{"dtype", std::string("dtype_")}, // Attribute name "dtype" corresponds to a string "dtype_"
|
||||
{"size", std::string("sparse_size_")}, // Attribute name "size" corresponds to a string "sparse_size_"
|
||||
{"ndim", std::string("sparse_ndim_")}, // Attribute name "ndim" corresponds to a string "sparse_ndim_"
|
||||
{"itemsize", std::string("itemsize_")}, // Attribute name "itemsize" corresponds to a string "itemsize_"
|
||||
{"values", prim::kPrimCOOTensorGetValues}, // F.coo_tensor_get_values
|
||||
{"indices", prim::kPrimCOOTensorGetIndices}, // F.coo_tensor_get_indices
|
||||
{"shape", prim::kPrimCOOTensorGetDenseShape}, // F.coo_tensor_get_dense_shape
|
||||
{"dtype", std::string("dtype_")}, // C.dtype_
|
||||
{"size", std::string("sparse_size_")}, // C.sparse_size_
|
||||
{"ndim", std::string("sparse_ndim_")}, // C.sparse_ndim_
|
||||
{"itemsize", std::string("itemsize_")}, // C.itemsize_
|
||||
}},
|
||||
|
||||
// For the key kObjectTypeCSRTensorType, assign a map of attribute names and their corresponding primitive operations
|
||||
{kObjectTypeCSRTensorType,
|
||||
{
|
||||
{"indptr", prim::kPrimCSRTensorGetIndptr}, // Attribute name "indptr" corresponds to primitive operation prim::kPrimCSRTensorGetIndptr
|
||||
{"values", prim::kPrimCSRTensorGetValues}, // Attribute name "values" corresponds to primitive operation prim::kPrimCSRTensorGetValues
|
||||
{"indices", prim::kPrimCSRTensorGetIndices}, // Attribute name "indices" corresponds to primitive operation prim::kPrimCSRTensorGetIndices
|
||||
{"shape", prim::kPrimCSRTensorGetDenseShape}, // Attribute name "shape" corresponds to primitive operation prim::kPrimCSRTensorGetDenseShape
|
||||
{"dtype", std::string("dtype_")}, // Attribute name "dtype" corresponds to a string "dtype_"
|
||||
{"size", std::string("sparse_size_")}, // Attribute name "size" corresponds to a string "sparse_size_"
|
||||
{"ndim", std::string("sparse_ndim_")}, // Attribute name "ndim" corresponds to a string "sparse_ndim_"
|
||||
{"itemsize", std::string("itemsize_")}, // Attribute name "itemsize" corresponds to a string "itemsize_"
|
||||
{"indptr", prim::kPrimCSRTensorGetIndptr}, // F.csr_tensor_get_indptr
|
||||
{"values", prim::kPrimCSRTensorGetValues}, // F.csr_tensor_get_values
|
||||
{"indices", prim::kPrimCSRTensorGetIndices}, // F.csr_tensor_get_indices
|
||||
{"shape", prim::kPrimCSRTensorGetDenseShape}, // F.csr_tensor_get_shape
|
||||
{"dtype", std::string("dtype_")}, // C.dtype_
|
||||
{"size", std::string("sparse_size_")}, // C.sparse_size_
|
||||
{"ndim", std::string("sparse_ndim_")}, // C.sparse_ndim_
|
||||
{"itemsize", std::string("itemsize_")}, // C.itemsize_
|
||||
}},
|
||||
};
|
||||
|
||||
// Return a reference to the attr_map object
|
||||
return attr_map;
|
||||
}
|
||||
};
|
||||
|
||||
// Return the attribute map
|
||||
return attr_map;
|
||||
}
|
||||
|
||||
// Define the constructor for the Resource class, which takes a py::object as input
|
||||
Resource::Resource(const py::object &obj)
|
||||
: engine_(std::make_shared<abstract::AnalysisEngine>(abstract::GetPrimEvaluatorConstructors(), manager_)), // Initialize the engine_ member variable with a shared pointer to an AnalysisEngine object
|
||||
source_input_(obj), // Initialize the source_input_ member variable with the provided py::object
|
||||
is_cleaned_(false) {} // Initialize the is_cleaned_ member variable to false
|
||||
: engine_(std::make_shared<abstract::AnalysisEngine>(abstract::GetPrimEvaluatorConstructors(), manager_)),
|
||||
source_input_(obj),
|
||||
is_cleaned_(false) {}
|
||||
|
||||
// Destructor for the Resource class
|
||||
Resource::~Resource() {
|
||||
|
||||
// Output a debug message using the MS_LOG macro to indicate that the resource is being cleared
|
||||
MS_LOG(DEBUG) << "Resource clear";
|
||||
}
|
||||
MS_LOG(DEBUG) << "Resource clear";
|
||||
|
||||
try {
|
||||
// Create an empty HashMap object of type <std::string, Any> and swap it with the existing results_ HashMap
|
||||
mindspore::HashMap<std::string, Any>().swap(results_);
|
||||
} catch (const std::exception &e) {
|
||||
// If an exception occurs during the swap operation, catch it and log an error message
|
||||
MS_LOG(ERROR) << "Exception when cleaning resource. Error info " << e.what();
|
||||
}
|
||||
|
||||
// If the program exits normally, these global variables will be cleaned up
|
||||
// in the Resource::Clean function called by MsPipeline::Compile. However, if the program
|
||||
// exits with an MS_LOGEXCEPTION, these global variables may not be cleaned up properly.
|
||||
// This can cause a segmentation fault when trying to free Python objects inside these
|
||||
// global variables after the Python interpreter has been freed. To prevent this,
|
||||
// these global variables are cleaned up here.
|
||||
// However, if the program exits normally, these global variables will be cleaned up twice.
|
||||
// Care should be taken to prevent double freeing in the following functions.
|
||||
|
||||
// Check if the global variables have already been cleaned up
|
||||
// If exit normally, these global variables will be cleaned
|
||||
// in Resource::Clean call by MsPipeline::Compile, but if exit with MS_LOGEXCEPTION,
|
||||
// these global variables may not being cleaned, it may
|
||||
// cause segmentfault when free python object inside these global variables
|
||||
// after python interpreter got freed, so these global variables
|
||||
// are cleaned here.
|
||||
// So if exit normally, these global variable will be cleaned twice,
|
||||
// care be taken to prevent double free in the following functions.
|
||||
if (!is_cleaned_) {
|
||||
try {
|
||||
// Call the Clean function to clean up the global variables
|
||||
Clean();
|
||||
} catch (const std::exception &e) {
|
||||
// Log an error message if an exception occurs during the cleaning process
|
||||
MS_LOG(ERROR) << "Exception when cleaning resource. Error info " << e.what();
|
||||
} catch (...) {
|
||||
// Log an error message if an unknown exception occurs during the cleaning process
|
||||
MS_LOG(ERROR) << "Exception when cleaning resource.";
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// This function takes in three parameters: name (a string), type_id (a TypeId object), and method_map (a BuiltInTypeMap object)
|
||||
Any GetMethodOrAttr(const string &name, const TypeId &type_id, const BuiltInTypeMap &method_map) {
|
||||
|
||||
// Find the entry in the method_map corresponding to the given type_id
|
||||
auto type_method_map = method_map.find(static_cast<int64_t>(type_id));
|
||||
|
||||
// If no entry is found, return an empty Any object
|
||||
if (type_method_map == method_map.end()) {
|
||||
return Any();
|
||||
}
|
||||
|
||||
// Find the entry in the type_method_map corresponding to the given name
|
||||
auto method = type_method_map->second.find(name);
|
||||
|
||||
// If no entry is found, return an empty Any object
|
||||
if (method == type_method_map->second.end()) {
|
||||
return Any();
|
||||
}
|
||||
|
||||
// Return the value associated with the found entry
|
||||
return method->second;
|
||||
}
|
||||
|
||||
// Check if the given type is present in the built-in map
|
||||
bool Resource::IsTypeInBuiltInMap(const TypeId &type) {
|
||||
|
||||
// Normalize the given type ID
|
||||
TypeId type_id = NormalizeTypeId(type);
|
||||
|
||||
// Get the method map from the resource
|
||||
const BuiltInTypeMap &method_map = GetMethodMap();
|
||||
|
||||
// Find the type ID in the method map
|
||||
auto iter = method_map.find(static_cast<int64_t>(type_id));
|
||||
|
||||
// If the type ID is not found in the method map
|
||||
if (iter == method_map.end()) {
|
||||
|
||||
// Get the attribute map from the resource
|
||||
const BuiltInTypeMap &attr_map = GetAttrMap();
|
||||
|
||||
// Find the type ID in the attribute map
|
||||
iter = attr_map.find(static_cast<int64_t>(type_id));
|
||||
|
||||
// If the type ID is not found in the attribute map, return false
|
||||
if (iter == attr_map.end()) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// If the type ID is found in either the method map or the attribute map, return true
|
||||
return true;
|
||||
}
|
||||
|
||||
// A function named "Resource::GetMethodPtr" that takes two parameters: a reference to a TypeId object named "type" and a reference to a std::string object named "name"
|
||||
Resource::GetMethodPtr(const TypeId &type, const std::string &name) {
|
||||
|
||||
// Normalize the given TypeId object by calling the "NormalizeTypeId" function and store the result in a local variable named "type_id"
|
||||
Any Resource::GetMethodPtr(const TypeId &type, const std::string &name) {
|
||||
TypeId type_id = NormalizeTypeId(type);
|
||||
|
||||
// Get the method map by calling the "GetMethodMap" function and store the result in a constant reference to a BuiltInTypeMap object named "method_map"
|
||||
const BuiltInTypeMap &method_map = GetMethodMap();
|
||||
|
||||
// Call the "GetMethodOrAttr" function with the given "name", "type_id", and "method_map" as arguments and return the result
|
||||
return GetMethodOrAttr(name, type_id, method_map);
|
||||
}
|
||||
|
||||
// A function named "Resource::GetAttrPtr" that takes two parameters: a reference to a TypeId object named "type" and a reference to a std::string object named "name"
|
||||
Resource::GetAttrPtr(const TypeId &type, const std::string &name) {
|
||||
|
||||
// Normalize the given TypeId object by calling the "NormalizeTypeId" function and store the result in a local variable named "type_id"
|
||||
Any Resource::GetAttrPtr(const TypeId &type, const std::string &name) {
|
||||
TypeId type_id = NormalizeTypeId(type);
|
||||
|
||||
// Get the attribute map by calling the "GetAttrMap" function and store the result in a constant reference to a BuiltInTypeMap object named "attr_map"
|
||||
const BuiltInTypeMap &attr_map = GetAttrMap();
|
||||
|
||||
// Call the "GetMethodOrAttr" function with the given "name", "type_id", and "attr_map" as parameters and return the result
|
||||
return GetMethodOrAttr(name, type_id, attr_map);
|
||||
}
|
||||
|
||||
// Define the function `GetCompileCacheResource` which takes in several parameters
|
||||
void Resource::GetCompileCacheResource(const py::list &compile_cache_dep_files, const py::dict &weights,
|
||||
const std::string &queue_name, size_t compile_cache_id,
|
||||
bool *compile_cache_consistent) {
|
||||
|
||||
// Create a shared pointer to a `CompileCacheManager` object and initialize it with the given `compile_cache_id`
|
||||
compile_cache_manager_ = std::make_shared<CompileCacheManager>(compile_cache_id);
|
||||
|
||||
// Initialize the parallel group checkpoint save file for the `compile_cache_manager_`
|
||||
compile_cache_manager_->InitParallelGroupCkptSaveFile();
|
||||
|
||||
// Check if the pointer `compile_cache_consistent` is not null
|
||||
MS_EXCEPTION_IF_NULL(compile_cache_consistent);
|
||||
|
||||
// Check if the value pointed to by `compile_cache_consistent` is false
|
||||
if (!*compile_cache_consistent) {
|
||||
// Print a warning message indicating that the consistency of dependency files hash failed
|
||||
MS_LOG(WARNING) << "Check the consistency of dependency files hash failed. Execute all the compilation actions.";
|
||||
|
||||
// Return from the function
|
||||
return;
|
||||
}
|
||||
|
||||
// Initialize the compile cache hash for the `compile_cache_manager_` using the `compile_cache_dep_files`
|
||||
compile_cache_manager_->InitCompileCacheHash(compile_cache_dep_files);
|
||||
|
||||
// Check the consistency of dependency files hash using the `compile_cache_manager_`
|
||||
*compile_cache_consistent = compile_cache_manager_->CheckDepFilesHashConsistency();
|
||||
|
||||
// Check if the value pointed to by `compile_cache_consistent` is false
|
||||
if (!*compile_cache_consistent) {
|
||||
// Print a warning message indicating that the consistency of dependency files hash failed
|
||||
MS_LOG(WARNING) << "Check the consistency of dependency files hash failed. Execute all the compilation actions.";
|
||||
|
||||
// Return from the function
|
||||
return;
|
||||
}
|
||||
|
||||
// Get the cached function graph from the `compile_cache_manager_` using the `manager_`, `weights`, and `queue_name`
|
||||
func_graph_ = compile_cache_manager_->GetCachedFuncGraph(manager_, weights, queue_name);
|
||||
|
||||
// Get the layout map from the `compile_cache_manager_`
|
||||
layout_map_ = compile_cache_manager_->layout_map();
|
||||
}
|
||||
|
||||
// Define the function `CacheFuncGraph` in the `Resource` class
|
||||
void Resource::CacheFuncGraph() const {
|
||||
|
||||
// Initialize a null pointer to `FuncGraphPtr`
|
||||
FuncGraphPtr layout_fg = nullptr;
|
||||
|
||||
// Get the parallel mode from the `ParallelContext` singleton instance
|
||||
std::string parallel_mode = parallel::ParallelContext::GetInstance()->parallel_mode();
|
||||
|
||||
// Check if the `func_graph_` has the `kAutoParallel` flag and if the parallel mode is either `kAutoParallel` or `kSemiAutoParallel`
|
||||
if (func_graph_->has_flag(parallel::kAutoParallel) &&
|
||||
((parallel_mode == parallel::kAutoParallel) || (parallel_mode == parallel::kSemiAutoParallel))) {
|
||||
|
||||
// If the conditions are met, assign the result of `GetResult(kStepParallelGraph)` casted to `FuncGraphPtr` to `layout_fg`
|
||||
layout_fg = GetResult(kStepParallelGraph).cast<FuncGraphPtr>();
|
||||
}
|
||||
|
||||
// Call the `CacheFuncGraph` function of `compile_cache_manager_` with `func_graph_` and `layout_fg` as arguments
|
||||
compile_cache_manager_->CacheFuncGraph(func_graph_, layout_fg);
|
||||
}
|
||||
|
||||
void Resource::Clean() {
|
||||
// Clear the args_spec_ vector
|
||||
args_spec_.clear();
|
||||
|
||||
// Set the source_input_ to None
|
||||
source_input_ = py::none();
|
||||
|
||||
// Check if the engine_ is not null, then clear the evaluator cache
|
||||
MS_EXCEPTION_IF_NULL(engine_);
|
||||
engine_->ClearEvaluatorCache();
|
||||
|
||||
// Clear the object cache used for parsing
|
||||
parse::data_converter::ClearObjectCache();
|
||||
|
||||
// Clean up the resources used by the parser
|
||||
parse::Parser::CleanParserResource();
|
||||
|
||||
// Clean the data class to class map
|
||||
parse::CleanDataClassToClassMap();
|
||||
|
||||
// Clear the trace stack
|
||||
trace::ClearTraceStack();
|
||||
|
||||
// Set the is_cleaned_ flag to true
|
||||
is_cleaned_ = true;
|
||||
// AbstractTensor->elements() will be saved in AbstractBasePtrList
|
||||
args_spec_.clear();
|
||||
source_input_ = py::none();
|
||||
// Context with AbstractBasePtrList may be saved in GraphEvaluator
|
||||
// some Evaluator like ResolveEvaluator may save Python object in cache,
|
||||
// it should be cleaned before Python Interpreter destructed.
|
||||
MS_EXCEPTION_IF_NULL(engine_);
|
||||
engine_->ClearEvaluatorCache();
|
||||
// Clean cache used for parse. As static variable is released after
|
||||
// Python threads is released.
|
||||
parse::data_converter::ClearObjectCache();
|
||||
parse::Parser::CleanParserResource();
|
||||
parse::CleanDataClassToClassMap();
|
||||
trace::ClearTraceStack();
|
||||
is_cleaned_ = true;
|
||||
}
|
||||
|
||||
} // namespace pipeline
|
||||
} // namespace mindspore
|
||||
|
||||
// Closing braces to end the namespace declarations for "pipeline" and "mindspore"
|
||||
File diff suppressed because it is too large
Load Diff
Loading…
Reference in New Issue